From ff6ef1bd07fa68307b7c82c910416d770e7b3416 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:47:13 -0600 Subject: [PATCH 001/126] fix: strip trailing _ from field mask paths (#228) --- google/api_core/protobuf_helpers.py | 7 +++++ noxfile.py | 5 ++-- tests/unit/test_protobuf_helpers.py | 44 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py index 365ef25c..8aff79aa 100644 --- a/google/api_core/protobuf_helpers.py +++ b/google/api_core/protobuf_helpers.py @@ -357,6 +357,13 @@ def _field_mask_helper(original, modified, current=""): def _get_path(current, name): + # gapic-generator-python appends underscores to field names + # that collide with python keywords. + # `_` is stripped away as it is not possible to + # natively define a field with a trailing underscore in protobuf. + # APIs will reject field masks if fields have trailing underscores. + # See https://github.com/googleapis/python-api-core/issues/227 + name = name.rstrip("_") if not current: return name return "%s.%s" % (current, name) diff --git a/noxfile.py b/noxfile.py index 25609920..2f11137d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -98,9 +98,10 @@ def default(session): ] pytest_args.extend(session.posargs) - # Inject AsyncIO content, if version >= 3.6. + # Inject AsyncIO content and proto-plus, if version >= 3.6. + # proto-plus is needed for a field mask test in test_protobuf_helpers.py if _greater_or_equal_than_36(session.python): - session.install("asyncmock", "pytest-asyncio") + session.install("asyncmock", "pytest-asyncio", "proto-plus") pytest_args.append("--cov=tests.asyncio") pytest_args.append(os.path.join("tests", "asyncio")) diff --git a/tests/unit/test_protobuf_helpers.py b/tests/unit/test_protobuf_helpers.py index db972383..3df45df5 100644 --- a/tests/unit/test_protobuf_helpers.py +++ b/tests/unit/test_protobuf_helpers.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import pytest from google.api import http_pb2 @@ -472,3 +474,45 @@ def test_field_mask_different_level_diffs(): "alpha", "red", ] + + +@pytest.mark.skipif( + sys.version_info.major == 2, + reason="Field names with trailing underscores can only be created" + "through proto-plus, which is Python 3 only.", +) +def test_field_mask_ignore_trailing_underscore(): + import proto + + class Foo(proto.Message): + type_ = proto.Field(proto.STRING, number=1) + input_config = proto.Field(proto.STRING, number=2) + + modified = Foo(type_="bar", input_config="baz") + + assert sorted(protobuf_helpers.field_mask(None, Foo.pb(modified)).paths) == [ + "input_config", + "type", + ] + + +@pytest.mark.skipif( + sys.version_info.major == 2, + reason="Field names with trailing underscores can only be created" + "through proto-plus, which is Python 3 only.", +) +def test_field_mask_ignore_trailing_underscore_with_nesting(): + import proto + + class Bar(proto.Message): + class Baz(proto.Message): + input_config = proto.Field(proto.STRING, number=1) + + type_ = proto.Field(Baz, number=1) + + modified = Bar() + modified.type_.input_config = "foo" + + assert sorted(protobuf_helpers.field_mask(None, Bar.pb(modified)).paths) == [ + "type.input_config", + ] From a30f004e74f709d46e905dd819c71f43354e9ac9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 3 Aug 2021 13:59:25 -0400 Subject: [PATCH 002/126] fix!: drop support for Python 2.7 / 3.5 (#212) Drop 'six' module Drop 'u"' prefixes for text Remove other Python 2.7 workarounds Drop use of 'pytz' Dxpand range to allow 'google-auth' 2.x versions Remove 'general_helpers.wraps': except for a backward-compatibility import, 'functools.wraps' does everything wee need on Python >= 3.6. Remove 'packaging' dependency Release-As: 2.0.0b1 Closes #74. Closes #215. --- CONTRIBUTING.rst | 34 ++------------- README.rst | 15 ++++--- docs/auth.rst | 19 -------- google/api_core/bidi.py | 10 ++--- google/api_core/client_info.py | 2 +- google/api_core/client_options.py | 2 +- google/api_core/datetime_helpers.py | 20 +++++---- google/api_core/exceptions.py | 47 +++++++++----------- google/api_core/future/base.py | 5 +-- google/api_core/gapic_v1/__init__.py | 20 ++++----- google/api_core/gapic_v1/client_info.py | 2 +- google/api_core/gapic_v1/config.py | 11 ++--- google/api_core/gapic_v1/method.py | 19 ++++---- google/api_core/gapic_v1/method_async.py | 6 ++- google/api_core/gapic_v1/routing_header.py | 7 +-- google/api_core/general_helpers.py | 21 +-------- google/api_core/grpc_helpers.py | 22 ++++------ google/api_core/iam.py | 8 +--- google/api_core/operations_v1/__init__.py | 8 +--- google/api_core/page_iterator.py | 17 ++------ google/api_core/path_template.py | 6 +-- google/api_core/protobuf_helpers.py | 20 ++++----- google/api_core/retry.py | 16 +++---- google/api_core/timeout.py | 10 ++--- google/api_core/version.py | 2 +- noxfile.py | 5 +-- owlbot.py | 11 ++++- setup.py | 10 +---- testing/constraints-2.7.txt | 1 - testing/constraints-3.6.txt | 1 - tests/unit/test_bidi.py | 10 +---- tests/unit/test_datetime_helpers.py | 51 ++++++++++++---------- tests/unit/test_exceptions.py | 32 +++++++------- tests/unit/test_general_helpers.py | 41 ----------------- tests/unit/test_page_iterator.py | 29 ++++++------ 35 files changed, 191 insertions(+), 349 deletions(-) delete mode 100644 testing/constraints-2.7.txt delete mode 100644 tests/unit/test_general_helpers.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c7860adb..358404f2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,4 +1,3 @@ -.. Generated by synthtool. DO NOT EDIT! ############ Contributing ############ @@ -22,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 2.7, 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -77,8 +76,8 @@ We use `nox `__ to instrument our tests. .. note:: - The unit tests and system tests are described in the - ``noxfile.py`` files in each directory. + The unit tests tests are described in the ``noxfile.py`` files + in each directory. .. nox: https://pypi.org/project/nox/ @@ -133,29 +132,6 @@ Exceptions to PEP8: "Function-Under-Test"), which is PEP8-incompliant, but more readable. Some also use a local variable, ``MUT`` (short for "Module-Under-Test"). -******************** -Running System Tests -******************** - -- To run system tests, you can execute:: - - # Run all system tests - $ nox -s system - - # Run a single system test - $ nox -s system-3.8 -- -k - - - .. note:: - - System tests are only configured to run under Python 2.7 and 3.8. - For expediency, we do not run them in older versions of Python 3. - - This alone will not run the tests. You'll need to change some local - auth settings and change some configuration in your project to - run all the tests. - -- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication `__. Some tests require a service account. For those tests see `Authenticating as a service account `__. ************* Test Coverage @@ -221,13 +197,11 @@ Supported Python Versions We support: -- `Python 2.7`_ - `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ -.. _Python 2.7: https://docs.python.org/2.7/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ @@ -239,7 +213,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-api-core/blob/master/noxfile.py -We also explicitly decided to support Python 3 beginning with version 2.7. +We also explicitly decided to support Python 3 beginning with version 3.6. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/README.rst b/README.rst index 244043ea..d94f3e88 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Core Library for Google Client Libraries ======================================== -|pypi| |versions| +|pypi| |versions| This library is not meant to stand-alone. Instead it defines common helpers used by all Google API clients. For more information, see the @@ -16,8 +16,13 @@ common helpers used by all Google API clients. For more information, see the Supported Python Versions ------------------------- -Python >= 3.5 +Python >= 3.6 -Deprecated Python Versions --------------------------- -Python == 2.7. Python 2.7 support will be removed on January 1, 2020. + +Unsupported Python Versions +--------------------------- + +Python == 2.7, Python == 3.5. + +The last version of this library compatible with Python 2.7 and 3.5 is +`google-api_core==1.31.1`. diff --git a/docs/auth.rst b/docs/auth.rst index faf0228f..a9b296d4 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -103,25 +103,6 @@ After creation, you can pass it directly to a :class:`Client ` -just for Google App Engine: - -.. code:: python - - from google.auth import app_engine - credentials = app_engine.Credentials() - Google Compute Engine Environment --------------------------------- diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index be52d97d..56a021a9 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -17,11 +17,10 @@ import collections import datetime import logging +import queue as queue_module import threading import time -from six.moves import queue - from google.api_core import exceptions _LOGGER = logging.getLogger(__name__) @@ -71,7 +70,7 @@ class _RequestQueueGenerator(object): CPU consumed by spinning is pretty minuscule. Args: - queue (queue.Queue): The request queue. + queue (queue_module.Queue): The request queue. period (float): The number of seconds to wait for items from the queue before checking if the RPC is cancelled. In practice, this determines the maximum amount of time the request consumption @@ -108,7 +107,7 @@ def __iter__(self): while True: try: item = self._queue.get(timeout=self._period) - except queue.Empty: + except queue_module.Empty: if not self._is_active(): _LOGGER.debug( "Empty queue and inactive call, exiting request " "generator." @@ -247,7 +246,7 @@ def __init__(self, start_rpc, initial_request=None, metadata=None): self._start_rpc = start_rpc self._initial_request = initial_request self._rpc_metadata = metadata - self._request_queue = queue.Queue() + self._request_queue = queue_module.Queue() self._request_generator = None self._is_active = False self._callbacks = [] @@ -645,6 +644,7 @@ def _thread_main(self, ready): # Keeping the lock throughout avoids that. # In the future, we could use `Condition.wait_for` if we drop # Python 2.7. + # See: https://github.com/googleapis/python-api-core/issues/211 with self._wake: while self._paused: _LOGGER.debug("paused, waiting for waking.") diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index adca5f32..d7f4367a 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -42,7 +42,7 @@ class ClientInfo(object): Args: python_version (str): The Python interpreter version, for example, - ``'2.7.13'``. + ``'3.9.6'``. grpc_version (Optional[str]): The gRPC library version. api_core_version (str): The google-api-core library version. gapic_version (Optional[str]): The sversion of gapic-generated client diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index 57000e95..be5523df 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -101,7 +101,7 @@ def from_dict(options): """Construct a client options object from a mapping object. Args: - options (six.moves.collections_abc.Mapping): A mapping object with client options. + options (collections.abc.Mapping): A mapping object with client options. See the docstring for ClientOptions for details on valid arguments. """ diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py index e52fb1dd..78268efc 100644 --- a/google/api_core/datetime_helpers.py +++ b/google/api_core/datetime_helpers.py @@ -18,12 +18,10 @@ import datetime import re -import pytz - from google.protobuf import timestamp_pb2 -_UTC_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) +_UTC_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) _RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ" _RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S" # datetime.strptime cannot handle nanosecond precision: parse w/ regex @@ -83,9 +81,9 @@ def to_microseconds(value): int: Microseconds since the unix epoch. """ if not value.tzinfo: - value = value.replace(tzinfo=pytz.utc) + value = value.replace(tzinfo=datetime.timezone.utc) # Regardless of what timezone is on the value, convert it to UTC. - value = value.astimezone(pytz.utc) + value = value.astimezone(datetime.timezone.utc) # Convert the datetime to a microsecond timestamp. return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond @@ -156,7 +154,7 @@ def from_rfc3339(value): nanos = int(fraction) * (10 ** scale) micros = nanos // 1000 - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) from_rfc3339_nanos = from_rfc3339 # from_rfc3339_nanos method was deprecated. @@ -256,7 +254,7 @@ def from_rfc3339(cls, stamp): bare.minute, bare.second, nanosecond=nanos, - tzinfo=pytz.UTC, + tzinfo=datetime.timezone.utc, ) def timestamp_pb(self): @@ -265,7 +263,11 @@ def timestamp_pb(self): Returns: (:class:`~google.protobuf.timestamp_pb2.Timestamp`): Timestamp message """ - inst = self if self.tzinfo is not None else self.replace(tzinfo=pytz.UTC) + inst = ( + self + if self.tzinfo is not None + else self.replace(tzinfo=datetime.timezone.utc) + ) delta = inst - _UTC_EPOCH seconds = int(delta.total_seconds()) nanos = self._nanosecond or self.microsecond * 1000 @@ -292,5 +294,5 @@ def from_timestamp_pb(cls, stamp): bare.minute, bare.second, nanosecond=stamp.nanos, - tzinfo=pytz.UTC, + tzinfo=datetime.timezone.utc, ) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 412fc2ee..13be917e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -21,8 +21,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -import six -from six.moves import http_client +import http.client try: import grpc @@ -56,7 +55,6 @@ class DuplicateCredentialArgs(GoogleAPIError): pass -@six.python_2_unicode_compatible class RetryError(GoogleAPIError): """Raised when a function has exhausted all of its available retries. @@ -92,9 +90,7 @@ def __new__(mcs, name, bases, class_dict): return cls -@six.python_2_unicode_compatible -@six.add_metaclass(_GoogleAPICallErrorMeta) -class GoogleAPICallError(GoogleAPIError): +class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): """Base class for exceptions raised by calling API methods. Args: @@ -153,25 +149,25 @@ class Redirection(GoogleAPICallError): class MovedPermanently(Redirection): """Exception mapping a ``301 Moved Permanently`` response.""" - code = http_client.MOVED_PERMANENTLY + code = http.client.MOVED_PERMANENTLY class NotModified(Redirection): """Exception mapping a ``304 Not Modified`` response.""" - code = http_client.NOT_MODIFIED + code = http.client.NOT_MODIFIED class TemporaryRedirect(Redirection): """Exception mapping a ``307 Temporary Redirect`` response.""" - code = http_client.TEMPORARY_REDIRECT + code = http.client.TEMPORARY_REDIRECT class ResumeIncomplete(Redirection): """Exception mapping a ``308 Resume Incomplete`` response. - .. note:: :attr:`http_client.PERMANENT_REDIRECT` is ``308``, but Google + .. note:: :attr:`http.client.PERMANENT_REDIRECT` is ``308``, but Google APIs differ in their use of this status code. """ @@ -185,7 +181,7 @@ class ClientError(GoogleAPICallError): class BadRequest(ClientError): """Exception mapping a ``400 Bad Request`` response.""" - code = http_client.BAD_REQUEST + code = http.client.BAD_REQUEST class InvalidArgument(BadRequest): @@ -210,7 +206,7 @@ class OutOfRange(BadRequest): class Unauthorized(ClientError): """Exception mapping a ``401 Unauthorized`` response.""" - code = http_client.UNAUTHORIZED + code = http.client.UNAUTHORIZED class Unauthenticated(Unauthorized): @@ -222,7 +218,7 @@ class Unauthenticated(Unauthorized): class Forbidden(ClientError): """Exception mapping a ``403 Forbidden`` response.""" - code = http_client.FORBIDDEN + code = http.client.FORBIDDEN class PermissionDenied(Forbidden): @@ -235,20 +231,20 @@ class NotFound(ClientError): """Exception mapping a ``404 Not Found`` response or a :attr:`grpc.StatusCode.NOT_FOUND` error.""" - code = http_client.NOT_FOUND + code = http.client.NOT_FOUND grpc_status_code = grpc.StatusCode.NOT_FOUND if grpc is not None else None class MethodNotAllowed(ClientError): """Exception mapping a ``405 Method Not Allowed`` response.""" - code = http_client.METHOD_NOT_ALLOWED + code = http.client.METHOD_NOT_ALLOWED class Conflict(ClientError): """Exception mapping a ``409 Conflict`` response.""" - code = http_client.CONFLICT + code = http.client.CONFLICT class AlreadyExists(Conflict): @@ -266,26 +262,25 @@ class Aborted(Conflict): class LengthRequired(ClientError): """Exception mapping a ``411 Length Required`` response.""" - code = http_client.LENGTH_REQUIRED + code = http.client.LENGTH_REQUIRED class PreconditionFailed(ClientError): """Exception mapping a ``412 Precondition Failed`` response.""" - code = http_client.PRECONDITION_FAILED + code = http.client.PRECONDITION_FAILED class RequestRangeNotSatisfiable(ClientError): """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" - code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE + code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE class TooManyRequests(ClientError): """Exception mapping a ``429 Too Many Requests`` response.""" - # http_client does not define a constant for this in Python 2. - code = 429 + code = http.client.TOO_MANY_REQUESTS class ResourceExhausted(TooManyRequests): @@ -312,7 +307,7 @@ class InternalServerError(ServerError): """Exception mapping a ``500 Internal Server Error`` response. or a :attr:`grpc.StatusCode.INTERNAL` error.""" - code = http_client.INTERNAL_SERVER_ERROR + code = http.client.INTERNAL_SERVER_ERROR grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None @@ -332,28 +327,28 @@ class MethodNotImplemented(ServerError): """Exception mapping a ``501 Not Implemented`` response or a :attr:`grpc.StatusCode.UNIMPLEMENTED` error.""" - code = http_client.NOT_IMPLEMENTED + code = http.client.NOT_IMPLEMENTED grpc_status_code = grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None class BadGateway(ServerError): """Exception mapping a ``502 Bad Gateway`` response.""" - code = http_client.BAD_GATEWAY + code = http.client.BAD_GATEWAY class ServiceUnavailable(ServerError): """Exception mapping a ``503 Service Unavailable`` response or a :attr:`grpc.StatusCode.UNAVAILABLE` error.""" - code = http_client.SERVICE_UNAVAILABLE + code = http.client.SERVICE_UNAVAILABLE grpc_status_code = grpc.StatusCode.UNAVAILABLE if grpc is not None else None class GatewayTimeout(ServerError): """Exception mapping a ``504 Gateway Timeout`` response.""" - code = http_client.GATEWAY_TIMEOUT + code = http.client.GATEWAY_TIMEOUT class DeadlineExceeded(GatewayTimeout): diff --git a/google/api_core/future/base.py b/google/api_core/future/base.py index e7888ca3..f3005860 100644 --- a/google/api_core/future/base.py +++ b/google/api_core/future/base.py @@ -16,11 +16,8 @@ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class Future(object): +class Future(object, metaclass=abc.ABCMeta): # pylint: disable=missing-docstring # We inherit the interfaces here from concurrent.futures. diff --git a/google/api_core/gapic_v1/__init__.py b/google/api_core/gapic_v1/__init__.py index 6632047a..e5b7ad35 100644 --- a/google/api_core/gapic_v1/__init__.py +++ b/google/api_core/gapic_v1/__init__.py @@ -12,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from google.api_core.gapic_v1 import client_info from google.api_core.gapic_v1 import config +from google.api_core.gapic_v1 import config_async from google.api_core.gapic_v1 import method +from google.api_core.gapic_v1 import method_async from google.api_core.gapic_v1 import routing_header -__all__ = ["client_info", "config", "method", "routing_header"] - -if sys.version_info >= (3, 6): - from google.api_core.gapic_v1 import config_async # noqa: F401 - from google.api_core.gapic_v1 import method_async # noqa: F401 - - __all__.append("config_async") - __all__.append("method_async") +__all__ = [ + "client_info", + "config", + "config_async", + "method", + "method_async", + "routing_header", +] diff --git a/google/api_core/gapic_v1/client_info.py b/google/api_core/gapic_v1/client_info.py index bdc2ce44..fab0f542 100644 --- a/google/api_core/gapic_v1/client_info.py +++ b/google/api_core/gapic_v1/client_info.py @@ -33,7 +33,7 @@ class ClientInfo(client_info.ClientInfo): Args: python_version (str): The Python interpreter version, for example, - ``'2.7.13'``. + ``'3.9.6'``. grpc_version (Optional[str]): The gRPC library version. api_core_version (str): The google-api-core library version. gapic_version (Optional[str]): The sversion of gapic-generated client diff --git a/google/api_core/gapic_v1/config.py b/google/api_core/gapic_v1/config.py index 29e8645b..9c722871 100644 --- a/google/api_core/gapic_v1/config.py +++ b/google/api_core/gapic_v1/config.py @@ -21,7 +21,6 @@ import collections import grpc -import six from google.api_core import exceptions from google.api_core import retry @@ -130,24 +129,20 @@ def parse_method_configs(interface_config, retry_impl=retry.Retry): # Grab all the retry codes retry_codes_map = { name: retry_codes - for name, retry_codes in six.iteritems(interface_config.get("retry_codes", {})) + for name, retry_codes in interface_config.get("retry_codes", {}).items() } # Grab all of the retry params retry_params_map = { name: retry_params - for name, retry_params in six.iteritems( - interface_config.get("retry_params", {}) - ) + for name, retry_params in interface_config.get("retry_params", {}).items() } # Iterate through all the API methods and create a flat MethodConfig # instance for each one. method_configs = {} - for method_name, method_params in six.iteritems( - interface_config.get("methods", {}) - ): + for method_name, method_params in interface_config.get("methods", {}).items(): retry_params_name = method_params.get("retry_params_name") if retry_params_name is not None: diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py index 8bf82569..79722c07 100644 --- a/google/api_core/gapic_v1/method.py +++ b/google/api_core/gapic_v1/method.py @@ -18,7 +18,8 @@ pagination, and long-running operations to gRPC methods. """ -from google.api_core import general_helpers +import functools + from google.api_core import grpc_helpers from google.api_core import timeout from google.api_core.gapic_v1 import client_info @@ -110,26 +111,22 @@ def __init__(self, target, retry, timeout, metadata=None): self._timeout = timeout self._metadata = metadata - def __call__(self, *args, **kwargs): + def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs): """Invoke the low-level RPC with retry, timeout, and metadata.""" - # Note: Due to Python 2 lacking keyword-only arguments we use kwargs to - # extract the retry and timeout params. - timeout_ = _determine_timeout( + timeout = _determine_timeout( self._timeout, - kwargs.pop("timeout", self._timeout), + timeout, # Use only the invocation-specified retry only for this, as we only # want to adjust the timeout deadline if the *user* specified # a different retry. - kwargs.get("retry", None), + retry, ) - retry = kwargs.pop("retry", self._retry) - if retry is DEFAULT: retry = self._retry # Apply all applicable decorators. - wrapped_func = _apply_decorators(self._target, [retry, timeout_]) + wrapped_func = _apply_decorators(self._target, [retry, timeout]) # Add the user agent metadata to the call. if self._metadata is not None: @@ -237,7 +234,7 @@ def get_topic(name, timeout=None): else: user_agent_metadata = None - return general_helpers.wraps(func)( + return functools.wraps(func)( _GapicCallable( func, default_retry, default_timeout, metadata=user_agent_metadata ) diff --git a/google/api_core/gapic_v1/method_async.py b/google/api_core/gapic_v1/method_async.py index 76e57577..84c99aa2 100644 --- a/google/api_core/gapic_v1/method_async.py +++ b/google/api_core/gapic_v1/method_async.py @@ -17,7 +17,9 @@ pagination, and long-running operations to gRPC methods. """ -from google.api_core import general_helpers, grpc_helpers_async +import functools + +from google.api_core import grpc_helpers_async from google.api_core.gapic_v1 import client_info from google.api_core.gapic_v1.method import _GapicCallable from google.api_core.gapic_v1.method import DEFAULT # noqa: F401 @@ -41,6 +43,6 @@ def wrap_method( metadata = [client_info.to_grpc_metadata()] if client_info is not None else None - return general_helpers.wraps(func)( + return functools.wraps(func)( _GapicCallable(func, default_retry, default_timeout, metadata=metadata) ) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 3fb12a6f..a7bcb5a8 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,9 +20,7 @@ Generally, these headers are specified as gRPC metadata. """ -import sys - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" @@ -37,9 +35,6 @@ def to_routing_header(params): Returns: str: The routing header string. """ - if sys.version_info[0] < 3: - # Python 2 does not have the "safe" parameter for urlencode. - return urlencode(params).replace("%2F", "/") return urlencode( params, # Per Google API policy (go/api-url-encoding), / is not encoded. diff --git a/google/api_core/general_helpers.py b/google/api_core/general_helpers.py index d2d0c440..fba78026 100644 --- a/google/api_core/general_helpers.py +++ b/google/api_core/general_helpers.py @@ -12,22 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helpers for general Python functionality.""" - -import functools - -import six - - -# functools.partial objects lack several attributes present on real function -# objects. In Python 2 wraps fails on this so use a restricted set instead. -_PARTIAL_VALID_ASSIGNMENTS = ("__doc__",) - - -def wraps(wrapped): - """A functools.wraps helper that handles partial objects on Python 2.""" - # https://github.com/google/pytype/issues/322 - if isinstance(wrapped, functools.partial): # pytype: disable=wrong-arg-types - return six.wraps(wrapped, assigned=_PARTIAL_VALID_ASSIGNMENTS) - else: - return six.wraps(wrapped) +# This import for backward compatibiltiy only. +from functools import wraps # noqa: F401 pragma: NO COVER diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 62d9e533..594df987 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -15,13 +15,12 @@ """Helpers for :mod:`grpc`.""" import collections +import functools import grpc import pkg_resources -import six from google.api_core import exceptions -from google.api_core import general_helpers import google.auth import google.auth.credentials import google.auth.transport.grpc @@ -61,12 +60,12 @@ def _wrap_unary_errors(callable_): """Map errors for Unary-Unary and Stream-Unary gRPC callables.""" _patch_callable_name(callable_) - @six.wraps(callable_) + @functools.wraps(callable_) def error_remapped_callable(*args, **kwargs): try: return callable_(*args, **kwargs) except grpc.RpcError as exc: - six.raise_from(exceptions.from_grpc_error(exc), exc) + raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable @@ -80,7 +79,7 @@ def __init__(self, wrapped, prefetch_first_result=True): # to retrieve the first result, in order to fail, in order to trigger a retry. try: if prefetch_first_result: - self._stored_first_result = six.next(self._wrapped) + self._stored_first_result = next(self._wrapped) except TypeError: # It is possible the wrapped method isn't an iterable (a grpc.Call # for instance). If this happens don't store the first result. @@ -93,7 +92,7 @@ def __iter__(self): """This iterator is also an iterable that returns itself.""" return self - def next(self): + def __next__(self): """Get the next response from the stream. Returns: @@ -104,13 +103,10 @@ def next(self): result = self._stored_first_result del self._stored_first_result return result - return six.next(self._wrapped) + return next(self._wrapped) except grpc.RpcError as exc: # If the stream has already returned data, we cannot recover here. - six.raise_from(exceptions.from_grpc_error(exc), exc) - - # Alias needed for Python 2/3 support. - __next__ = next + raise exceptions.from_grpc_error(exc) from exc # grpc.Call & grpc.RpcContext interface @@ -148,7 +144,7 @@ def _wrap_stream_errors(callable_): """ _patch_callable_name(callable_) - @general_helpers.wraps(callable_) + @functools.wraps(callable_) def error_remapped_callable(*args, **kwargs): try: result = callable_(*args, **kwargs) @@ -161,7 +157,7 @@ def error_remapped_callable(*args, **kwargs): result, prefetch_first_result=prefetch_first ) except grpc.RpcError as exc: - six.raise_from(exceptions.from_grpc_error(exc), exc) + raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable diff --git a/google/api_core/iam.py b/google/api_core/iam.py index 59e53874..4437c701 100644 --- a/google/api_core/iam.py +++ b/google/api_core/iam.py @@ -52,14 +52,10 @@ """ import collections +import collections.abc import operator import warnings -try: - from collections import abc as collections_abc -except ImportError: # Python 2.7 - import collections as collections_abc - # Generic IAM roles OWNER_ROLE = "roles/owner" @@ -84,7 +80,7 @@ class InvalidOperationException(Exception): pass -class Policy(collections_abc.MutableMapping): +class Policy(collections.abc.MutableMapping): """IAM Policy Args: diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py index bc9befcb..d7f963e2 100644 --- a/google/api_core/operations_v1/__init__.py +++ b/google/api_core/operations_v1/__init__.py @@ -14,11 +14,7 @@ """Package for interacting with the google.longrunning.operations meta-API.""" -import sys - +from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient from google.api_core.operations_v1.operations_client import OperationsClient -__all__ = ["OperationsClient"] -if sys.version_info >= (3, 6, 0): - from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient # noqa: F401 - __all__.append("OperationsAsyncClient") +__all__ = ["OperationsAsyncClient", "OperationsClient"] diff --git a/google/api_core/page_iterator.py b/google/api_core/page_iterator.py index 49879bc9..7ddc5cbc 100644 --- a/google/api_core/page_iterator.py +++ b/google/api_core/page_iterator.py @@ -81,8 +81,6 @@ import abc -import six - class Page(object): """Single page of results in an iterator. @@ -127,18 +125,15 @@ def __iter__(self): """The :class:`Page` is an iterator of items.""" return self - def next(self): + def __next__(self): """Get the next value in the page.""" - item = six.next(self._item_iter) + item = next(self._item_iter) result = self._item_to_value(self._parent, item) # Since we've successfully got the next value from the # iterator, we update the number of remaining. self._remaining -= 1 return result - # Alias needed for Python 2/3 support. - __next__ = next - def _item_to_value_identity(iterator, item): """An item to value transformer that returns the item un-changed.""" @@ -147,8 +142,7 @@ def _item_to_value_identity(iterator, item): return item -@six.add_metaclass(abc.ABCMeta) -class Iterator(object): +class Iterator(object, metaclass=abc.ABCMeta): """A generic class for iterating through API list responses. Args: @@ -235,9 +229,6 @@ def __next__(self): self.__active_iterator = iter(self) return next(self.__active_iterator) - # Preserve Python 2 compatibility. - next = __next__ - def _page_iter(self, increment): """Generator of pages of API responses. @@ -484,7 +475,7 @@ def _next_page(self): there are no pages left. """ try: - items = six.next(self._gax_page_iter) + items = next(self._gax_page_iter) page = Page(self, items, self.item_to_value) self.next_page_token = self._gax_page_iter.page_token or None return page diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index f202d40f..c5969c14 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -28,8 +28,6 @@ import functools import re -import six - # Regular expression for extracting variable parts from a path template. # The variables can be expressed as: # @@ -83,7 +81,7 @@ def _expand_variable_match(positional_vars, named_vars, match): name = match.group("name") if name is not None: try: - return six.text_type(named_vars[name]) + return str(named_vars[name]) except KeyError: raise ValueError( "Named variable '{}' not specified and needed by template " @@ -91,7 +89,7 @@ def _expand_variable_match(positional_vars, named_vars, match): ) elif positional is not None: try: - return six.text_type(positional_vars.pop(0)) + return str(positional_vars.pop(0)) except IndexError: raise ValueError( "Positional variable not specified and needed by template " diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py index 8aff79aa..896e89c1 100644 --- a/google/api_core/protobuf_helpers.py +++ b/google/api_core/protobuf_helpers.py @@ -15,6 +15,7 @@ """Helpers for :mod:`protobuf`.""" import collections +import collections.abc import copy import inspect @@ -22,11 +23,6 @@ from google.protobuf import message from google.protobuf import wrappers_pb2 -try: - from collections import abc as collections_abc -except ImportError: # Python 2.7 - import collections as collections_abc - _SENTINEL = object() _WRAPPER_TYPES = ( @@ -179,7 +175,7 @@ def get(msg_or_dict, key, default=_SENTINEL): # If we get something else, complain. if isinstance(msg_or_dict, message.Message): answer = getattr(msg_or_dict, key, default) - elif isinstance(msg_or_dict, collections_abc.Mapping): + elif isinstance(msg_or_dict, collections.abc.Mapping): answer = msg_or_dict.get(key, default) else: raise TypeError( @@ -204,7 +200,7 @@ def _set_field_on_message(msg, key, value): """Set helper for protobuf Messages.""" # Attempt to set the value on the types of objects we know how to deal # with. - if isinstance(value, (collections_abc.MutableSequence, tuple)): + if isinstance(value, (collections.abc.MutableSequence, tuple)): # Clear the existing repeated protobuf message of any elements # currently inside it. while getattr(msg, key): @@ -212,13 +208,13 @@ def _set_field_on_message(msg, key, value): # Write our new elements to the repeated field. for item in value: - if isinstance(item, collections_abc.Mapping): + if isinstance(item, collections.abc.Mapping): getattr(msg, key).add(**item) else: # protobuf's RepeatedCompositeContainer doesn't support # append. getattr(msg, key).extend([item]) - elif isinstance(value, collections_abc.Mapping): + elif isinstance(value, collections.abc.Mapping): # Assign the dictionary values to the protobuf message. for item_key, item_value in value.items(): set(getattr(msg, key), item_key, item_value) @@ -241,7 +237,7 @@ def set(msg_or_dict, key, value): TypeError: If ``msg_or_dict`` is not a Message or dictionary. """ # Sanity check: Is our target object valid? - if not isinstance(msg_or_dict, (collections_abc.MutableMapping, message.Message)): + if not isinstance(msg_or_dict, (collections.abc.MutableMapping, message.Message)): raise TypeError( "set() expected a dict or protobuf message, got {!r}.".format( type(msg_or_dict) @@ -254,12 +250,12 @@ def set(msg_or_dict, key, value): # If a subkey exists, then get that object and call this method # recursively against it using the subkey. if subkey is not None: - if isinstance(msg_or_dict, collections_abc.MutableMapping): + if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict.setdefault(basekey, {}) set(get(msg_or_dict, basekey), subkey, value) return - if isinstance(msg_or_dict, collections_abc.MutableMapping): + if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict[key] = value else: _set_field_on_message(msg_or_dict, key, value) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 84967930..d39f97c1 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -63,11 +63,9 @@ def check_if_exists(): import time import requests.exceptions -import six from google.api_core import datetime_helpers from google.api_core import exceptions -from google.api_core import general_helpers from google.auth import exceptions as auth_exceptions _LOGGER = logging.getLogger(__name__) @@ -201,15 +199,12 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if deadline_datetime is not None: if deadline_datetime <= now: - six.raise_from( - exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target - ), - last_exc, + raise exceptions.RetryError( + "Deadline of {:.1f}s exceeded while calling {}".format( + deadline, target ), last_exc, - ) + ) from last_exc else: time_to_deadline = (deadline_datetime - now).total_seconds() sleep = min(time_to_deadline, sleep) @@ -222,7 +217,6 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): raise ValueError("Sleep generator stopped yielding sleep values.") -@six.python_2_unicode_compatible class Retry(object): """Exponential retry decorator. @@ -276,7 +270,7 @@ def __call__(self, func, on_error=None): if self._on_error is not None: on_error = self._on_error - @general_helpers.wraps(func) + @functools.wraps(func) def retry_wrapped_func(*args, **kwargs): """A wrapper that calls target function with retry.""" target = functools.partial(func, *args, **kwargs) diff --git a/google/api_core/timeout.py b/google/api_core/timeout.py index 17c1beab..73232180 100644 --- a/google/api_core/timeout.py +++ b/google/api_core/timeout.py @@ -54,11 +54,9 @@ def is_thing_ready(timeout=None): from __future__ import unicode_literals import datetime - -import six +import functools from google.api_core import datetime_helpers -from google.api_core import general_helpers _DEFAULT_INITIAL_TIMEOUT = 5.0 # seconds _DEFAULT_MAXIMUM_TIMEOUT = 30.0 # seconds @@ -68,7 +66,6 @@ def is_thing_ready(timeout=None): _DEFAULT_DEADLINE = None -@six.python_2_unicode_compatible class ConstantTimeout(object): """A decorator that adds a constant timeout argument. @@ -95,7 +92,7 @@ def __call__(self, func): Callable: The wrapped function. """ - @general_helpers.wraps(func) + @functools.wraps(func) def func_with_timeout(*args, **kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = self._timeout @@ -140,7 +137,6 @@ def _exponential_timeout_generator(initial, maximum, multiplier, deadline): timeout = timeout * multiplier -@six.python_2_unicode_compatible class ExponentialTimeout(object): """A decorator that adds an exponentially increasing timeout argument. @@ -207,7 +203,7 @@ def __call__(self, func): self._initial, self._maximum, self._multiplier, self._deadline ) - @general_helpers.wraps(func) + @functools.wraps(func) def func_with_timeout(*args, **kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = next(timeouts) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5fd0b133..c9cdad6f 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.31.1" +__version__ = "2.0.0-b1" diff --git a/noxfile.py b/noxfile.py index 2f11137d..a8f464e0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -111,13 +111,13 @@ def default(session): session.run(*pytest_args) -@nox.session(python=["2.7", "3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["2.7", "3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( @@ -137,7 +137,6 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -# No 2.7 due to https://github.com/google/importlab/issues/26. # No 3.7 because pytype supports up to 3.6 only. @nox.session(python="3.6") def pytype(session): diff --git a/owlbot.py b/owlbot.py index 5c590e4c..451f7c48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -22,8 +22,15 @@ # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- -templated_files = common.py_library(cov_level=100) -s.move(templated_files, excludes=["noxfile.py", ".flake8", ".coveragerc", "setup.cfg"]) +excludes = [ + "noxfile.py", # pytype + "setup.cfg", # pytype + ".flake8", # flake8-import-order, layout + ".coveragerc", # layout + "CONTRIBUTING.rst", # no systests +] +templated_files = common.py_library(microgenerator=True, cov_level=100) +s.move(templated_files, excludes=excludes) # Add pytype support s.replace( diff --git a/setup.py b/setup.py index 26fab7e6..d98c69c5 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,9 @@ dependencies = [ "googleapis-common-protos >= 1.6.0, < 2.0dev", "protobuf >= 3.12.0", - "google-auth >= 1.25.0, < 2.0dev", + "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", - "packaging >= 14.3", - "six >= 1.13.0", - "pytz", 'futures >= 3.2.0; python_version < "3.2"', ] extras = { @@ -86,10 +83,7 @@ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -102,7 +96,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*", + python_requires=">=3.6", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-2.7.txt b/testing/constraints-2.7.txt deleted file mode 100644 index 246c89d5..00000000 --- a/testing/constraints-2.7.txt +++ /dev/null @@ -1 +0,0 @@ -googleapis-common-protos >= 1.6.0, < 1.53dev diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index ff5b4a7f..2544e8e5 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,7 +11,6 @@ google-auth==1.25.0 requests==2.18.0 setuptools==40.3.0 packaging==14.3 -six==1.13.0 grpcio==1.29.0 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 602d640f..b876d9ad 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -14,12 +14,12 @@ import datetime import logging +import queue import threading import grpc import mock import pytest -from six.moves import queue from google.api_core import bidi from google.api_core import exceptions @@ -221,18 +221,12 @@ def cancel_side_effect(): class ClosedCall(object): - # NOTE: This is needed because defining `.next` on an **instance** - # rather than the **class** will not be iterable in Python 2. - # This is problematic since a `Mock` just sets members. - def __init__(self, exception): self.exception = exception def __next__(self): raise self.exception - next = __next__ # Python 2 - def is_active(self): return False @@ -354,8 +348,6 @@ def __next__(self): raise item return item - next = __next__ # Python 2 - def is_active(self): return self._is_active diff --git a/tests/unit/test_datetime_helpers.py b/tests/unit/test_datetime_helpers.py index 4ddcf361..5f5470a6 100644 --- a/tests/unit/test_datetime_helpers.py +++ b/tests/unit/test_datetime_helpers.py @@ -16,7 +16,6 @@ import datetime import pytest -import pytz from google.api_core import datetime_helpers from google.protobuf import timestamp_pb2 @@ -31,7 +30,7 @@ def test_utcnow(): def test_to_milliseconds(): - dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc) + dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=datetime.timezone.utc) assert datetime_helpers.to_milliseconds(dt) == 1000 @@ -42,7 +41,7 @@ def test_to_microseconds(): def test_to_microseconds_non_utc(): - zone = pytz.FixedOffset(-1) + zone = datetime.timezone(datetime.timedelta(minutes=-1)) dt = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=zone) assert datetime_helpers.to_microseconds(dt) == ONE_MINUTE_IN_MICROSECONDS @@ -56,7 +55,7 @@ def test_to_microseconds_naive(): def test_from_microseconds(): five_mins_from_epoch_in_microseconds = 5 * ONE_MINUTE_IN_MICROSECONDS five_mins_from_epoch_datetime = datetime.datetime( - 1970, 1, 1, 0, 5, 0, tzinfo=pytz.utc + 1970, 1, 1, 0, 5, 0, tzinfo=datetime.timezone.utc ) result = datetime_helpers.from_microseconds(five_mins_from_epoch_in_microseconds) @@ -78,28 +77,28 @@ def test_from_iso8601_time(): def test_from_rfc3339(): value = "2009-12-17T12:44:32.123456Z" assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + 2009, 12, 17, 12, 44, 32, 123456, datetime.timezone.utc ) def test_from_rfc3339_nanos(): value = "2009-12-17T12:44:32.123456Z" assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + 2009, 12, 17, 12, 44, 32, 123456, datetime.timezone.utc ) def test_from_rfc3339_without_nanos(): value = "2009-12-17T12:44:32Z" assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 0, pytz.utc + 2009, 12, 17, 12, 44, 32, 0, datetime.timezone.utc ) def test_from_rfc3339_nanos_without_nanos(): value = "2009-12-17T12:44:32Z" assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 0, pytz.utc + 2009, 12, 17, 12, 44, 32, 0, datetime.timezone.utc ) @@ -119,7 +118,7 @@ def test_from_rfc3339_nanos_without_nanos(): def test_from_rfc3339_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, micros, pytz.utc + 2009, 12, 17, 12, 44, 32, micros, datetime.timezone.utc ) @@ -148,7 +147,7 @@ def test_from_rfc3339_nanos_is_deprecated(): def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, micros, pytz.utc + 2009, 12, 17, 12, 44, 32, micros, datetime.timezone.utc ) @@ -171,20 +170,20 @@ def test_to_rfc3339(): def test_to_rfc3339_with_utc(): - value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=pytz.utc) + value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=datetime.timezone.utc) expected = "2016-04-05T13:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=False) == expected def test_to_rfc3339_with_non_utc(): - zone = pytz.FixedOffset(-60) + zone = datetime.timezone(datetime.timedelta(minutes=-60)) value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone) expected = "2016-04-05T14:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=False) == expected def test_to_rfc3339_with_non_utc_ignore_zone(): - zone = pytz.FixedOffset(-60) + zone = datetime.timezone(datetime.timedelta(minutes=-60)) value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone) expected = "2016-04-05T13:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=True) == expected @@ -283,7 +282,7 @@ def test_from_rfc3339_w_invalid(): def test_from_rfc3339_wo_fraction(): timestamp = "2016-12-20T21:13:47Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -292,7 +291,7 @@ def test_from_rfc3339_wo_fraction(): def test_from_rfc3339_w_partial_precision(): timestamp = "2016-12-20T21:13:47.1Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, microsecond=100000, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, microsecond=100000, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -301,7 +300,7 @@ def test_from_rfc3339_w_partial_precision(): def test_from_rfc3339_w_full_precision(): timestamp = "2016-12-20T21:13:47.123456789Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -332,7 +331,9 @@ def test_timestamp_pb_wo_nanos_naive(): stamp = datetime_helpers.DatetimeWithNanoseconds( 2016, 12, 20, 21, 13, 47, 123456 ) - delta = stamp.replace(tzinfo=pytz.UTC) - datetime_helpers._UTC_EPOCH + delta = ( + stamp.replace(tzinfo=datetime.timezone.utc) - datetime_helpers._UTC_EPOCH + ) seconds = int(delta.total_seconds()) nanos = 123456000 timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) @@ -341,7 +342,7 @@ def test_timestamp_pb_wo_nanos_naive(): @staticmethod def test_timestamp_pb_w_nanos(): stamp = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=datetime.timezone.utc ) delta = stamp - datetime_helpers._UTC_EPOCH timestamp = timestamp_pb2.Timestamp( @@ -351,7 +352,9 @@ def test_timestamp_pb_w_nanos(): @staticmethod def test_from_timestamp_pb_wo_nanos(): - when = datetime.datetime(2016, 12, 20, 21, 13, 47, 123456, tzinfo=pytz.UTC) + when = datetime.datetime( + 2016, 12, 20, 21, 13, 47, 123456, tzinfo=datetime.timezone.utc + ) delta = when - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds) @@ -361,11 +364,13 @@ def test_from_timestamp_pb_wo_nanos(): assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 0 assert stamp.nanosecond == 0 - assert stamp.tzinfo == pytz.UTC + assert stamp.tzinfo == datetime.timezone.utc @staticmethod def test_from_timestamp_pb_w_nanos(): - when = datetime.datetime(2016, 12, 20, 21, 13, 47, 123456, tzinfo=pytz.UTC) + when = datetime.datetime( + 2016, 12, 20, 21, 13, 47, 123456, tzinfo=datetime.timezone.utc + ) delta = when - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=123456789) @@ -375,7 +380,7 @@ def test_from_timestamp_pb_w_nanos(): assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 123456 assert stamp.nanosecond == 123456789 - assert stamp.tzinfo == pytz.UTC + assert stamp.tzinfo == datetime.timezone.utc def _to_seconds(value): @@ -387,5 +392,5 @@ def _to_seconds(value): Returns: int: Microseconds since the unix epoch. """ - assert value.tzinfo is pytz.UTC + assert value.tzinfo is datetime.timezone.utc return calendar.timegm(value.timetuple()) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index fb29015f..10599457 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import http.client import json import grpc import mock import requests -from six.moves import http_client from google.api_core import exceptions @@ -50,8 +50,8 @@ def test_create_google_cloud_error_with_args(): def test_from_http_status(): message = "message" - exception = exceptions.from_http_status(http_client.NOT_FOUND, message) - assert exception.code == http_client.NOT_FOUND + exception = exceptions.from_http_status(http.client.NOT_FOUND, message) + assert exception.code == http.client.NOT_FOUND assert exception.message == message assert exception.errors == [] @@ -61,11 +61,11 @@ def test_from_http_status_with_errors_and_response(): errors = ["1", "2"] response = mock.sentinel.response exception = exceptions.from_http_status( - http_client.NOT_FOUND, message, errors=errors, response=response + http.client.NOT_FOUND, message, errors=errors, response=response ) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == message assert exception.errors == errors assert exception.response == response @@ -82,7 +82,7 @@ def test_from_http_status_unknown_code(): def make_response(content): response = requests.Response() response._content = content - response.status_code = http_client.NOT_FOUND + response.status_code = http.client.NOT_FOUND response.request = requests.Request( method="POST", url="https://example.com" ).prepare() @@ -95,7 +95,7 @@ def test_from_http_response_no_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: unknown error" assert exception.response == response @@ -106,7 +106,7 @@ def test_from_http_response_text_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: message" @@ -120,7 +120,7 @@ def test_from_http_response_json_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: json message" assert exception.errors == ["1", "2"] @@ -131,22 +131,22 @@ def test_from_http_response_bad_json_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: unknown error" def test_from_http_response_json_unicode_content(): response = make_response( json.dumps( - {"error": {"message": u"\u2019 message", "errors": ["1", "2"]}} + {"error": {"message": "\u2019 message", "errors": ["1", "2"]}} ).encode("utf-8") ) exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND - assert exception.message == u"POST https://example.com/: \u2019 message" + assert exception.code == http.client.NOT_FOUND + assert exception.message == "POST https://example.com/: \u2019 message" assert exception.errors == ["1", "2"] @@ -155,7 +155,7 @@ def test_from_grpc_status(): exception = exceptions.from_grpc_status(grpc.StatusCode.OUT_OF_RANGE, message) assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.OutOfRange) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE assert exception.message == message assert exception.errors == [] @@ -166,7 +166,7 @@ def test_from_grpc_status_as_int(): exception = exceptions.from_grpc_status(11, message) assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.OutOfRange) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE assert exception.message == message assert exception.errors == [] @@ -203,7 +203,7 @@ def test_from_grpc_error(): assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.InvalidArgument) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.INVALID_ARGUMENT assert exception.message == message assert exception.errors == [error] diff --git a/tests/unit/test_general_helpers.py b/tests/unit/test_general_helpers.py deleted file mode 100644 index 027d4892..00000000 --- a/tests/unit/test_general_helpers.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017, Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools - -from google.api_core import general_helpers - - -def test_wraps_normal_func(): - def func(): - return 42 - - @general_helpers.wraps(func) - def replacement(): - return func() - - assert replacement() == 42 - - -def test_wraps_partial(): - def func(): - return 42 - - partial = functools.partial(func) - - @general_helpers.wraps(partial) - def replacement(): - return func() - - assert replacement() == 42 diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py index 668cf392..a44e998b 100644 --- a/tests/unit/test_page_iterator.py +++ b/tests/unit/test_page_iterator.py @@ -17,7 +17,6 @@ import mock import pytest -import six from google.api_core import page_iterator @@ -56,17 +55,17 @@ def test_iterator_calls_parent_item_to_value(self): assert item_to_value.call_count == 0 assert page.remaining == 100 - assert six.next(page) == 10 + assert next(page) == 10 assert item_to_value.call_count == 1 item_to_value.assert_called_with(parent, 10) assert page.remaining == 99 - assert six.next(page) == 11 + assert next(page) == 11 assert item_to_value.call_count == 2 item_to_value.assert_called_with(parent, 11) assert page.remaining == 98 - assert six.next(page) == 12 + assert next(page) == 12 assert item_to_value.call_count == 3 item_to_value.assert_called_with(parent, 12) assert page.remaining == 97 @@ -197,17 +196,17 @@ def test__items_iter(self): # Consume items and check the state of the iterator. assert iterator.num_results == 0 - assert six.next(items_iter) == item1 + assert next(items_iter) == item1 assert iterator.num_results == 1 - assert six.next(items_iter) == item2 + assert next(items_iter) == item2 assert iterator.num_results == 2 - assert six.next(items_iter) == item3 + assert next(items_iter) == item3 assert iterator.num_results == 3 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) def test___iter__(self): iterator = PageIteratorImpl(None, None) @@ -289,16 +288,16 @@ def test_iterate(self): items_iter = iter(iterator) - val1 = six.next(items_iter) + val1 = next(items_iter) assert val1 == item1 assert iterator.num_results == 1 - val2 = six.next(items_iter) + val2 = next(items_iter) assert val2 == item2 assert iterator.num_results == 2 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) api_request.assert_called_once_with(method="GET", path=path, query_params={}) @@ -503,7 +502,7 @@ def api_request(*args, **kw): items_iter = iter(iterator.pages) npages = int(math.ceil(float(n_results) / page_size)) for ipage in range(npages): - assert list(six.next(items_iter)) == [ + assert list(next(items_iter)) == [ dict(name=str(i)) for i in range( ipage * page_size, min((ipage + 1) * page_size, n_results), @@ -512,11 +511,11 @@ def api_request(*args, **kw): else: items_iter = iter(iterator) for i in range(n_results): - assert six.next(items_iter) == dict(name=str(i)) + assert next(items_iter) == dict(name=str(i)) assert iterator.num_results == i + 1 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) class TestGRPCIterator(object): @@ -621,7 +620,7 @@ def __init__(self, pages, page_token=None): self.page_token = page_token def next(self): - return six.next(self._pages) + return next(self._pages) __next__ = next From d566f2893eb708c109ebc3b4fa16f57aa4c65f26 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:02:27 -0400 Subject: [PATCH 003/126] chore: release 2.0.0b1 (#243) --- CHANGELOG.md | 11 +++++++++++ google/api_core/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e00e424c..e8e5e7a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.0.0b1](https://www.github.com/googleapis/python-api-core/compare/v1.31.1...v2.0.0b1) (2021-08-03) + + +### ⚠ BREAKING CHANGES + +* drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) + +### Bug Fixes + +* strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) + ### [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index c9cdad6f..5fa77444 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0-b1" +__version__ = "2.0.0b1" From 6789e675814983cbd413dc045c11a7c7a176042c Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 6 Aug 2021 10:52:12 -0600 Subject: [PATCH 004/126] chore: remove non-custom rules from sync-repo-settings (#241) See https://github.com/googleapis/repo-automation-bots/blob/63c858e539e1f4d9bb8ea66e12f9c0a0de5fef55/packages/sync-repo-settings/src/required-checks.json#L40-L50 for defaults. --- .github/sync-repo-settings.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 4a559eaf..e621885d 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,12 +1,3 @@ -rebaseMergeAllowed: true -squashMergeAllowed: true -mergeCommitAllowed: false -branchProtectionRules: -- pattern: master - isAdminEnforced: true - requiredApprovingReviewCount: 1 - requiresCodeOwnerReviews: true - requiresStrictStatusChecks: true permissionRules: - team: actools-python permission: admin From 258ba4a0119dbd1426afcd79d2f549b4c0fb6ec6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 15:55:59 +0000 Subject: [PATCH 005/126] chore(python): avoid `.nox` directories when building docs (#249) Source-Link: https://github.com/googleapis/synthtool/commit/7e1f6da50524b5d98eb67adbf6dd0805df54233d Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 9ee60f7e..b771c37c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:aea14a583128771ae8aefa364e1652f3c56070168ef31beb203534222d842b8b + digest: sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d diff --git a/docs/conf.py b/docs/conf.py index 93516048..aec958f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,6 +110,7 @@ # directories to ignore when looking for source files. exclude_patterns = [ "_build", + "**/.nox/**/*", "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", "samples/snippets/README.rst", From ffa528e088dc5e426ed5652be022c8fb43834e25 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 12 Aug 2021 17:42:46 -0400 Subject: [PATCH 006/126] chore: avoid duplicating pins of grpcio in noxfile (#246) Rely on the pins in 'setup.py' as the Source of Truth. See https://github.com/googleapis/python-api-core/pull/234#pullrequestreview-724669326 --- noxfile.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index a8f464e0..84470f5a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -80,8 +80,8 @@ def default(session): ) # Install all test dependencies, then install this package in-place. - session.install("mock", "pytest", "pytest-cov", "grpcio >= 1.0.2") - session.install("-e", ".", "-c", constraints_path) + session.install("mock", "pytest", "pytest-cov") + session.install("-e", ".[grpc]", "-c", constraints_path) pytest_args = [ "python", @@ -124,7 +124,7 @@ def unit_grpc_gcp(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) # Install grpcio-gcp - session.install("grpcio-gcp", "-c", constraints_path) + session.install("-e", ".[grpcgcp]", "-c", constraints_path) default(session) @@ -141,9 +141,7 @@ def lint_setup_py(session): @nox.session(python="3.6") def pytype(session): """Run type-checking.""" - session.install( - ".", "grpcio >= 1.8.2", "grpcio-gcp >= 0.2.2", "pytype >= 2019.3.21" - ) + session.install(".[grpc, grpcgcp]", "pytype >= 2019.3.21") session.run("pytype") @@ -163,8 +161,7 @@ def cover(session): def docs(session): """Build the docs for this library.""" - session.install(".", "grpcio >= 1.8.2", "grpcio-gcp >= 0.2.2") - session.install("-e", ".") + session.install("-e", ".[grpc, grpcgcp]") session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) From a54cc10bdcf8dd8df59d24c58a9b9053d456c846 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:08:59 -0400 Subject: [PATCH 007/126] chore: drop mention of Python 2.7 from templates (#252) Source-Link: https://github.com/googleapis/synthtool/commit/facee4cc1ea096cd8bcc008bb85929daa7c414c0 Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- scripts/readme-gen/templates/install_deps.tmpl.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b771c37c..a9fcd07c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d + digest: sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index a0406dba..275d6498 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash From 362ca6ce33075f7e5f5696b2116401162babf6da Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 13 Aug 2021 12:04:15 -0400 Subject: [PATCH 008/126] chore: update if_transient_error docs to match behaviour (#253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #185 🦕 --- google/api_core/retry.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index d39f97c1..44cd7469 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -113,8 +113,11 @@ def if_exception_type_predicate(exception): ``INTERNAL(13)`` and its subclasses. - :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429 - :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503 -- :class:`google.api_core.exceptions.ResourceExhausted` - gRPC - ``RESOURCE_EXHAUSTED(8)`` +- :class:`requests.exceptions.ConnectionError` +- :class:`requests.exceptions.ChunkedEncodingError` - The server declared + chunked encoding but sent an invalid chunk. +- :class:`google.auth.exceptions.TransportError` - Used to indicate an + error occurred during an HTTP request. """ # pylint: enable=invalid-name From bdbf889210b709d7c1945f2160bcba9161b4dd2e Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 13 Aug 2021 09:05:01 -0700 Subject: [PATCH 009/126] fix: bump grpcio version to use stable aio API (#234) --- google/api_core/grpc_helpers_async.py | 2 +- setup.py | 2 +- testing/constraints-3.6.txt | 2 +- tests/asyncio/gapic/test_method_async.py | 2 +- .../test_operations_async_client.py | 2 +- tests/asyncio/test_grpc_helpers_async.py | 28 +++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 92df645b..452e7871 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -22,7 +22,7 @@ import functools import grpc -from grpc.experimental import aio +from grpc import aio from google.api_core import exceptions, grpc_helpers diff --git a/setup.py b/setup.py index d98c69c5..cade3d82 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'futures >= 3.2.0; python_version < "3.2"', ] extras = { - "grpc": "grpcio >= 1.29.0, < 2.0dev", + "grpc": "grpcio >= 1.33.2, < 2.0dev", "grpcgcp": "grpcio-gcp >= 0.2.2", "grpcio-gcp": "grpcio-gcp >= 0.2.2", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 2544e8e5..744e991b 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,6 +11,6 @@ google-auth==1.25.0 requests==2.18.0 setuptools==40.3.0 packaging==14.3 -grpcio==1.29.0 +grpcio==1.33.2 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index 2c6bbab9..e24fe444 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -14,7 +14,7 @@ import datetime -from grpc.experimental import aio +from grpc import aio import mock import pytest diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py index 5473e8ae..3fb8427b 100644 --- a/tests/asyncio/operations_v1/test_operations_async_client.py +++ b/tests/asyncio/operations_v1/test_operations_async_client.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc.experimental import aio +from grpc import aio import mock import pytest diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index a511ed46..f0b6c5f1 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -13,7 +13,7 @@ # limitations under the License. import grpc -from grpc.experimental import aio +from grpc import aio import mock import pytest @@ -270,7 +270,7 @@ def test_wrap_errors_streaming(wrap_stream_errors): autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_call): target = "example.com:443" composite_creds = composite_creds_call.return_value @@ -295,7 +295,7 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_default_host( grpc_secure_channel, default, composite_creds_call, request, auth_metadata_plugin ): @@ -319,7 +319,7 @@ def test_create_channel_implicit_with_default_host( "google.auth.default", return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_ssl_creds( grpc_secure_channel, default, composite_creds_call ): @@ -341,7 +341,7 @@ def test_create_channel_implicit_with_ssl_creds( autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_scopes( grpc_secure_channel, default, composite_creds_call ): @@ -362,7 +362,7 @@ def test_create_channel_implicit_with_scopes( autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_default_scopes( grpc_secure_channel, default, composite_creds_call ): @@ -394,7 +394,7 @@ def test_create_channel_explicit_with_duplicate_credentials(): @mock.patch("grpc.composite_channel_credentials") @mock.patch("google.auth.credentials.with_scopes_if_required", autospec=True) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_creds_call): target = "example.com:443" composite_creds = composite_creds_call.return_value @@ -411,7 +411,7 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_call): target = "example.com:443" scopes = ["1", "2"] @@ -430,7 +430,7 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_default_scopes( grpc_secure_channel, composite_creds_call ): @@ -453,7 +453,7 @@ def test_create_channel_explicit_default_scopes( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_with_quota_project( grpc_secure_channel, composite_creds_call ): @@ -474,7 +474,7 @@ def test_create_channel_explicit_with_quota_project( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -500,7 +500,7 @@ def test_create_channnel_with_credentials_file( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -527,7 +527,7 @@ def test_create_channel_with_credentials_file_and_scopes( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -556,7 +556,7 @@ def test_create_channel_with_credentials_file_and_default_scopes( @pytest.mark.skipif( grpc_helpers_async.HAS_GRPC_GCP, reason="grpc_gcp module not available" ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_without_grpc_gcp(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From bb3b26d54dbe6acca1e105039b2fb405253980c0 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 18 Aug 2021 12:08:52 -0400 Subject: [PATCH 010/126] tests: silence a warning from 'charset_normalizer' (#247) --- google/api_core/retry.py | 4 ++-- tests/unit/test_exceptions.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 44cd7469..bd3a4a65 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -114,9 +114,9 @@ def if_exception_type_predicate(exception): - :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429 - :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503 - :class:`requests.exceptions.ConnectionError` -- :class:`requests.exceptions.ChunkedEncodingError` - The server declared +- :class:`requests.exceptions.ChunkedEncodingError` - The server declared chunked encoding but sent an invalid chunk. -- :class:`google.auth.exceptions.TransportError` - Used to indicate an +- :class:`google.auth.exceptions.TransportError` - Used to indicate an error occurred during an HTTP request. """ # pylint: enable=invalid-name diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 10599457..e9709f26 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -102,6 +102,7 @@ def test_from_http_response_no_content(): def test_from_http_response_text_content(): response = make_response(b"message") + response.encoding = "UTF8" # suppress charset_normalizer warning exception = exceptions.from_http_response(response) From 40f52bf1100cf56b7f9af267d210b8a72fc34f08 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:27:49 -0600 Subject: [PATCH 011/126] chore: release 2.0.0 (#254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :robot: I have created a release \*beep\* \*boop\* --- ## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) ### ⚠ BREAKING CHANGES * drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) ### Bug Fixes * bump grpcio version to use stable aio API ([#234](https://www.github.com/googleapis/python-api-core/issues/234)) ([bdbf889](https://www.github.com/googleapis/python-api-core/commit/bdbf889210b709d7c1945f2160bcba9161b4dd2e)) * strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 11 +++++++++++ google/api_core/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e5e7a5..3629c6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) + +### ⚠ BREAKING CHANGES + +* drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) + +### Bug Fixes + +* bump grpcio version to use stable aio API ([#234](https://www.github.com/googleapis/python-api-core/issues/234)) ([bdbf889](https://www.github.com/googleapis/python-api-core/commit/bdbf889210b709d7c1945f2160bcba9161b4dd2e)) +* strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) + ## [2.0.0b1](https://www.github.com/googleapis/python-api-core/compare/v1.31.1...v2.0.0b1) (2021-08-03) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5fa77444..a12de3d2 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0b1" +__version__ = "2.0.0" From e3d34d4cf7098f3d4317f28ee3a8f60a3f8c8558 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 15:26:19 +0000 Subject: [PATCH 012/126] chore(python): disable dependency dashboard (#266) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a9fcd07c..b75186cf 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 + digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 diff --git a/renovate.json b/renovate.json index c0489556..9fa8816f 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,8 @@ { "extends": [ - "config:base", ":preserveSemverRanges" + "config:base", + ":preserveSemverRanges", + ":disableDependencyDashboard" ], "ignorePaths": [".pre-commit-config.yaml"], "pip_requirements": { From 1db493cafff62e3a9f0b2d8ddf3071199db1af7e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 30 Aug 2021 13:45:35 -0400 Subject: [PATCH 013/126] chore: migrate default branch to main (#264) --- .kokoro/build.sh | 2 +- .kokoro/test-samples-impl.sh | 2 +- CONTRIBUTING.rst | 12 +++---- docs/conf.py | 10 +++--- google/api_core/exceptions.py | 3 +- google/api_core/future/async_future.py | 6 ++-- google/api_core/future/polling.py | 6 ++-- noxfile.py | 12 ++++++- owlbot.py | 45 ++++++++++++++++++++++++++ 9 files changed, 78 insertions(+), 20 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8b9fd8a1..0394c8aa 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -41,7 +41,7 @@ python3 -m pip install --upgrade --quiet nox python3 -m nox --version # If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. +# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then cleanup() { chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 311a8d54..8a324c9c 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -80,7 +80,7 @@ for file in samples/**/requirements.txt; do EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 358404f2..6db668ef 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -49,9 +49,9 @@ You'll have to create a development environment using a Git checkout: # Configure remotes such that you can pull changes from the googleapis/python-api-core # repository into your local repository. $ git remote add upstream git@github.com:googleapis/python-api-core.git - # fetch and merge changes from upstream into master + # fetch and merge changes from upstream into main $ git fetch upstream - $ git merge upstream/master + $ git merge upstream/main Now your local repo is set up such that you will push changes to your GitHub repo, from which you can submit a pull request. @@ -109,12 +109,12 @@ Coding Style variables:: export GOOGLE_CLOUD_TESTING_REMOTE="upstream" - export GOOGLE_CLOUD_TESTING_BRANCH="master" + export GOOGLE_CLOUD_TESTING_BRANCH="main" By doing this, you are specifying the location of the most up-to-date version of ``python-api-core``. The the suggested remote name ``upstream`` should point to the official ``googleapis`` checkout and the - the branch should be the main branch on that remote (``master``). + the branch should be the main branch on that remote (``main``). - This repository contains configuration for the `pre-commit `__ tool, which automates checking @@ -185,7 +185,7 @@ The `description on PyPI`_ for the project comes directly from the ``README``. Due to the reStructuredText (``rst``) parser used by PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst`` instead of -``https://github.com/googleapis/python-api-core/blob/master/CONTRIBUTING.rst``) +``https://github.com/googleapis/python-api-core/blob/main/CONTRIBUTING.rst``) may cause problems creating links or rendering the description. .. _description on PyPI: https://pypi.org/project/google-api-core @@ -210,7 +210,7 @@ We support: Supported versions can be found in our ``noxfile.py`` `config`_. -.. _config: https://github.com/googleapis/python-api-core/blob/master/noxfile.py +.. _config: https://github.com/googleapis/python-api-core/blob/main/noxfile.py We also explicitly decided to support Python 3 beginning with version 3.6. diff --git a/docs/conf.py b/docs/conf.py index aec958f3..09f0c2b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -76,8 +76,8 @@ # The encoding of source files. # source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = "index" +# The root toctree document. +root_doc = "index" # General information about the project. project = "google-api-core" @@ -280,7 +280,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ( - master_doc, + root_doc, "google-api-core.tex", "google-api-core Documentation", author, @@ -314,7 +314,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, "google-api-core", "google-api-core Documentation", [author], 1,) + (root_doc, "google-api-core", "google-api-core Documentation", [author], 1,) ] # If true, show URL addresses after external links. @@ -328,7 +328,7 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, + root_doc, "google-api-core", "google-api-core Documentation", author, diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 13be917e..b0909f1e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -293,8 +293,7 @@ class Cancelled(ClientError): """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error.""" # This maps to HTTP status code 499. See - # https://github.com/googleapis/googleapis/blob/master/google/rpc\ - # /code.proto + # https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto code = 499 grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None diff --git a/google/api_core/future/async_future.py b/google/api_core/future/async_future.py index 0343fbe2..88c183f9 100644 --- a/google/api_core/future/async_future.py +++ b/google/api_core/future/async_future.py @@ -43,8 +43,10 @@ class AsyncFuture(base.Future): The :meth:`done` method should be implemented by subclasses. The polling behavior will repeatedly call ``done`` until it returns True. - .. note: Privacy here is intended to prevent the final class from - overexposing, not to prevent subclasses from accessing methods. + .. note:: + + Privacy here is intended to prevent the final class from + overexposing, not to prevent subclasses from accessing methods. Args: retry (google.api_core.retry.Retry): The retry configuration used diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 2f80efb5..02e680f6 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -45,8 +45,10 @@ class PollingFuture(base.Future): The :meth:`done` method should be implemented by subclasses. The polling behavior will repeatedly call ``done`` until it returns True. - .. note: Privacy here is intended to prevent the final class from - overexposing, not to prevent subclasses from accessing methods. + .. note:: + + Privacy here is intended to prevent the final class from + overexposing, not to prevent subclasses from accessing methods. Args: retry (google.api_core.retry.Retry): The retry configuration used diff --git a/noxfile.py b/noxfile.py index 84470f5a..6478bfde 100644 --- a/noxfile.py +++ b/noxfile.py @@ -29,7 +29,17 @@ DEFAULT_PYTHON_VERSION = "3.7" CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() -_MINIMAL_ASYNCIO_SUPPORT_PYTHON_VERSION = [3, 6] +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "unit", + "unit_grpc_gcp", + "cover", + "pytype", + "lint", + "lint_setup_py", + "blacken", + "docs", +] def _greater_or_equal_than_36(version_string): diff --git a/owlbot.py b/owlbot.py index 451f7c48..a2f04592 100644 --- a/owlbot.py +++ b/owlbot.py @@ -44,4 +44,49 @@ """, ) +# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged + +# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files +s.replace( + ".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main" +) + +# Customize CONTRIBUTING.rst to replace master with main +s.replace( + "CONTRIBUTING.rst", + "fetch and merge changes from upstream into master", + "fetch and merge changes from upstream into main", +) + +s.replace( + "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main", +) + +s.replace( + "CONTRIBUTING.rst", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", +) + +s.replace( + "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)", +) + +s.replace( + "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst", +) + +s.replace( + "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py", +) + +s.replace( + "docs/conf.py", "master_doc", "root_doc", +) + +s.replace( + "docs/conf.py", "# The master toctree document.", "# The root toctree document.", +) + + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From 82ca2fd3792e20bea20ba96273fde4f2bb07b497 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 31 Aug 2021 13:16:20 -0400 Subject: [PATCH 014/126] tests: close coverage gap for race condition (#261) Closes #260 --- tests/unit/test_bidi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index b876d9ad..15994ee0 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -836,7 +836,7 @@ def test_consumer_unexpected_error(self, caplog): # Wait for the consumer's thread to exit. while consumer.is_active: - pass + pass # pragma: NO COVER (race condition) on_response.assert_not_called() bidi_rpc.recv.assert_called_once() From 618f19201af729205892fcecd9c8e315ba3174a3 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 31 Aug 2021 12:08:42 -0600 Subject: [PATCH 015/126] fix: do not error on LROs with no response or error (#258) Co-authored-by: Tres Seaver --- google/api_core/operation.py | 10 +++++----- google/api_core/operation_async.py | 10 +++++----- tests/asyncio/test_operation_async.py | 6 +++--- tests/unit/test_operation.py | 6 ++---- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/google/api_core/operation.py b/google/api_core/operation.py index b17f753b..a66e4a50 100644 --- a/google/api_core/operation.py +++ b/google/api_core/operation.py @@ -140,11 +140,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - exception = exceptions.GoogleAPICallError( - "Unexpected state: Long-running operation had neither " - "response nor error set." - ) - self.set_exception(exception) + # Some APIs set `done: true`, with an empty response. + # Set the result to an empty message of the expected + # result type. + # https://google.aip.dev/151 + self.set_result(self._result_type()) def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/google/api_core/operation_async.py b/google/api_core/operation_async.py index 6bae8654..17624d62 100644 --- a/google/api_core/operation_async.py +++ b/google/api_core/operation_async.py @@ -136,11 +136,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - exception = exceptions.GoogleAPICallError( - "Unexpected state: Long-running operation had neither " - "response nor error set." - ) - self.set_exception(exception) + # Some APIs set `done: true`, with an empty response. + # Set the result to an empty message of the expected + # result type. + # https://google.aip.dev/151 + self.set_result(self._result_type()) async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 907cda7c..342184fb 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -153,7 +153,7 @@ async def test_exception(): @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio -async def test_unexpected_result(unused_sleep): +async def test_done_with_no_error_or_response(unused_sleep): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -161,9 +161,9 @@ async def test_unexpected_result(unused_sleep): ] future, _, _ = make_operation_future(responses) - exception = await future.exception() + result = await future.result() - assert "Unexpected state" in "{!r}".format(exception) + assert isinstance(result, struct_pb2.Struct) def test_from_gapic(): diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index 28fbfe27..7a3e3c6c 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -163,7 +163,7 @@ def test_exception_with_error_code(): assert isinstance(exception, exceptions.NotFound) -def test_unexpected_result(): +def test_done_with_no_error_or_response(): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -171,9 +171,7 @@ def test_unexpected_result(): ] future, _, _ = make_operation_future(responses) - exception = future.exception() - - assert "Unexpected state" in "{!r}".format(exception) + assert isinstance(future.result(), struct_pb2.Struct) def test__refresh_http(): From dcb6ebd9994fddcb1729150df1675ebf8c503a73 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 15:47:14 -0400 Subject: [PATCH 016/126] chore: release 2.0.1 (#267) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3629c6e9..fdca59ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) + + +### Bug Fixes + +* do not error on LROs with no response or error ([#258](https://www.github.com/googleapis/python-api-core/issues/258)) ([618f192](https://www.github.com/googleapis/python-api-core/commit/618f19201af729205892fcecd9c8e315ba3174a3)) + ## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) ### ⚠ BREAKING CHANGES diff --git a/google/api_core/version.py b/google/api_core/version.py index a12de3d2..956a957b 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0" +__version__ = "2.0.1" From 01e31ca4ab7b3ca1f60112a02939d62cd27041c7 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 15:44:22 +0000 Subject: [PATCH 017/126] chore(python): group renovate prs (#270) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b75186cf..ef3cb34f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 + digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 diff --git a/renovate.json b/renovate.json index 9fa8816f..c21036d3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "extends": [ "config:base", + "group:all", ":preserveSemverRanges", ":disableDependencyDashboard" ], From 4679219fc1b95bed7f63f6da3b4c0095d359db05 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 1 Sep 2021 12:31:34 -0400 Subject: [PATCH 018/126] chore: remove obsolete dependency in setup.py (#269) Co-authored-by: Tres Seaver --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index cade3d82..48c8979f 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", - 'futures >= 3.2.0; python_version < "3.2"', ] extras = { "grpc": "grpcio >= 1.33.2, < 2.0dev", From 8b65c93aef908c99784200b73ad270a0591481a8 Mon Sep 17 00:00:00 2001 From: nicain Date: Thu, 2 Sep 2021 17:50:25 -0700 Subject: [PATCH 019/126] chore: removing owlbot directives for conversion to main (#272) chore: removing owlbot directives for conversion to main --- .github/.OwlBot.lock.yaml | 2 +- owlbot.py | 45 --------------------------------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ef3cb34f..c07f148f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 + digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d diff --git a/owlbot.py b/owlbot.py index a2f04592..451f7c48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -44,49 +44,4 @@ """, ) -# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged - -# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files -s.replace( - ".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main" -) - -# Customize CONTRIBUTING.rst to replace master with main -s.replace( - "CONTRIBUTING.rst", - "fetch and merge changes from upstream into master", - "fetch and merge changes from upstream into main", -) - -s.replace( - "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main", -) - -s.replace( - "CONTRIBUTING.rst", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", -) - -s.replace( - "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py", -) - -s.replace( - "docs/conf.py", "master_doc", "root_doc", -) - -s.replace( - "docs/conf.py", "# The master toctree document.", "# The root toctree document.", -) - - s.shell.run(["nox", "-s", "blacken"], hide_output=False) From afe0fa14c21289c8244606a9f81544cff8ac5f7c Mon Sep 17 00:00:00 2001 From: Yih-Jen Ku Date: Wed, 15 Sep 2021 16:12:18 -0400 Subject: [PATCH 020/126] feat: add grpc transcoding + tests (#259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add grpc transcoding + tests * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: tweak for clarity / idiomatic usage * chore: attempt to appease Sphinx * feat: add grpc transcoding + tests * Add functions to properly handle subfields * Add unit tests for get_field and delete_field. * Add function docstrings and incorporate correct native dict functions. * Add function docstrings and incorporate correct native dict functions. * Increase code coverage * Increase code coverage * Increase code coverage * Reformat files Co-authored-by: Yonatan Getahun Co-authored-by: Owl Bot Co-authored-by: Tres Seaver --- google/api_core/path_template.py | 107 +++++++++++- tests/unit/test_path_template.py | 274 +++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 1 deletion(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index c5969c14..41fbd4fe 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -25,6 +25,8 @@ from __future__ import unicode_literals +from collections import deque +import copy import functools import re @@ -64,7 +66,7 @@ def _expand_variable_match(positional_vars, named_vars, match): """Expand a matched variable with its value. Args: - positional_vars (list): A list of positonal variables. This list will + positional_vars (list): A list of positional variables. This list will be modified. named_vars (dict): A dictionary of named variables. match (re.Match): A regular expression match. @@ -170,6 +172,46 @@ def _generate_pattern_for_template(tmpl): return _VARIABLE_RE.sub(_replace_variable_with_pattern, tmpl) +def get_field(request, field): + """Get the value of a field from a given dictionary. + + Args: + request (dict): A dictionary object. + field (str): The key to the request in dot notation. + + Returns: + The value of the field. + """ + parts = field.split(".") + value = request + for part in parts: + if not isinstance(value, dict): + return + value = value.get(part) + if isinstance(value, dict): + return + return value + + +def delete_field(request, field): + """Delete the value of a field from a given dictionary. + + Args: + request (dict): A dictionary object. + field (str): The key to the request in dot notation. + """ + parts = deque(field.split(".")) + while len(parts) > 1: + if not isinstance(request, dict): + return + part = parts.popleft() + request = request.get(part) + part = parts.popleft() + if not isinstance(request, dict): + return + request.pop(part, None) + + def validate(tmpl, path): """Validate a path against the path template. @@ -193,3 +235,66 @@ def validate(tmpl, path): """ pattern = _generate_pattern_for_template(tmpl) + "$" return True if re.match(pattern, path) is not None else False + + +def transcode(http_options, **request_kwargs): + """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, + https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 + + Args: + http_options (list(dict)): A list of dicts which consist of these keys, + 'method' (str): The http method + 'uri' (str): The path template + 'body' (str): The body field name (optional) + (This is a simplified representation of the proto option `google.api.http`) + + request_kwargs (dict) : A dict representing the request object + + Returns: + dict: The transcoded request with these keys, + 'method' (str) : The http method + 'uri' (str) : The expanded uri + 'body' (dict) : A dict representing the body (optional) + 'query_params' (dict) : A dict mapping query parameter variables and values + + Raises: + ValueError: If the request does not match the given template. + """ + for http_option in http_options: + request = {} + + # Assign path + uri_template = http_option["uri"] + path_fields = [ + match.group("name") for match in _VARIABLE_RE.finditer(uri_template) + ] + path_args = {field: get_field(request_kwargs, field) for field in path_fields} + request["uri"] = expand(uri_template, **path_args) + + # Remove fields used in uri path from request + leftovers = copy.deepcopy(request_kwargs) + for path_field in path_fields: + delete_field(leftovers, path_field) + + if not validate(uri_template, request["uri"]) or not all(path_args.values()): + continue + + # Assign body and query params + body = http_option.get("body") + + if body: + if body == "*": + request["body"] = leftovers + request["query_params"] = {} + else: + try: + request["body"] = leftovers.pop(body) + except KeyError: + continue + request["query_params"] = leftovers + else: + request["query_params"] = leftovers + request["method"] = http_option["method"] + return request + + raise ValueError("Request obj does not match any template") diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index 4c8a7c5e..2c5216e0 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -84,6 +84,61 @@ def test_expanded_failure(tmpl, args, kwargs, exc_match): path_template.expand(tmpl, *args, **kwargs) +@pytest.mark.parametrize( + "request_obj, field, expected_result", + [ + [{"field": "stringValue"}, "field", "stringValue"], + [{"field": "stringValue"}, "nosuchfield", None], + [{"field": "stringValue"}, "field.subfield", None], + [{"field": {"subfield": "stringValue"}}, "field", None], + [{"field": {"subfield": "stringValue"}}, "field.subfield", "stringValue"], + [{"field": {"subfield": [1, 2, 3]}}, "field.subfield", [1, 2, 3]], + [{"field": {"subfield": "stringValue"}}, "field", None], + [{"field": {"subfield": "stringValue"}}, "field.nosuchfield", None], + [ + {"field": {"subfield": {"subsubfield": "stringValue"}}}, + "field.subfield.subsubfield", + "stringValue", + ], + ["string", "field", None], + ], +) +def test_get_field(request_obj, field, expected_result): + result = path_template.get_field(request_obj, field) + assert result == expected_result + + +@pytest.mark.parametrize( + "request_obj, field, expected_result", + [ + [{"field": "stringValue"}, "field", {}], + [{"field": "stringValue"}, "nosuchfield", {"field": "stringValue"}], + [{"field": "stringValue"}, "field.subfield", {"field": "stringValue"}], + [{"field": {"subfield": "stringValue"}}, "field.subfield", {"field": {}}], + [ + {"field": {"subfield": "stringValue", "q": "w"}, "e": "f"}, + "field.subfield", + {"field": {"q": "w"}, "e": "f"}, + ], + [ + {"field": {"subfield": "stringValue"}}, + "field.nosuchfield", + {"field": {"subfield": "stringValue"}}, + ], + [ + {"field": {"subfield": {"subsubfield": "stringValue", "q": "w"}}}, + "field.subfield.subsubfield", + {"field": {"subfield": {"q": "w"}}}, + ], + ["string", "field", "string"], + ["string", "field.subfield", "string"], + ], +) +def test_delete_field(request_obj, field, expected_result): + path_template.delete_field(request_obj, field) + assert request_obj == expected_result + + @pytest.mark.parametrize( "tmpl, path", [ @@ -113,3 +168,222 @@ def test__replace_variable_with_pattern(): match.group.return_value = None with pytest.raises(ValueError, match="Unknown"): path_template._replace_variable_with_pattern(match) + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + [ + [["get", "/v1/no/template", ""]], + {"foo": "bar"}, + ["get", "/v1/no/template", {}, {"foo": "bar"}], + ], + # Single templates + [ + [["get", "/v1/{field}", ""]], + {"field": "parent"}, + ["get", "/v1/parent", {}, {}], + ], + [ + [["get", "/v1/{field.sub}", ""]], + {"field": {"sub": "parent"}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], + ], + ], +) +def test_transcode_base_case(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + [ + [["get", "/v1/{field.subfield}", ""]], + {"field": {"subfield": "parent"}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], + ], + [ + [["get", "/v1/{field.subfield.subsubfield}", ""]], + {"field": {"subfield": {"subsubfield": "parent"}}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {"subfield": {}}, "foo": "bar"}], + ], + [ + [["get", "/v1/{field.subfield1}/{field.subfield2}", ""]], + {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, + ["get", "/v1/parent/child", {}, {"field": {}, "foo": "bar"}], + ], + ], +) +def test_transcode_subfields(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Single segment wildcard + [ + [["get", "/v1/{field=*}", ""]], + {"field": "parent"}, + ["get", "/v1/parent", {}, {}], + ], + [ + [["get", "/v1/{field=a/*/b/*}", ""]], + {"field": "a/parent/b/child", "foo": "bar"}, + ["get", "/v1/a/parent/b/child", {}, {"foo": "bar"}], + ], + # Double segment wildcard + [ + [["get", "/v1/{field=**}", ""]], + {"field": "parent/p1"}, + ["get", "/v1/parent/p1", {}, {}], + ], + [ + [["get", "/v1/{field=a/**/b/**}", ""]], + {"field": "a/parent/p1/b/child/c1", "foo": "bar"}, + ["get", "/v1/a/parent/p1/b/child/c1", {}, {"foo": "bar"}], + ], + # Combined single and double segment wildcard + [ + [["get", "/v1/{field=a/*/b/**}", ""]], + {"field": "a/parent/b/child/c1"}, + ["get", "/v1/a/parent/b/child/c1", {}, {}], + ], + [ + [["get", "/v1/{field=a/**/b/*}/v2/{name}", ""]], + {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, + ["get", "/v1/a/parent/p1/b/child/v2/first", {}, {"foo": "bar"}], + ], + ], +) +def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Single field body + [ + [["post", "/v1/no/template", "data"]], + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + ["post", "/v1/no/template", {"id": 1, "info": "some info"}, {"foo": "bar"}], + ], + [ + [["post", "/v1/{field=a/*}/b/{name=**}", "data"]], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"id": 1, "info": "some info"}, + {"foo": "bar"}, + ], + ], + # Wildcard body + [ + [["post", "/v1/{field=a/*}/b/{name=**}", "*"]], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + {}, + ], + ], + ], +) +def test_transcode_with_body(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Additional bindings + [ + [ + ["post", "/v1/{field=a/*}/b/{name=**}", "extra_data"], + ["post", "/v1/{field=a/*}/b/{name=**}", "*"], + ], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + {}, + ], + ], + [ + [ + ["get", "/v1/{field=a/*}/b/{name=**}", ""], + ["get", "/v1/{field=a/*}/b/first/last", ""], + ], + {"field": "a/parent", "foo": "bar"}, + ["get", "/v1/a/parent/b/first/last", {}, {"foo": "bar"}], + ], + ], +) +def test_transcode_with_additional_bindings( + http_options, request_kwargs, expected_result +): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs", + [ + [[["get", "/v1/{name}", ""]], {"foo": "bar"}], + [[["get", "/v1/{name}", ""]], {"name": "first/last"}], + [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], + [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], + ], +) +def test_transcode_fails(http_options, request_kwargs): + http_options, _ = helper_test_transcode(http_options, range(4)) + with pytest.raises(ValueError): + path_template.transcode(http_options, **request_kwargs) + + +def helper_test_transcode(http_options_list, expected_result_list): + http_options = [] + for opt_list in http_options_list: + http_option = {"method": opt_list[0], "uri": opt_list[1]} + if opt_list[2]: + http_option["body"] = opt_list[2] + http_options.append(http_option) + + expected_result = { + "method": expected_result_list[0], + "uri": expected_result_list[1], + "query_params": expected_result_list[3], + } + if expected_result_list[2]: + expected_result["body"] = expected_result_list[2] + + return (http_options, expected_result) From 1c5eb4df93d78e791082d9282330ebf0faacd222 Mon Sep 17 00:00:00 2001 From: Ken Bandes Date: Mon, 20 Sep 2021 14:40:51 -0400 Subject: [PATCH 021/126] feat: Add helper function to format query_params for rest transport. (#275) Co-authored-by: Kenneth Bandes --- google/api_core/rest_helpers.py | 94 +++++++++++++++++++++++++++++++++ tests/unit/test_rest_helpers.py | 77 +++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 google/api_core/rest_helpers.py create mode 100644 tests/unit/test_rest_helpers.py diff --git a/google/api_core/rest_helpers.py b/google/api_core/rest_helpers.py new file mode 100644 index 00000000..23fb614f --- /dev/null +++ b/google/api_core/rest_helpers.py @@ -0,0 +1,94 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for rest transports.""" + +import functools +import operator + + +def flatten_query_params(obj): + """Flatten a nested dict into a list of (name,value) tuples. + + The result is suitable for setting query params on an http request. + + .. code-block:: python + + >>> obj = {'a': + ... {'b': + ... {'c': ['x', 'y', 'z']} }, + ... 'd': 'uvw', } + >>> flatten_query_params(obj) + [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw')] + + Note that, as described in + https://github.com/googleapis/googleapis/blob/48d9fb8c8e287c472af500221c6450ecd45d7d39/google/api/http.proto#L117, + repeated fields (i.e. list-valued fields) may only contain primitive types (not lists or dicts). + This is enforced in this function. + + Args: + obj: a nested dictionary (from json), or None + + Returns: a list of tuples, with each tuple having a (possibly) multi-part name + and a scalar value. + + Raises: + TypeError if obj is not a dict or None + ValueError if obj contains a list of non-primitive values. + """ + + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + + return _flatten(obj, key_path=[]) + + +def _flatten(obj, key_path): + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path) + return _flatten_value(obj, key_path=key_path) + + +def _is_primitive_value(obj): + if obj is None: + return False + + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + + return True + + +def _flatten_value(obj, key_path): + return [(".".join(key_path), obj)] + + +def _flatten_dict(obj, key_path): + items = (_flatten(value, key_path=key_path + [key]) for key, value in obj.items()) + return functools.reduce(operator.concat, items, []) + + +def _flatten_list(elems, key_path): + # Only lists of scalar values are supported. + # The name (key_path) is repeated for each value. + items = ( + _flatten_value(elem, key_path=key_path) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) diff --git a/tests/unit/test_rest_helpers.py b/tests/unit/test_rest_helpers.py new file mode 100644 index 00000000..5932fa55 --- /dev/null +++ b/tests/unit/test_rest_helpers.py @@ -0,0 +1,77 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from google.api_core import rest_helpers + + +def test_flatten_simple_value(): + with pytest.raises(TypeError): + rest_helpers.flatten_query_params("abc") + + +def test_flatten_list(): + with pytest.raises(TypeError): + rest_helpers.flatten_query_params(["abc", "def"]) + + +def test_flatten_none(): + assert rest_helpers.flatten_query_params(None) == [] + + +def test_flatten_empty_dict(): + assert rest_helpers.flatten_query_params({}) == [] + + +def test_flatten_simple_dict(): + assert rest_helpers.flatten_query_params({"a": "abc", "b": "def"}) == [ + ("a", "abc"), + ("b", "def"), + ] + + +def test_flatten_repeated_field(): + assert rest_helpers.flatten_query_params({"a": ["x", "y", "z", None]}) == [ + ("a", "x"), + ("a", "y"), + ("a", "z"), + ] + + +def test_flatten_nested_dict(): + obj = {"a": {"b": {"c": ["x", "y", "z"]}}, "d": {"e": "uvw"}} + expected_result = [("a.b.c", "x"), ("a.b.c", "y"), ("a.b.c", "z"), ("d.e", "uvw")] + + assert rest_helpers.flatten_query_params(obj) == expected_result + + +def test_flatten_repeated_dict(): + obj = { + "a": {"b": {"c": [{"v": 1}, {"v": 2}]}}, + "d": "uvw", + } + + with pytest.raises(ValueError): + rest_helpers.flatten_query_params(obj) + + +def test_flatten_repeated_list(): + obj = { + "a": {"b": {"c": [["e", "f"], ["g", "h"]]}}, + "d": "uvw", + } + + with pytest.raises(ValueError): + rest_helpers.flatten_query_params(obj) From 083e6e92ac870f2ab0bfcbae83fd0f30fe013eb7 Mon Sep 17 00:00:00 2001 From: Jeffrey Rennie Date: Tue, 21 Sep 2021 12:36:22 -0700 Subject: [PATCH 022/126] chore: relocate owl bot post processor (#280) chore: relocate owl bot post processor --- .github/.OwlBot.lock.yaml | 4 ++-- .github/.OwlBot.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index c07f148f..2567653c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:87eee22d276554e4e52863ec9b1cb6a7245815dfae20439712bf644348215a5a diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 3caf68d4..c8b40cc7 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -13,7 +13,7 @@ # limitations under the License. docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest begin-after-commit-hash: 7af2cb8b2b725641ac0d07e2f256d453682802e6 From 3b0912a08f115f352bac65167912400e55ef857e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 27 Sep 2021 14:45:43 -0400 Subject: [PATCH 023/126] tests: add explicit unit tests for '_StreamedResponseIterator' class (#281) --- tests/unit/test_grpc_helpers.py | 122 ++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 12bf1849..f5849290 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -66,6 +66,128 @@ def test_wrap_unary_errors(): assert exc_info.value.response == grpc_error +class Test_StreamingResponseIterator: + @staticmethod + def _make_wrapped(*items): + return iter(items) + + @staticmethod + def _make_one(wrapped, **kw): + return grpc_helpers._StreamingResponseIterator(wrapped, **kw) + + def test_ctor_defaults(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert iterator._stored_first_result == "a" + assert list(wrapped) == ["b", "c"] + + def test_ctor_explicit(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert getattr(iterator, "_stored_first_result", self) is self + assert list(wrapped) == ["a", "b", "c"] + + def test_ctor_w_rpc_error_on_prefetch(self): + wrapped = mock.MagicMock() + wrapped.__next__.side_effect = grpc.RpcError() + + with pytest.raises(grpc.RpcError): + self._make_one(wrapped) + + def test___iter__(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert iter(iterator) is iterator + + def test___next___w_cached_first_result(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert next(iterator) == "a" + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert next(iterator) == "b" + assert next(iterator) == "c" + + def test___next___wo_cached_first_result(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert next(iterator) == "a" + assert next(iterator) == "b" + assert next(iterator) == "c" + + def test___next___w_rpc_error(self): + wrapped = mock.MagicMock() + wrapped.__next__.side_effect = grpc.RpcError() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + with pytest.raises(exceptions.GoogleAPICallError): + next(iterator) + + def test_add_callback(self): + wrapped = mock.MagicMock() + callback = mock.Mock(spec={}) + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.add_callback(callback) is wrapped.add_callback.return_value + + wrapped.add_callback.assert_called_once_with(callback) + + def test_cancel(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.cancel() is wrapped.cancel.return_value + + wrapped.cancel.assert_called_once_with() + + def test_code(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.code() is wrapped.code.return_value + + wrapped.code.assert_called_once_with() + + def test_details(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.details() is wrapped.details.return_value + + wrapped.details.assert_called_once_with() + + def test_initial_metadata(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.initial_metadata() is wrapped.initial_metadata.return_value + + wrapped.initial_metadata.assert_called_once_with() + + def test_is_active(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.is_active() is wrapped.is_active.return_value + + wrapped.is_active.assert_called_once_with() + + def test_time_remaining(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.time_remaining() is wrapped.time_remaining.return_value + + wrapped.time_remaining.assert_called_once_with() + + def test_trailing_metadata(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.trailing_metadata() is wrapped.trailing_metadata.return_value + + wrapped.trailing_metadata.assert_called_once_with() + + def test_wrap_stream_okay(): expected_responses = [1, 2, 3] callable_ = mock.Mock(spec=["__call__"], return_value=iter(expected_responses)) From f3fd05901da03f000b2f3287e7710612653cd6c0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 4 Oct 2021 11:02:06 -0400 Subject: [PATCH 024/126] chore: add default_version and codeowner_team to .repo-metadata.json (#282) * chore: add default_version and codeowner_team to .repo-metadata.json * Assign @googleapis/actools-python as codeowner --- .repo-metadata.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index 59aa936d..e16c9d27 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -1,10 +1,12 @@ { - "name": "google-api-core", - "name_pretty": "Google API client core library", - "client_documentation": "https://googleapis.dev/python/google-api-core/latest", - "release_level": "ga", - "language": "python", - "library_type": "CORE", - "repo": "googleapis/python-api-core", - "distribution_name": "google-api-core" -} \ No newline at end of file + "name": "google-api-core", + "name_pretty": "Google API client core library", + "client_documentation": "https://googleapis.dev/python/google-api-core/latest", + "release_level": "ga", + "language": "python", + "library_type": "CORE", + "repo": "googleapis/python-api-core", + "distribution_name": "google-api-core", + "default_version": "", + "codeowner_team": "@googleapis/actools-python" +} From 577da9d35335e4ec3ccb720add22c197444211d9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 11:07:45 -0600 Subject: [PATCH 025/126] build: use trampoline_v2 for python samples and allow custom dockerfile (#283) Source-Link: https://github.com/googleapis/synthtool/commit/a7ed11ec0863c422ba2e73aafa75eab22c32b33d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/samples/lint/common.cfg | 2 +- .kokoro/samples/python3.6/common.cfg | 2 +- .kokoro/samples/python3.6/periodic.cfg | 2 +- .kokoro/samples/python3.7/common.cfg | 2 +- .kokoro/samples/python3.7/periodic.cfg | 2 +- .kokoro/samples/python3.8/common.cfg | 2 +- .kokoro/samples/python3.8/periodic.cfg | 2 +- .kokoro/samples/python3.9/common.cfg | 2 +- .kokoro/samples/python3.9/periodic.cfg | 2 +- .kokoro/test-samples-against-head.sh | 2 -- .kokoro/test-samples.sh | 2 -- .trampolinerc | 17 ++++++++++++++--- 13 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2567653c..ee94722a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:87eee22d276554e4e52863ec9b1cb6a7245815dfae20439712bf644348215a5a + digest: sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc diff --git a/.kokoro/samples/lint/common.cfg b/.kokoro/samples/lint/common.cfg index f5dddb4b..1a2b87b2 100644 --- a/.kokoro/samples/lint/common.cfg +++ b/.kokoro/samples/lint/common.cfg @@ -31,4 +31,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg index 7b4f5cd0..3bb6b3a6 100644 --- a/.kokoro/samples/python3.6/common.cfg +++ b/.kokoro/samples/python3.6/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.6/periodic.cfg +++ b/.kokoro/samples/python3.6/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.7/common.cfg b/.kokoro/samples/python3.7/common.cfg index 1198d7ba..a3aa10b5 100644 --- a/.kokoro/samples/python3.7/common.cfg +++ b/.kokoro/samples/python3.7/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.7/periodic.cfg b/.kokoro/samples/python3.7/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.7/periodic.cfg +++ b/.kokoro/samples/python3.7/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.8/common.cfg b/.kokoro/samples/python3.8/common.cfg index b7ec7f5e..20c941aa 100644 --- a/.kokoro/samples/python3.8/common.cfg +++ b/.kokoro/samples/python3.8/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.8/periodic.cfg b/.kokoro/samples/python3.8/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.8/periodic.cfg +++ b/.kokoro/samples/python3.8/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.9/common.cfg b/.kokoro/samples/python3.9/common.cfg index cf034ec1..234887c6 100644 --- a/.kokoro/samples/python3.9/common.cfg +++ b/.kokoro/samples/python3.9/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.9/periodic.cfg b/.kokoro/samples/python3.9/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.9/periodic.cfg +++ b/.kokoro/samples/python3.9/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh index a7858e4c..ba3a707b 100755 --- a/.kokoro/test-samples-against-head.sh +++ b/.kokoro/test-samples-against-head.sh @@ -23,6 +23,4 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-api-core - exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index ee3146bd..11c042d3 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -24,8 +24,6 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-api-core - # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then # preserving the test runner implementation. diff --git a/.trampolinerc b/.trampolinerc index 383b6ec8..0eee72ab 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -16,15 +16,26 @@ # Add required env vars here. required_envvars+=( - "STAGING_BUCKET" - "V2_STAGING_BUCKET" ) # Add env vars which are passed down into the container here. pass_down_envvars+=( + "NOX_SESSION" + ############### + # Docs builds + ############### "STAGING_BUCKET" "V2_STAGING_BUCKET" - "NOX_SESSION" + ################## + # Samples builds + ################## + "INSTALL_LIBRARY_FROM_SOURCE" + "RUN_TESTS_SESSION" + "BUILD_SPECIFIC_GCLOUD_PROJECT" + # Target directories. + "RUN_TESTS_DIRS" + # The nox session to run. + "RUN_TESTS_SESSION" ) # Prevent unintentional override on the default image. From a422a5d72cb6f363d57e7a4effe421ba8e049cde Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 5 Oct 2021 16:54:45 -0400 Subject: [PATCH 026/126] feat: add support for Python 3.10 (#284) Fix new deprecation warning for 'threading.Condition.notifyAll'. --- CONTRIBUTING.rst | 6 ++++-- google/api_core/bidi.py | 2 +- noxfile.py | 2 +- setup.py | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6db668ef..6b375f03 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.6, 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -71,7 +71,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.9 -- -k + $ nox -s unit-3.10 -- -k .. note:: @@ -201,11 +201,13 @@ We support: - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ +- `Python 3.10`_ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ +.. _Python 3.10: https://docs.python.org/3.10/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 56a021a9..4b4963f7 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -727,7 +727,7 @@ def resume(self): """Resumes the response stream.""" with self._wake: self._paused = False - self._wake.notifyAll() + self._wake.notify_all() @property def is_paused(self): diff --git a/noxfile.py b/noxfile.py index 6478bfde..617dc580 100644 --- a/noxfile.py +++ b/noxfile.py @@ -121,7 +121,7 @@ def default(session): session.run(*pytest_args) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def unit(session): """Run the unit test suite.""" default(session) diff --git a/setup.py b/setup.py index 48c8979f..d150bc01 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", ], From 121debdfa3461e0060d3f386b81ca6fa34920234 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 17:08:24 -0400 Subject: [PATCH 027/126] chore: release 2.1.0 (#274) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ google/api_core/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdca59ba..00910e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.1.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.1...v2.1.0) (2021-10-05) + + +### Features + +* add grpc transcoding + tests ([#259](https://www.github.com/googleapis/python-api-core/issues/259)) ([afe0fa1](https://www.github.com/googleapis/python-api-core/commit/afe0fa14c21289c8244606a9f81544cff8ac5f7c)) +* Add helper function to format query_params for rest transport. ([#275](https://www.github.com/googleapis/python-api-core/issues/275)) ([1c5eb4d](https://www.github.com/googleapis/python-api-core/commit/1c5eb4df93d78e791082d9282330ebf0faacd222)) +* add support for Python 3.10 ([#284](https://www.github.com/googleapis/python-api-core/issues/284)) ([a422a5d](https://www.github.com/googleapis/python-api-core/commit/a422a5d72cb6f363d57e7a4effe421ba8e049cde)) + ### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) diff --git a/google/api_core/version.py b/google/api_core/version.py index 956a957b..8b5d3328 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.1" +__version__ = "2.1.0" From e6a3489a79da00a5de919a4a41782cc3b1d7f583 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 8 Oct 2021 10:26:00 -0400 Subject: [PATCH 028/126] chore(python): Add kokoro configs for python 3.10 samples testing (#285) Source-Link: https://github.com/googleapis/synthtool/commit/c6e69c4726a233ad8d496961ec265d29e54010b7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/samples/python3.10/common.cfg | 40 ++++++++++++++++++++ .kokoro/samples/python3.10/continuous.cfg | 6 +++ .kokoro/samples/python3.10/periodic-head.cfg | 11 ++++++ .kokoro/samples/python3.10/periodic.cfg | 6 +++ .kokoro/samples/python3.10/presubmit.cfg | 6 +++ 6 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .kokoro/samples/python3.10/common.cfg create mode 100644 .kokoro/samples/python3.10/continuous.cfg create mode 100644 .kokoro/samples/python3.10/periodic-head.cfg create mode 100644 .kokoro/samples/python3.10/periodic.cfg create mode 100644 .kokoro/samples/python3.10/presubmit.cfg diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ee94722a..7d98291c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc + digest: sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b diff --git a/.kokoro/samples/python3.10/common.cfg b/.kokoro/samples/python3.10/common.cfg new file mode 100644 index 00000000..40fb8d81 --- /dev/null +++ b/.kokoro/samples/python3.10/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.10" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-310" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-api-core/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.10/continuous.cfg b/.kokoro/samples/python3.10/continuous.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.10/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.10/periodic-head.cfg b/.kokoro/samples/python3.10/periodic-head.cfg new file mode 100644 index 00000000..a18c0cfc --- /dev/null +++ b/.kokoro/samples/python3.10/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-api-core/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.10/periodic.cfg b/.kokoro/samples/python3.10/periodic.cfg new file mode 100644 index 00000000..71cd1e59 --- /dev/null +++ b/.kokoro/samples/python3.10/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.10/presubmit.cfg b/.kokoro/samples/python3.10/presubmit.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.10/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file From 0023ee1fe0e8b80c7a9e8987e0f322a829e5d613 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 13 Oct 2021 17:54:33 -0400 Subject: [PATCH 029/126] fix: add mypy checking + 'py.typed' file (#290) --- google/__init__.py | 2 +- google/api_core/client_info.py | 3 +++ google/api_core/exceptions.py | 9 +++++---- google/api_core/py.typed | 2 ++ mypy.ini | 7 +++++++ noxfile.py | 9 +++++++++ 6 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 google/api_core/py.typed create mode 100644 mypy.ini diff --git a/google/__init__.py b/google/__init__.py index 0d0a4c3a..70a7bd99 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -21,4 +21,4 @@ except ImportError: import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index d7f4367a..e093ffda 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -19,6 +19,7 @@ """ import platform +from typing import Union import pkg_resources @@ -27,6 +28,8 @@ _PY_VERSION = platform.python_version() _API_CORE_VERSION = api_core_version.__version__ +_GRPC_VERSION: Union[str, None] + try: _GRPC_VERSION = pkg_resources.get_distribution("grpcio").version except pkg_resources.DistributionNotFound: # pragma: NO COVER diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index b0909f1e..2cfc2e4a 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -22,17 +22,18 @@ from __future__ import unicode_literals import http.client +from typing import Dict +from typing import Union try: import grpc - except ImportError: # pragma: NO COVER grpc = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta -_HTTP_CODE_TO_EXCEPTION = {} -_GRPC_CODE_TO_EXCEPTION = {} +_HTTP_CODE_TO_EXCEPTION: Dict[int, Exception] = {} +_GRPC_CODE_TO_EXCEPTION: Dict[int, Exception] = {} # Additional lookup table to map integer status codes to grpc status code # grpc does not currently support initializing enums from ints @@ -100,7 +101,7 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): gRPC call metadata. """ - code = None + code: Union[int, None] = None """Optional[int]: The HTTP status code associated with this error. This may be ``None`` if the exception does not have a direct mapping diff --git a/google/api_core/py.typed b/google/api_core/py.typed new file mode 100644 index 00000000..1d5517b1 --- /dev/null +++ b/google/api_core/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-api-core package uses inline types. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..5663b40d --- /dev/null +++ b/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +python_version = 3.6 +namespace_packages = True +ignore_missing_imports = True + +[mypy-google.protobuf] +ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index 617dc580..926f9f5d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,6 +35,7 @@ "unit_grpc_gcp", "cover", "pytype", + "mypy", "lint", "lint_setup_py", "blacken", @@ -155,6 +156,14 @@ def pytype(session): session.run("pytype") +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Run type-checking.""" + session.install(".[grpc, grpcgcp]", "mypy") + session.install("types-setuptools", "types-requests", "types-mock") + session.run("mypy", "google", "tests") + + @nox.session(python="3.6") def cover(session): """Run the final coverage report. From f4e776e8889382c28943946e33e29411b02f7436 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 13 Oct 2021 18:09:59 -0400 Subject: [PATCH 030/126] chore: release 2.1.1 (#291) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00910e1f..e8e4e94a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) + + +### Bug Fixes + +* add mypy checking + 'py.typed' file ([#290](https://www.github.com/googleapis/python-api-core/issues/290)) ([0023ee1](https://www.github.com/googleapis/python-api-core/commit/0023ee1fe0e8b80c7a9e8987e0f322a829e5d613)) + ## [2.1.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.1...v2.1.0) (2021-10-05) diff --git a/google/api_core/version.py b/google/api_core/version.py index 8b5d3328..7945f6f4 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.0" +__version__ = "2.1.1" From 09cf285536fee519106db335670e1560b0846bcf Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 19 Oct 2021 12:58:59 -0400 Subject: [PATCH 031/126] tests: add testing w/o 'grpc' installed (#289) Closes #288. --- noxfile.py | 14 ++++++++++++-- tests/asyncio/gapic/test_config_async.py | 8 ++++++++ tests/asyncio/gapic/test_method_async.py | 6 +++++- .../operations_v1/test_operations_async_client.py | 10 ++++++++-- tests/asyncio/test_grpc_helpers_async.py | 15 ++++++++++++--- tests/asyncio/test_operation_async.py | 5 +++++ tests/unit/gapic/test_client_info.py | 7 +++++++ tests/unit/gapic/test_config.py | 7 +++++++ tests/unit/gapic/test_method.py | 7 +++++++ tests/unit/gapic/test_routing_header.py | 7 +++++++ .../unit/operations_v1/test_operations_client.py | 7 +++++++ tests/unit/test_bidi.py | 6 +++++- tests/unit/test_client_info.py | 12 +++++++++++- tests/unit/test_exceptions.py | 13 ++++++++++++- tests/unit/test_grpc_helpers.py | 6 +++++- tests/unit/test_operation.py | 6 ++++++ 16 files changed, 124 insertions(+), 12 deletions(-) diff --git a/noxfile.py b/noxfile.py index 926f9f5d..ac1bdd14 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,6 +33,7 @@ nox.options.sessions = [ "unit", "unit_grpc_gcp", + "unit_wo_grpc", "cover", "pytype", "mypy", @@ -78,7 +79,7 @@ def blacken(session): session.run("black", *BLACK_EXCLUDES, *BLACK_PATHS) -def default(session): +def default(session, install_grpc=True): """Default unit test session. This is intended to be run **without** an interpreter set, so @@ -92,7 +93,10 @@ def default(session): # Install all test dependencies, then install this package in-place. session.install("mock", "pytest", "pytest-cov") - session.install("-e", ".[grpc]", "-c", constraints_path) + if install_grpc: + session.install("-e", ".[grpc]", "-c", constraints_path) + else: + session.install("-e", ".", "-c", constraints_path) pytest_args = [ "python", @@ -140,6 +144,12 @@ def unit_grpc_gcp(session): default(session) +@nox.session(python=["3.6", "3.10"]) +def unit_wo_grpc(session): + """Run the unit test suite w/o grpcio installed""" + default(session, install_grpc=False) + + @nox.session(python="3.6") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" diff --git a/tests/asyncio/gapic/test_config_async.py b/tests/asyncio/gapic/test_config_async.py index 1f6ea9e2..dbb05d5e 100644 --- a/tests/asyncio/gapic/test_config_async.py +++ b/tests/asyncio/gapic/test_config_async.py @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. + +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core.gapic_v1 import config_async diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index e24fe444..1410747d 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -14,10 +14,14 @@ import datetime -from grpc import aio import mock import pytest +try: + from grpc import aio +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import gapic_v1 from google.api_core import grpc_helpers_async diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py index 3fb8427b..47c3b4b4 100644 --- a/tests/asyncio/operations_v1/test_operations_async_client.py +++ b/tests/asyncio/operations_v1/test_operations_async_client.py @@ -12,11 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc import aio import mock import pytest -from google.api_core import grpc_helpers_async, operations_v1, page_iterator_async +try: + from grpc import aio +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + +from google.api_core import grpc_helpers_async +from google.api_core import operations_v1 +from google.api_core import page_iterator_async from google.longrunning import operations_pb2 from google.protobuf import empty_pb2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index f0b6c5f1..0413abf3 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -12,10 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc -from grpc import aio import mock -import pytest +import pytest # noqa: I202 + +try: + import grpc + from grpc import aio +except ImportError: + grpc = aio = None + + +if grpc is None: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import grpc_helpers_async diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 342184fb..886b1c8d 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -16,6 +16,11 @@ import mock import pytest +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import operation_async from google.api_core import operations_v1 diff --git a/tests/unit/gapic/test_client_info.py b/tests/unit/gapic/test_client_info.py index 64080ffd..2ca5c404 100644 --- a/tests/unit/gapic/test_client_info.py +++ b/tests/unit/gapic/test_client_info.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core.gapic_v1 import client_info diff --git a/tests/unit/gapic/test_config.py b/tests/unit/gapic/test_config.py index 1c15261d..5e42fde8 100644 --- a/tests/unit/gapic/test_config.py +++ b/tests/unit/gapic/test_config.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core.gapic_v1 import config diff --git a/tests/unit/gapic/test_method.py b/tests/unit/gapic/test_method.py index e0ea57ac..9778d23a 100644 --- a/tests/unit/gapic/test_method.py +++ b/tests/unit/gapic/test_method.py @@ -15,6 +15,13 @@ import datetime import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import retry diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 77300e87..30378676 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core.gapic_v1 import routing_header diff --git a/tests/unit/operations_v1/test_operations_client.py b/tests/unit/operations_v1/test_operations_client.py index 001b8fea..187f0be3 100644 --- a/tests/unit/operations_v1/test_operations_client.py +++ b/tests/unit/operations_v1/test_operations_client.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import grpc_helpers from google.api_core import operations_v1 from google.api_core import page_iterator diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 15994ee0..7fb16209 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -17,10 +17,14 @@ import queue import threading -import grpc import mock import pytest +try: + import grpc +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import bidi from google.api_core import exceptions diff --git a/tests/unit/test_client_info.py b/tests/unit/test_client_info.py index f2274ec2..f5eebfbe 100644 --- a/tests/unit/test_client_info.py +++ b/tests/unit/test_client_info.py @@ -13,6 +13,11 @@ # limitations under the License. +try: + import grpc +except ImportError: + grpc = None + from google.api_core import client_info @@ -20,7 +25,12 @@ def test_constructor_defaults(): info = client_info.ClientInfo() assert info.python_version is not None - assert info.grpc_version is not None + + if grpc is not None: + assert info.grpc_version is not None + else: + assert info.grpc_version is None + assert info.api_core_version is not None assert info.gapic_version is None assert info.client_library_version is None diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index e9709f26..95317dbb 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -15,10 +15,15 @@ import http.client import json -import grpc import mock +import pytest import requests +try: + import grpc +except ImportError: + grpc = None + from google.api_core import exceptions @@ -151,6 +156,7 @@ def test_from_http_response_json_unicode_content(): assert exception.errors == ["1", "2"] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status(): message = "message" exception = exceptions.from_grpc_status(grpc.StatusCode.OUT_OF_RANGE, message) @@ -162,6 +168,7 @@ def test_from_grpc_status(): assert exception.errors == [] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_as_int(): message = "message" exception = exceptions.from_grpc_status(11, message) @@ -173,6 +180,7 @@ def test_from_grpc_status_as_int(): assert exception.errors == [] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_with_errors_and_response(): message = "message" response = mock.sentinel.response @@ -187,6 +195,7 @@ def test_from_grpc_status_with_errors_and_response(): assert exception.response == response +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_unknown_code(): message = "message" exception = exceptions.from_grpc_status(grpc.StatusCode.OK, message) @@ -194,6 +203,7 @@ def test_from_grpc_status_unknown_code(): assert exception.message == message +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_error(): message = "message" error = mock.create_autospec(grpc.Call, instance=True) @@ -211,6 +221,7 @@ def test_from_grpc_error(): assert exception.response == error +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_error_non_call(): message = "message" error = mock.create_autospec(grpc.RpcError, instance=True) diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index f5849290..4c613a9b 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc import mock import pytest +try: + import grpc +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import grpc_helpers import google.auth.credentials diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index 7a3e3c6c..bafff8b0 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -14,6 +14,12 @@ import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) from google.api_core import exceptions from google.api_core import operation From ef6f0fcfdfe771172056e35e3c990998b3b00416 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Tue, 19 Oct 2021 11:17:49 -0700 Subject: [PATCH 032/126] feat: add 'GoogleAPICallError.error_details' property (#286) Based on 'google.rpc.status.details'. --- google/api_core/exceptions.py | 67 +++++++++++++++- setup.py | 4 +- testing/constraints-3.6.txt | 3 +- tests/asyncio/test_grpc_helpers_async.py | 3 + tests/unit/test_exceptions.py | 98 ++++++++++++++++++++++-- tests/unit/test_grpc_helpers.py | 3 + 6 files changed, 166 insertions(+), 12 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 2cfc2e4a..fdb21090 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -25,10 +25,14 @@ from typing import Dict from typing import Union +from google.rpc import error_details_pb2 + try: import grpc + from grpc_status import rpc_status except ImportError: # pragma: NO COVER grpc = None + rpc_status = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta @@ -97,6 +101,7 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): Args: message (str): The exception message. errors (Sequence[Any]): An optional list of error details. + details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. """ @@ -117,15 +122,19 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): This may be ``None`` if the exception does not match up to a gRPC error. """ - def __init__(self, message, errors=(), response=None): + def __init__(self, message, errors=(), details=(), response=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors + self._details = details self._response = response def __str__(self): - return "{} {}".format(self.code, self.message) + if self.details: + return "{} {} {}".format(self.code, self.message, self.details) + else: + return "{} {}".format(self.code, self.message) @property def errors(self): @@ -136,6 +145,19 @@ def errors(self): """ return list(self._errors) + @property + def details(self): + """Information contained in google.rpc.status.details. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto + + Returns: + Sequence[Any]: A list of structured objects from error_details.proto + """ + return list(self._details) + @property def response(self): """Optional[Union[requests.Request, grpc.Call]]: The response or @@ -409,13 +431,15 @@ def from_http_response(response): error_message = payload.get("error", {}).get("message", "unknown error") errors = payload.get("error", {}).get("errors", ()) + # In JSON, details are already formatted in developer-friendly way. + details = payload.get("error", {}).get("details", ()) message = "{method} {url}: {error}".format( method=response.request.method, url=response.request.url, error=error_message ) exception = from_http_status( - response.status_code, message, errors=errors, response=response + response.status_code, message, errors=errors, details=details, response=response ) return exception @@ -462,6 +486,37 @@ def _is_informative_grpc_error(rpc_exc): return hasattr(rpc_exc, "code") and hasattr(rpc_exc, "details") +def _parse_grpc_error_details(rpc_exc): + status = rpc_status.from_call(rpc_exc) + if not status: + return [] + possible_errors = [ + error_details_pb2.BadRequest, + error_details_pb2.PreconditionFailure, + error_details_pb2.QuotaFailure, + error_details_pb2.ErrorInfo, + error_details_pb2.RetryInfo, + error_details_pb2.ResourceInfo, + error_details_pb2.RequestInfo, + error_details_pb2.DebugInfo, + error_details_pb2.Help, + error_details_pb2.LocalizedMessage, + ] + error_details = [] + for detail in status.details: + matched_detail_cls = list( + filter(lambda x: detail.Is(x.DESCRIPTOR), possible_errors) + ) + # If nothing matched, use detail directly. + if len(matched_detail_cls) == 0: + info = detail + else: + info = matched_detail_cls[0]() + detail.Unpack(info) + error_details.append(info) + return error_details + + def from_grpc_error(rpc_exc): """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. @@ -476,7 +531,11 @@ def from_grpc_error(rpc_exc): # However, check for grpc.RpcError breaks backward compatibility. if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): return from_grpc_status( - rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), response=rpc_exc + rpc_exc.code(), + rpc_exc.details(), + errors=(rpc_exc,), + details=_parse_grpc_error_details(rpc_exc), + response=rpc_exc, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/setup.py b/setup.py index d150bc01..ddc56004 100644 --- a/setup.py +++ b/setup.py @@ -29,14 +29,14 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "googleapis-common-protos >= 1.6.0, < 2.0dev", + "googleapis-common-protos >= 1.52.0, < 2.0dev", "protobuf >= 3.12.0", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", ] extras = { - "grpc": "grpcio >= 1.33.2, < 2.0dev", + "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], "grpcgcp": "grpcio-gcp >= 0.2.2", "grpcio-gcp": "grpcio-gcp >= 0.2.2", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 744e991b..0c2a07b6 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,7 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -googleapis-common-protos==1.6.0 +googleapis-common-protos==1.52.0 protobuf==3.12.0 google-auth==1.25.0 requests==2.18.0 @@ -14,3 +14,4 @@ packaging==14.3 grpcio==1.33.2 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 +grpcio-status==1.33.2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index 0413abf3..3681a40d 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -42,6 +42,9 @@ def code(self): def details(self): return None + def trailing_metadata(self): + return None + @pytest.mark.asyncio async def test_wrap_unary_errors(): diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 95317dbb..f6345fe1 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -21,10 +21,13 @@ try: import grpc + from grpc_status import rpc_status except ImportError: - grpc = None + grpc = rpc_status = None from google.api_core import exceptions +from google.protobuf import any_pb2, json_format +from google.rpc import error_details_pb2, status_pb2 def test_create_google_cloud_error(): @@ -38,11 +41,8 @@ def test_create_google_cloud_error(): def test_create_google_cloud_error_with_args(): error = { - "domain": "global", - "location": "test", - "locationType": "testing", + "code": 600, "message": "Testing", - "reason": "test", } response = mock.sentinel.response exception = exceptions.GoogleAPICallError("Testing", [error], response=response) @@ -235,3 +235,91 @@ def test_from_grpc_error_non_call(): assert exception.message == message assert exception.errors == [error] assert exception.response == error + + +def create_bad_request_details(): + bad_request_details = error_details_pb2.BadRequest() + field_violation = bad_request_details.field_violations.add() + field_violation.field = "document.content" + field_violation.description = "Must have some text content to annotate." + status_detail = any_pb2.Any() + status_detail.Pack(bad_request_details) + return status_detail + + +def test_error_details_from_rest_response(): + bad_request_detail = create_bad_request_details() + status = status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status.details.append(bad_request_detail) + + # See JSON schema in https://cloud.google.com/apis/design/errors#http_mapping + http_response = make_response( + json.dumps({"error": json.loads(json_format.MessageToJson(status))}).encode( + "utf-8" + ) + ) + exception = exceptions.from_http_response(http_response) + want_error_details = [json.loads(json_format.MessageToJson(bad_request_detail))] + assert want_error_details == exception.details + # 404 POST comes from make_response. + assert str(exception) == ( + "404 POST https://example.com/: 3 INVALID_ARGUMENT:" + " One of content, or gcs_content_uri must be set." + " [{'@type': 'type.googleapis.com/google.rpc.BadRequest'," + " 'fieldViolations': [{'field': 'document.content'," + " 'description': 'Must have some text content to annotate.'}]}]" + ) + + +def test_error_details_from_v1_rest_response(): + response = make_response( + json.dumps( + {"error": {"message": "\u2019 message", "errors": ["1", "2"]}} + ).encode("utf-8") + ) + exception = exceptions.from_http_response(response) + assert exception.details == [] + + +@pytest.mark.skipif(grpc is None, reason="gRPC not importable") +def test_error_details_from_grpc_response(): + status = rpc_status.status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status_detail = create_bad_request_details() + status.details.append(status_detail) + + # Actualy error doesn't matter as long as its grpc.Call, + # because from_call is mocked. + error = mock.create_autospec(grpc.Call, instance=True) + with mock.patch("grpc_status.rpc_status.from_call") as m: + m.return_value = status + exception = exceptions.from_grpc_error(error) + + bad_request_detail = error_details_pb2.BadRequest() + status_detail.Unpack(bad_request_detail) + assert exception.details == [bad_request_detail] + + +@pytest.mark.skipif(grpc is None, reason="gRPC not importable") +def test_error_details_from_grpc_response_unknown_error(): + status_detail = any_pb2.Any() + + status = rpc_status.status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status.details.append(status_detail) + + error = mock.create_autospec(grpc.Call, instance=True) + with mock.patch("grpc_status.rpc_status.from_call") as m: + m.return_value = status + exception = exceptions.from_grpc_error(error) + assert exception.details == [status_detail] diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 4c613a9b..ca969e48 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -56,6 +56,9 @@ def code(self): def details(self): return None + def trailing_metadata(self): + return None + def test_wrap_unary_errors(): grpc_error = RpcErrorImpl(grpc.StatusCode.INVALID_ARGUMENT) From 5f4fb50d2d23b937be81be4f64b54005c5638eb6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 17:22:41 -0400 Subject: [PATCH 033/126] chore(python): push cloud library docs to staging bucket for Cloud RAD (#295) Source-Link: https://github.com/googleapis/synthtool/commit/7fd61f8efae782a7cfcecc599faf52f9737fe584 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/docs/common.cfg | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7d98291c..108063d4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b + digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 4847856f..48e89855 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -30,7 +30,9 @@ env_vars: { env_vars: { key: "V2_STAGING_BUCKET" - value: "docs-staging-v2" + # Push non-cloud library docs to `docs-staging-v2-staging` instead of the + # Cloud RAD bucket `docs-staging-v2` + value: "docs-staging-v2-staging" } # It will upload the docker image after successful builds. From 240edb6a822482865a10ff0bcd5439ccf38d73a3 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:09:39 -0600 Subject: [PATCH 034/126] chore: release 2.2.0 (#293) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e4e94a..f4ea830c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.2.0](https://www.github.com/googleapis/python-api-core/compare/v2.1.1...v2.2.0) (2021-10-25) + + +### Features + +* add 'GoogleAPICallError.error_details' property ([#286](https://www.github.com/googleapis/python-api-core/issues/286)) ([ef6f0fc](https://www.github.com/googleapis/python-api-core/commit/ef6f0fcfdfe771172056e35e3c990998b3b00416)) + ### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 7945f6f4..bd0f8e5c 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.1" +__version__ = "2.2.0" From 9e6091ee59a30e72a6278b369f6a08e7aef32f22 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:40:20 -0600 Subject: [PATCH 035/126] fix: revert "fix: do not error on LROs with no response or error" (#294) Reverts googleapis/python-api-core#258 From @truchle > A while ago you helped me submit this pull request for the Python LRO client library. This was about making the LRO library not throw an error when receiving an empty LRO response. It turns out that the Python LRO library has always been behaving correctly by accepting empty LRO responses. It would just throw an error if the response field is NULL (see screenshot). > > Since then I've consulted other LRO client library owners (@vam-google ) and it turns out that the fault was in the CCAI Insights server code. CCAI Insights was returning NULL responses instead of empty responses (more context here b/202432905). Since then the issue has been fixed in our server and has rolled out to production, so reverting the Python LRO false fix wouldn't break our code sample. --- google/api_core/operation.py | 10 +++++----- google/api_core/operation_async.py | 10 +++++----- tests/asyncio/test_operation_async.py | 6 +++--- tests/unit/test_operation.py | 6 ++++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/google/api_core/operation.py b/google/api_core/operation.py index a66e4a50..b17f753b 100644 --- a/google/api_core/operation.py +++ b/google/api_core/operation.py @@ -140,11 +140,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - # Some APIs set `done: true`, with an empty response. - # Set the result to an empty message of the expected - # result type. - # https://google.aip.dev/151 - self.set_result(self._result_type()) + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/google/api_core/operation_async.py b/google/api_core/operation_async.py index 17624d62..6bae8654 100644 --- a/google/api_core/operation_async.py +++ b/google/api_core/operation_async.py @@ -136,11 +136,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - # Some APIs set `done: true`, with an empty response. - # Set the result to an empty message of the expected - # result type. - # https://google.aip.dev/151 - self.set_result(self._result_type()) + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 886b1c8d..26ad7cef 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -158,7 +158,7 @@ async def test_exception(): @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio -async def test_done_with_no_error_or_response(unused_sleep): +async def test_unexpected_result(unused_sleep): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -166,9 +166,9 @@ async def test_done_with_no_error_or_response(unused_sleep): ] future, _, _ = make_operation_future(responses) - result = await future.result() + exception = await future.exception() - assert isinstance(result, struct_pb2.Struct) + assert "Unexpected state" in "{!r}".format(exception) def test_from_gapic(): diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index bafff8b0..22e23bc3 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -169,7 +169,7 @@ def test_exception_with_error_code(): assert isinstance(exception, exceptions.NotFound) -def test_done_with_no_error_or_response(): +def test_unexpected_result(): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -177,7 +177,9 @@ def test_done_with_no_error_or_response(): ] future, _, _ = make_operation_future(responses) - assert isinstance(future.result(), struct_pb2.Struct) + exception = future.exception() + + assert "Unexpected state" in "{!r}".format(exception) def test__refresh_http(): From d2a729e0c42039bfbcd4b59c3d97d8759a9f0d57 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 28 Oct 2021 13:28:08 -0400 Subject: [PATCH 036/126] chore: release 2.2.1 (#296) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ea830c..8941ad76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) + + +### Bug Fixes + +* revert "fix: do not error on LROs with no response or error" ([#294](https://www.github.com/googleapis/python-api-core/issues/294)) ([9e6091e](https://www.github.com/googleapis/python-api-core/commit/9e6091ee59a30e72a6278b369f6a08e7aef32f22)) + ## [2.2.0](https://www.github.com/googleapis/python-api-core/compare/v2.1.1...v2.2.0) (2021-10-25) diff --git a/google/api_core/version.py b/google/api_core/version.py index bd0f8e5c..b1be7d8c 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.0" +__version__ = "2.2.1" From ffc51f03c7ce5d9f009ba859b8df385d52925578 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 2 Nov 2021 18:12:55 -0400 Subject: [PATCH 037/126] fix: make 'gapic_v1.method.DEFAULT' a typed object (#292) FBO checkers which need to verify default values for 'retry' / 'timeout' --- google/api_core/gapic_v1/method.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py index 79722c07..73c8d4bc 100644 --- a/google/api_core/gapic_v1/method.py +++ b/google/api_core/gapic_v1/method.py @@ -18,6 +18,7 @@ pagination, and long-running operations to gRPC methods. """ +import enum import functools from google.api_core import grpc_helpers @@ -25,7 +26,18 @@ from google.api_core.gapic_v1 import client_info USE_DEFAULT_METADATA = object() -DEFAULT = object() + + +class _MethodDefault(enum.Enum): + # Uses enum so that pytype/mypy knows that this is the only possible value. + # https://stackoverflow.com/a/60605919/101923 + # + # Literal[_DEFAULT_VALUE] is an alternative, but only added in Python 3.8. + # https://docs.python.org/3/library/typing.html#typing.Literal + _DEFAULT_VALUE = object() + + +DEFAULT = _MethodDefault._DEFAULT_VALUE """Sentinel value indicating that a retry or timeout argument was unspecified, so the default should be used.""" From 214d88b54142628f7ecf5d49d3515f353db8aaf5 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 2 Nov 2021 22:16:11 +0000 Subject: [PATCH 038/126] chore: release 2.2.2 (#302) :robot: I have created a release \*beep\* \*boop\* --- ### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) ### Bug Fixes * make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8941ad76..70ce6afc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) + + +### Bug Fixes + +* make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) + ### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index b1be7d8c..bcc59c44 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.1" +__version__ = "2.2.2" From 060b339e3af296dd1772bfc1b4a0d2b4264cae1f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 3 Nov 2021 14:18:12 -0400 Subject: [PATCH 039/126] fix: handle bare 'grpc.Call' in 'from_grpc_error' (#298) * fix: handle bare 'grpc.Call' in 'from_grpc_error' Fixes: #297. * tests: add assertion for 'exception.details' --- google/api_core/exceptions.py | 7 ++++++- tests/unit/test_exceptions.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index fdb21090..6b1b6f7e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -487,9 +487,14 @@ def _is_informative_grpc_error(rpc_exc): def _parse_grpc_error_details(rpc_exc): - status = rpc_status.from_call(rpc_exc) + try: + status = rpc_status.from_call(rpc_exc) + except NotImplementedError: # workaround + return [] + if not status: return [] + possible_errors = [ error_details_pb2.BadRequest, error_details_pb2.PreconditionFailure, diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index f6345fe1..622f58ab 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -237,6 +237,34 @@ def test_from_grpc_error_non_call(): assert exception.response == error +@pytest.mark.skipif(grpc is None, reason="No grpc") +def test_from_grpc_error_bare_call(): + message = "Testing" + + class TestingError(grpc.Call, grpc.RpcError): + def __init__(self, exception): + self.exception = exception + + def code(self): + return self.exception.grpc_status_code + + def details(self): + return message + + nested_message = "message" + error = TestingError(exceptions.GoogleAPICallError(nested_message)) + + exception = exceptions.from_grpc_error(error) + + assert isinstance(exception, exceptions.GoogleAPICallError) + assert exception.code is None + assert exception.grpc_status_code is None + assert exception.message == message + assert exception.errors == [error] + assert exception.response == error + assert exception.details == [] + + def create_bad_request_details(): bad_request_details = error_details_pb2.BadRequest() field_violation = bad_request_details.field_violations.add() From 4b407899cdcaaada0e1b9e12fbf8213690402ce5 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:12:09 -0500 Subject: [PATCH 040/126] chore(python): add .github/CODEOWNERS as a templated file (#308) Source-Link: https://github.com/googleapis/synthtool/commit/c5026b3217973a8db55db8ee85feee0e9a65e295 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .github/CODEOWNERS | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 108063d4..7519fa3a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 + digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 557e39e1..ee818917 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,9 +3,10 @@ # # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. -# The @googleapis/yoshi-python is the default owner for changes in this repo -* @googleapis/yoshi-python @googleapis/actools-python +# @googleapis/yoshi-python @googleapis/actools-python are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/actools-python -# The python-samples-reviewers team is the default owner for samples changes -/samples/ @googleapis/python-samples-owners +# @googleapis/python-samples-owners @googleapis/actools-python are the default owners for samples changes +/samples/ @googleapis/python-samples-owners @googleapis/actools-python From bc0abe48e1270d58245d04ba0bd5f878fa945c69 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 18 Nov 2021 14:58:20 -0500 Subject: [PATCH 041/126] ci: tweak mypy to check 'google.protobuf' (#310) Install the 'types-protobuf' package in support. --- google/__init__.py | 1 + mypy.ini | 3 --- noxfile.py | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/__init__.py b/google/__init__.py index 70a7bd99..9f1d5491 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -21,4 +21,5 @@ except ImportError: import pkgutil + # See: https://github.com/python/mypy/issues/1422 __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/mypy.ini b/mypy.ini index 5663b40d..5c111571 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,6 +2,3 @@ python_version = 3.6 namespace_packages = True ignore_missing_imports = True - -[mypy-google.protobuf] -ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index ac1bdd14..db37c561 100644 --- a/noxfile.py +++ b/noxfile.py @@ -170,7 +170,9 @@ def pytype(session): def mypy(session): """Run type-checking.""" session.install(".[grpc, grpcgcp]", "mypy") - session.install("types-setuptools", "types-requests", "types-mock") + session.install( + "types-setuptools", "types-requests", "types-protobuf", "types-mock" + ) session.run("mypy", "google", "tests") From ce1adf395982ede157c0f25a920946bb52789873 Mon Sep 17 00:00:00 2001 From: Ken Bandes Date: Wed, 24 Nov 2021 21:17:42 -0500 Subject: [PATCH 042/126] feat: add operations rest client to support long-running operations. (#311) * feat: add operations rest client to support long-running operations. * fix: address test coverage gaps in operations rest client. * fix: removed stray print statement. * fix: address lint, blacken, and mypy issues. * fix: address pytype, more coverage issues * fix: addressed additional pytype issues and one coverage line. * fix: renamed OperationsRestClient to AbstractOperationsClient. Co-authored-by: Kenneth Bandes --- google/api_core/operations_v1/__init__.py | 9 +- .../abstract_operations_client.py | 564 +++++++++++ google/api_core/operations_v1/pagers.py | 86 ++ .../operations_v1/transports/__init__.py | 30 + .../api_core/operations_v1/transports/base.py | 232 +++++ .../api_core/operations_v1/transports/rest.py | 455 +++++++++ .../test_operations_rest_client.py | 944 ++++++++++++++++++ 7 files changed, 2319 insertions(+), 1 deletion(-) create mode 100644 google/api_core/operations_v1/abstract_operations_client.py create mode 100644 google/api_core/operations_v1/pagers.py create mode 100644 google/api_core/operations_v1/transports/__init__.py create mode 100644 google/api_core/operations_v1/transports/base.py create mode 100644 google/api_core/operations_v1/transports/rest.py create mode 100644 tests/unit/operations_v1/test_operations_rest_client.py diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py index d7f963e2..61186451 100644 --- a/google/api_core/operations_v1/__init__.py +++ b/google/api_core/operations_v1/__init__.py @@ -14,7 +14,14 @@ """Package for interacting with the google.longrunning.operations meta-API.""" +from google.api_core.operations_v1.abstract_operations_client import AbstractOperationsClient from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient from google.api_core.operations_v1.operations_client import OperationsClient +from google.api_core.operations_v1.transports.rest import OperationsRestTransport -__all__ = ["OperationsAsyncClient", "OperationsClient"] +__all__ = [ + "AbstractOperationsClient", + "OperationsAsyncClient", + "OperationsClient", + "OperationsRestTransport" +] diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py new file mode 100644 index 00000000..631094e7 --- /dev/null +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -0,0 +1,564 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import Dict, Optional, Sequence, Tuple, Type, Union + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.api_core.operations_v1 import pagers +from google.api_core.operations_v1.transports.base import ( + DEFAULT_CLIENT_INFO, + OperationsTransport, +) +from google.api_core.operations_v1.transports.rest import OperationsRestTransport +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.auth.transport import mtls # type: ignore +from google.longrunning import operations_pb2 +from google.oauth2 import service_account # type: ignore + +OptionalRetry = Union[retries.Retry, object] + + +class AbstractOperationsClientMeta(type): + """Metaclass for the Operations client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + + _transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]] + _transport_registry["rest"] = OperationsRestTransport + + def get_transport_class( + cls, label: Optional[str] = None, + ) -> Type[OperationsTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class AbstractOperationsClient(metaclass=AbstractOperationsClientMeta): + """Manages long-running operations with an API service. + + When an API method normally takes long time to complete, it can be + designed to return [Operation][google.api_core.operations_v1.Operation] to the + client, and the client can use this interface to receive the real + response asynchronously by polling the operation resource, or pass + the operation resource to another API (such as Google Cloud Pub/Sub + API) to receive the response. Any API service that returns + long-running operations should implement the ``Operations`` + interface so developers can have a consistent client experience. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "longrunning.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + AbstractOperationsClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + AbstractOperationsClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> OperationsTransport: + """Returns the transport used by the client instance. + + Returns: + OperationsTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def common_billing_account_path(billing_account: str,) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str, str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str,) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder,) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str, str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str,) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization,) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str, str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str,) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project,) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str, str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str,) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str, str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__( + self, + *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, OperationsTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the operations client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, OperationsTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool( + util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) + ) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, OperationsTransport): + # transport is a OperationsTransport instance. + if credentials or client_options.credentials_file: + raise ValueError( + "When providing a transport instance, " + "provide its credentials directly." + ) + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + ) + + def list_operations( + self, + name: str, + filter_: Optional[str] = None, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> pagers.ListOperationsPager: + r"""Lists operations that match the specified filter in the request. + If the server doesn't support this method, it returns + ``UNIMPLEMENTED``. + + NOTE: the ``name`` binding allows API services to override the + binding to use different resource name schemes, such as + ``users/*/operations``. To override the binding, API services + can add a binding such as ``"/v1/{name=users/*}/operations"`` to + their service configuration. For backwards compatibility, the + default name includes the operations collection id, however + overriding users must ensure the name binding is the parent + resource, without the operations collection id. + + Args: + name (str): + The name of the operation's parent + resource. + filter_ (str): + The standard list filter. + This corresponds to the ``filter`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operations_v1.pagers.ListOperationsPager: + The response message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + Iterating over this object will yield results and + resolve additional pages automatically. + + """ + # Create a protobuf request object. + request = operations_pb2.ListOperationsRequest(name=name, filter=filter_) + if page_size is not None: + request.page_size = page_size + if page_token is not None: + request.page_token = page_token + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.list_operations] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # This method is paged; wrap the response in a pager, which provides + # an `__iter__` convenience method. + response = pagers.ListOperationsPager( + method=rpc, request=request, response=response, metadata=metadata, + ) + + # Done; return the response. + return response + + def get_operation( + self, + name: str, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.Operation: + r"""Gets the latest state of a long-running operation. + Clients can use this method to poll the operation result + at intervals as recommended by the API service. + + Args: + name (str): + The name of the operation resource. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.longrunning.operations_pb2.Operation: + This resource represents a long- + unning operation that is the result of a + network API call. + + """ + + request = operations_pb2.GetOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def delete_operation( + self, + name: str, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Deletes a long-running operation. This method indicates that the + client is no longer interested in the operation result. It does + not cancel the operation. If the server doesn't support this + method, it returns ``google.rpc.Code.UNIMPLEMENTED``. + + Args: + name (str): + The name of the operation resource to + be deleted. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create the request object. + request = operations_pb2.DeleteOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.delete_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + rpc( + request, retry=retry, timeout=timeout, metadata=metadata, + ) + + def cancel_operation( + self, + name: Optional[str] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Starts asynchronous cancellation on a long-running operation. + The server makes a best effort to cancel the operation, but + success is not guaranteed. If the server doesn't support this + method, it returns ``google.rpc.Code.UNIMPLEMENTED``. Clients + can use + [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation] + or other methods to check whether the cancellation succeeded or + whether the operation completed despite cancellation. On + successful cancellation, the operation is not deleted; instead, + it becomes an operation with an + [Operation.error][google.api_core.operations_v1.Operation.error] value with + a [google.rpc.Status.code][google.rpc.Status.code] of 1, + corresponding to ``Code.CANCELLED``. + + Args: + name (str): + The name of the operation resource to + be cancelled. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create the request object. + request = operations_pb2.CancelOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.cancel_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + rpc( + request, retry=retry, timeout=timeout, metadata=metadata, + ) diff --git a/google/api_core/operations_v1/pagers.py b/google/api_core/operations_v1/pagers.py new file mode 100644 index 00000000..b8a47757 --- /dev/null +++ b/google/api_core/operations_v1/pagers.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import ( + Any, + Callable, + Iterator, + Sequence, + Tuple, +) + +from google.longrunning import operations_pb2 + + +class ListOperationsPager: + """A pager for iterating through ``list_operations`` requests. + + This class thinly wraps an initial + :class:`google.longrunning.operations_pb2.ListOperationsResponse` object, and + provides an ``__iter__`` method to iterate through its + ``operations`` field. + + If there are more pages, the ``__iter__`` method will make additional + ``ListOperations`` requests and continue to iterate + through the ``operations`` field on the + corresponding responses. + + All the usual :class:`google.longrunning.operations_pb2.ListOperationsResponse` + attributes are available on the pager. If multiple requests are made, only + the most recent response is retained, and thus used for attribute lookup. + """ + + def __init__( + self, + method: Callable[..., operations_pb2.ListOperationsResponse], + request: operations_pb2.ListOperationsRequest, + response: operations_pb2.ListOperationsResponse, + *, + metadata: Sequence[Tuple[str, str]] = () + ): + """Instantiate the pager. + + Args: + method (Callable): The method that was originally called, and + which instantiated this pager. + request (google.longrunning.operations_pb2.ListOperationsRequest): + The initial request object. + response (google.longrunning.operations_pb2.ListOperationsResponse): + The initial response object. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + self._method = method + self._request = request + self._response = response + self._metadata = metadata + + def __getattr__(self, name: str) -> Any: + return getattr(self._response, name) + + @property + def pages(self) -> Iterator[operations_pb2.ListOperationsResponse]: + yield self._response + while self._response.next_page_token: + self._request.page_token = self._response.next_page_token + self._response = self._method(self._request, metadata=self._metadata) + yield self._response + + def __iter__(self) -> Iterator[operations_pb2.Operation]: + for page in self.pages: + yield from page.operations + + def __repr__(self) -> str: + return "{0}<{1!r}>".format(self.__class__.__name__, self._response) diff --git a/google/api_core/operations_v1/transports/__init__.py b/google/api_core/operations_v1/transports/__init__.py new file mode 100644 index 00000000..b443c078 --- /dev/null +++ b/google/api_core/operations_v1/transports/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import OperationsTransport +from .rest import OperationsRestTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]] +_transport_registry["rest"] = OperationsRestTransport + +__all__ = ( + "OperationsTransport", + "OperationsRestTransport", +) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py new file mode 100644 index 00000000..460e6460 --- /dev/null +++ b/google/api_core/operations_v1/transports/base.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Optional, Sequence, Union + +import pkg_resources + +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.longrunning import operations_pb2 +from google.oauth2 import service_account # type: ignore +from google.protobuf import empty_pb2 # type: ignore + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google.api_core.operations_v1", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +class OperationsTransport(abc.ABC): + """Abstract transport class for Operations.""" + + AUTH_SCOPES = () + + DEFAULT_HOST: str = "longrunning.googleapis.com" + + def __init__( + self, + *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ":" not in host: + host += ":443" + self._host = host + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs( + "'credentials_file' and 'credentials' are mutually exclusive" + ) + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, **scopes_kwargs, quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default( + **scopes_kwargs, quota_project_id=quota_project_id + ) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if ( + always_use_jwt_access + and isinstance(credentials, service_account.Credentials) + and hasattr(service_account.Credentials, "with_always_use_jwt_access") + ): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.list_operations: gapic_v1.method.wrap_method( + self.list_operations, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.get_operation: gapic_v1.method.wrap_method( + self.get_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.delete_operation: gapic_v1.method.wrap_method( + self.delete_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.cancel_operation: gapic_v1.method.wrap_method( + self.cancel_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def list_operations( + self, + ) -> Callable[ + [operations_pb2.ListOperationsRequest], + Union[ + operations_pb2.ListOperationsResponse, + Awaitable[operations_pb2.ListOperationsResponse], + ], + ]: + raise NotImplementedError() + + @property + def get_operation( + self, + ) -> Callable[ + [operations_pb2.GetOperationRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + + @property + def delete_operation( + self, + ) -> Callable[ + [operations_pb2.DeleteOperationRequest], + Union[empty_pb2.Empty, Awaitable[empty_pb2.Empty]], + ]: + raise NotImplementedError() + + @property + def cancel_operation( + self, + ) -> Callable[ + [operations_pb2.CancelOperationRequest], + Union[empty_pb2.Empty, Awaitable[empty_pb2.Empty]], + ]: + raise NotImplementedError() + + +__all__ = ("OperationsTransport",) diff --git a/google/api_core/operations_v1/transports/rest.py b/google/api_core/operations_v1/transports/rest.py new file mode 100644 index 00000000..27ed7661 --- /dev/null +++ b/google/api_core/operations_v1/transports/rest.py @@ -0,0 +1,455 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from requests import __version__ as requests_version + +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import path_template # type: ignore +from google.api_core import rest_helpers # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.requests import AuthorizedSession # type: ignore +from google.longrunning import operations_pb2 # type: ignore +from google.protobuf import empty_pb2 # type: ignore +from google.protobuf import json_format # type: ignore +from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO, OperationsTransport + +OptionalRetry = Union[retries.Retry, object] + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=BASE_DEFAULT_CLIENT_INFO.gapic_version, + grpc_version=None, + rest_version=requests_version, +) + + +class OperationsRestTransport(OperationsTransport): + """REST backend transport for Operations. + + Manages long-running operations with an API service. + + When an API method normally takes long time to complete, it can be + designed to return [Operation][google.api_core.operations_v1.Operation] to the + client, and the client can use this interface to receive the real + response asynchronously by polling the operation resource, or pass + the operation resource to another API (such as Google Cloud Pub/Sub + API) to receive the response. Any API service that returns + long-running operations should implement the ``Operations`` + interface so developers can have a consistent client experience. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends JSON representations of protocol buffers over HTTP/1.1 + """ + + def __init__( + self, + *, + host: str = "longrunning.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + url_scheme: str = "https", + http_options: Optional[Dict] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client + certificate to configure mutual TLS HTTP channel. It is ignored + if ``channel`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + url_scheme: the protocol scheme for the API endpoint. Normally + "https", but for testing or local servers, + "http" can be specified. + http_options: a dictionary of http_options for transcoding, to override + the defaults from operatons.proto. Each method has an entry + with the corresponding http rules as value. + + """ + # Run the base constructor + # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. + # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the + # credentials object + super().__init__( + host=host, + credentials=credentials, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + self._session = AuthorizedSession( + self._credentials, default_host=self.DEFAULT_HOST + ) + if client_cert_source_for_mtls: + self._session.configure_mtls_channel(client_cert_source_for_mtls) + self._prep_wrapped_messages(client_info) + self._http_options = http_options or {} + + def _list_operations( + self, + request: operations_pb2.ListOperationsRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.ListOperationsResponse: + r"""Call the list operations method over HTTP. + + Args: + request (~.operations_pb2.ListOperationsRequest): + The request object. The request message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + ~.operations_pb2.ListOperationsResponse: + The response message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + """ + + http_options = [ + {"method": "get", "uri": "/v1/{name=operations}"}, + ] + if "google.longrunning.Operations.ListOperations" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.ListOperations" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.ListOperationsRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + api_response = operations_pb2.ListOperationsResponse() + json_format.Parse(response.content, api_response, ignore_unknown_fields=False) + return api_response + + def _get_operation( + self, + request: operations_pb2.GetOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.Operation: + r"""Call the get operation method over HTTP. + + Args: + request (~.operations_pb2.GetOperationRequest): + The request object. The request message for + [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + ~.operations_pb2.Operation: + This resource represents a long- + unning operation that is the result of a + network API call. + + """ + + http_options = [ + {"method": "get", "uri": "/v1/{name=operations/**}"}, + ] + if "google.longrunning.Operations.GetOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.GetOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.GetOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + api_response = operations_pb2.Operation() + json_format.Parse(response.content, api_response, ignore_unknown_fields=False) + return api_response + + def _delete_operation( + self, + request: operations_pb2.DeleteOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> empty_pb2.Empty: + r"""Call the delete operation method over HTTP. + + Args: + request (~.operations_pb2.DeleteOperationRequest): + The request object. The request message for + [Operations.DeleteOperation][google.api_core.operations_v1.Operations.DeleteOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + + http_options = [ + {"method": "delete", "uri": "/v1/{name=operations/**}"}, + ] + if "google.longrunning.Operations.DeleteOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.DeleteOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.DeleteOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + return empty_pb2.Empty() + + def _cancel_operation( + self, + request: operations_pb2.CancelOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> empty_pb2.Empty: + r"""Call the cancel operation method over HTTP. + + Args: + request (~.operations_pb2.CancelOperationRequest): + The request object. The request message for + [Operations.CancelOperation][google.api_core.operations_v1.Operations.CancelOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + + http_options = [ + {"method": "post", "uri": "/v1/{name=operations/**}:cancel", "body": "*"}, + ] + if "google.longrunning.Operations.CancelOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.CancelOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + # Jsonify the request body + body_request = operations_pb2.CancelOperationRequest() + json_format.ParseDict(transcoded_request["body"], body_request) + body = json_format.MessageToDict( + body_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.CancelOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + data=body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + return empty_pb2.Empty() + + @property + def list_operations( + self, + ) -> Callable[ + [operations_pb2.ListOperationsRequest], operations_pb2.ListOperationsResponse + ]: + return self._list_operations + + @property + def get_operation( + self, + ) -> Callable[[operations_pb2.GetOperationRequest], operations_pb2.Operation]: + return self._get_operation + + @property + def delete_operation( + self, + ) -> Callable[[operations_pb2.DeleteOperationRequest], empty_pb2.Empty]: + return self._delete_operation + + @property + def cancel_operation( + self, + ) -> Callable[[operations_pb2.CancelOperationRequest], empty_pb2.Empty]: + return self._cancel_operation + + +__all__ = ("OperationsRestTransport",) diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py new file mode 100644 index 00000000..dddf6b71 --- /dev/null +++ b/tests/unit/operations_v1/test_operations_rest_client.py @@ -0,0 +1,944 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) +from requests import Response # noqa I201 +from requests.sessions import Session + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core.operations_v1 import AbstractOperationsClient +from google.api_core.operations_v1 import pagers +from google.api_core.operations_v1 import transports +import google.auth +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.longrunning import operations_pb2 +from google.oauth2 import service_account +from google.protobuf import json_format # type: ignore +from google.rpc import status_pb2 # type: ignore + + +HTTP_OPTIONS = { + "google.longrunning.Operations.CancelOperation": [ + {"method": "post", "uri": "/v3/{name=operations/*}:cancel", "body": "*"}, + ], + "google.longrunning.Operations.DeleteOperation": [ + {"method": "delete", "uri": "/v3/{name=operations/*}"}, + ], + "google.longrunning.Operations.GetOperation": [ + {"method": "get", "uri": "/v3/{name=operations/*}"}, + ], + "google.longrunning.Operations.ListOperations": [ + {"method": "get", "uri": "/v3/{name=operations}"}, + ], +} + + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +def _get_operations_client(http_options=HTTP_OPTIONS): + transport = transports.rest.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), http_options=http_options + ) + + return AbstractOperationsClient(transport=transport) + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return ( + "foo.googleapis.com" + if ("localhost" in client.DEFAULT_ENDPOINT) + else client.DEFAULT_ENDPOINT + ) + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert AbstractOperationsClient._get_default_mtls_endpoint(None) is None + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(api_endpoint) + == api_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(api_mtls_endpoint) + == api_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(sandbox_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(non_googleapi) + == non_googleapi + ) + + +@pytest.mark.parametrize("client_class", [AbstractOperationsClient]) +def test_operations_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_info" + ) as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "longrunning.googleapis.com:443" + + +@pytest.mark.parametrize( + "transport_class,transport_name", [(transports.OperationsRestTransport, "rest")] +) +def test_operations_client_service_account_always_use_jwt( + transport_class, transport_name +): + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [AbstractOperationsClient]) +def test_operations_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_file" + ) as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "longrunning.googleapis.com:443" + + +def test_operations_client_get_transport_class(): + transport = AbstractOperationsClient.get_transport_class() + available_transports = [ + transports.OperationsRestTransport, + ] + assert transport in available_transports + + transport = AbstractOperationsClient.get_transport_class("rest") + assert transport == transports.OperationsRestTransport + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +@mock.patch.object( + AbstractOperationsClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(AbstractOperationsClient), +) +def test_operations_client_client_options( + client_class, transport_class, transport_name +): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc: + transport = transport_class(credentials=ga_credentials.AnonymousCredentials()) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,use_client_cert_env", + [ + (AbstractOperationsClient, transports.OperationsRestTransport, "rest", "true"), + (AbstractOperationsClient, transports.OperationsRestTransport, "rest", "false"), + ], +) +@mock.patch.object( + AbstractOperationsClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(AbstractOperationsClient), +) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_operations_client_mtls_env_auto( + client_class, transport_class, transport_name, use_client_cert_env +): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + options = client_options.ClientOptions( + client_cert_source=client_cert_source_callback + ) + + def fake_init(client_cert_source_for_mtls=None, **kwargs): + """Invoke client_cert source if provided.""" + + if client_cert_source_for_mtls: + client_cert_source_for_mtls() + return None + + with mock.patch.object(transport_class, "__init__") as patched: + patched.side_effect = fake_init + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=client_cert_source_callback, + ): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +def test_operations_client_client_options_scopes( + client_class, transport_class, transport_name +): + # Check the case scopes are provided. + options = client_options.ClientOptions(scopes=["1", "2"],) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +def test_operations_client_client_options_credentials_file( + client_class, transport_class, transport_name +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_list_operations_rest( + transport: str = "rest", request_type=operations_pb2.ListOperationsRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Designate an appropriate value for the returned response. + return_value = operations_pb2.ListOperationsResponse( + next_page_token="next_page_token_value", + ) + + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + response = client.list_operations( + name="operations", filter_="my_filter", page_size=10, page_token="abc" + ) + + actual_args = req.call_args + assert actual_args.args[0] == "GET" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations" + ) + assert actual_args.kwargs["params"] == [ + ("filter", "my_filter"), + ("pageSize", 10), + ("pageToken", "abc"), + ] + + # Establish that the response is the type that we expect. + assert isinstance(response, pagers.ListOperationsPager) + assert response.next_page_token == "next_page_token_value" + + +def test_list_operations_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "GET" + mock_request.url = "https://longrunning.googleapis.com:443/v1/operations" + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.list_operations(name="operations") + + +def test_list_operations_rest_pager(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # TODO(kbandes): remove this mock unless there's a good reason for it. + # with mock.patch.object(path_template, 'transcode') as transcode: + # Set the response as a series of pages + response = ( + operations_pb2.ListOperationsResponse( + operations=[ + operations_pb2.Operation(), + operations_pb2.Operation(), + operations_pb2.Operation(), + ], + next_page_token="abc", + ), + operations_pb2.ListOperationsResponse( + operations=[], next_page_token="def", + ), + operations_pb2.ListOperationsResponse( + operations=[operations_pb2.Operation()], next_page_token="ghi", + ), + operations_pb2.ListOperationsResponse( + operations=[operations_pb2.Operation(), operations_pb2.Operation()], + ), + ) + # Two responses for two calls + response = response + response + + # Wrap the values into proper Response objs + response = tuple(json_format.MessageToJson(x) for x in response) + return_values = tuple(Response() for i in response) + for return_val, response_val in zip(return_values, response): + return_val._content = response_val.encode("UTF-8") + return_val.status_code = 200 + req.side_effect = return_values + + pager = client.list_operations(name="operations") + + results = list(pager) + assert len(results) == 6 + assert all(isinstance(i, operations_pb2.Operation) for i in results) + + pages = list(client.list_operations(name="operations").pages) + for page_, token in zip(pages, ["abc", "def", "ghi", ""]): + assert page_.next_page_token == token + + +def test_get_operation_rest( + transport: str = "rest", request_type=operations_pb2.GetOperationRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Designate an appropriate value for the returned response. + return_value = operations_pb2.Operation( + name="operations/sample1", done=True, error=status_pb2.Status(code=411), + ) + + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + response = client.get_operation("operations/sample1") + + actual_args = req.call_args + assert actual_args.args[0] == "GET" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1" + ) + + # Establish that the response is the type that we expect. + assert isinstance(response, operations_pb2.Operation) + assert response.name == "operations/sample1" + assert response.done is True + + +def test_get_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "GET" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.get_operation("operations/sample1") + + +def test_delete_operation_rest( + transport: str = "rest", request_type=operations_pb2.DeleteOperationRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = "" + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + client.delete_operation(name="operations/sample1") + assert req.call_count == 1 + actual_args = req.call_args + assert actual_args.args[0] == "DELETE" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1" + ) + + +def test_delete_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "DELETE" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.delete_operation(name="operations/sample1") + + +def test_cancel_operation_rest(transport: str = "rest"): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = "" + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + client.cancel_operation(name="operations/sample1") + assert req.call_count == 1 + actual_args = req.call_args + assert actual_args.args[0] == "POST" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1:cancel" + ) + + +def test_cancel_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "POST" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1:cancel" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.cancel_operation(name="operations/sample1") + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + client_options={"scopes": ["1", "2"]}, transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = AbstractOperationsClient(transport=transport) + assert client.transport is transport + + +@pytest.mark.parametrize("transport_class", [transports.OperationsRestTransport]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + + +def test_operations_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transports.OperationsTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json", + ) + + +def test_operations_base_transport(): + # Instantiate the base transport. + with mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport.__init__" + ) as Transport: + Transport.return_value = None + transport = transports.OperationsTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + "list_operations", + "get_operation", + "delete_operation", + "cancel_operation", + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + +def test_operations_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transports.OperationsTransport( + credentials_file="credentials.json", quota_project_id="octopus", + ) + load_creds.assert_called_once_with( + "credentials.json", + scopes=None, + default_scopes=(), + quota_project_id="octopus", + ) + + +def test_operations_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transports.OperationsTransport() + adc.assert_called_once() + + +def test_operations_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + AbstractOperationsClient() + adc.assert_called_once_with( + scopes=None, default_scopes=(), quota_project_id=None, + ) + + +def test_operations_http_transport_client_cert_source_for_mtls(): + cred = ga_credentials.AnonymousCredentials() + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ) as mock_configure_mtls_channel: + transports.OperationsRestTransport( + credentials=cred, client_cert_source_for_mtls=client_cert_source_callback + ) + mock_configure_mtls_channel.assert_called_once_with(client_cert_source_callback) + + +def test_operations_host_no_port(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="longrunning.googleapis.com" + ), + ) + assert client.transport._host == "longrunning.googleapis.com:443" + + +def test_operations_host_with_port(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="longrunning.googleapis.com:8000" + ), + ) + assert client.transport._host == "longrunning.googleapis.com:8000" + + +def test_common_billing_account_path(): + billing_account = "squid" + expected = "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + actual = AbstractOperationsClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "clam", + } + path = AbstractOperationsClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_billing_account_path(path) + assert expected == actual + + +def test_common_folder_path(): + folder = "whelk" + expected = "folders/{folder}".format(folder=folder,) + actual = AbstractOperationsClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "octopus", + } + path = AbstractOperationsClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_folder_path(path) + assert expected == actual + + +def test_common_organization_path(): + organization = "oyster" + expected = "organizations/{organization}".format(organization=organization,) + actual = AbstractOperationsClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "nudibranch", + } + path = AbstractOperationsClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_organization_path(path) + assert expected == actual + + +def test_common_project_path(): + project = "cuttlefish" + expected = "projects/{project}".format(project=project,) + actual = AbstractOperationsClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "mussel", + } + path = AbstractOperationsClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_project_path(path) + assert expected == actual + + +def test_common_location_path(): + project = "winkle" + location = "nautilus" + expected = "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + actual = AbstractOperationsClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "scallop", + "location": "abalone", + } + path = AbstractOperationsClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object( + transports.OperationsTransport, "_prep_wrapped_messages" + ) as prep: + AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object( + transports.OperationsTransport, "_prep_wrapped_messages" + ) as prep: + transport_class = AbstractOperationsClient.get_transport_class() + transport_class( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info) From cb091f8669124be83df78424afed442b37767acc Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 7 Dec 2021 15:18:37 -0800 Subject: [PATCH 043/126] chore: release 2.3.0 (#314) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 ++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ce6afc..04d47daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.3.0](https://www.github.com/googleapis/python-api-core/compare/v2.2.2...v2.3.0) (2021-11-25) + + +### Features + +* add operations rest client to support long-running operations. ([#311](https://www.github.com/googleapis/python-api-core/issues/311)) ([ce1adf3](https://www.github.com/googleapis/python-api-core/commit/ce1adf395982ede157c0f25a920946bb52789873)) + + +### Bug Fixes + +* handle bare 'grpc.Call' in 'from_grpc_error' ([#298](https://www.github.com/googleapis/python-api-core/issues/298)) ([060b339](https://www.github.com/googleapis/python-api-core/commit/060b339e3af296dd1772bfc1b4a0d2b4264cae1f)) + ### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) diff --git a/google/api_core/version.py b/google/api_core/version.py index bcc59c44..999199f5 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.2" +__version__ = "2.3.0" From 479d6a7474f79947115d560f34fb971295c6af8b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 8 Dec 2021 18:25:04 -0500 Subject: [PATCH 044/126] ci: run lint / mypy / unit tests / coverage as GH actions (#287) Make GH Action checks required --- .github/sync-repo-settings.yaml | 25 ++++++ .github/workflows/unittest_lint.yml | 119 ++++++++++++++++++++++++++++ noxfile.py | 2 +- 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/unittest_lint.yml diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index e621885d..67fcd59f 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,3 +1,28 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: true + requiredStatusCheckContexts: + - 'cla/google' + # No Kokoro: the following are Github actions + - 'lint-mypy' + - 'unit-3.6' + - 'unit-3.7' + - 'unit-3.8' + - 'unit-3.9' + - 'unit-3.10' + - 'unit_grpc_gcp-3.6' + - 'unit_grpc_gcp-3.7' + - 'unit_grpc_gcp-3.8' + - 'unit_grpc_gcp-3.9' + - 'unit_grpc_gcp-3.10' + - 'unit_wo_grpc-3.6' + - 'unit_wo_grpc-3.10' + - 'cover' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/unittest_lint.yml b/.github/workflows/unittest_lint.yml new file mode 100644 index 00000000..302be970 --- /dev/null +++ b/.github/workflows/unittest_lint.yml @@ -0,0 +1,119 @@ +name: "Lint / Unit tests / Cover / Mypy" + +on: + pull_request: + branches: + - main + + +jobs: + + run-lint-mypy: + name: lint-mypy + runs-on: ubuntu-latest + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run lint + run: | + nox -s lint + + - name: Run lint_setup_py + run: | + nox -s lint_setup_py + + - name: Run mypy + run: | + nox -s mypy + + run-unittests: + name: unit${{ matrix.option }}-${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + matrix: + option: ["", "_grpc_gcp", "_wo_grpc"] + python: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" + exclude: + - option: "_wo_grpc" + python: 3.7 + - option: "_wo_grpc" + python: 3.8 + - option: "_wo_grpc" + python: 3.9 + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run unit tests + env: + COVERAGE_FILE: .coverage${{ matrix.option }}-${{matrix.python }} + run: | + nox -s unit${{ matrix.option }}-${{ matrix.python }} + + - name: Upload coverage results + uses: actions/upload-artifact@v2 + with: + name: coverage-artifacts + path: .coverage${{ matrix.option }}-${{ matrix.python }} + + report-coverage: + name: cover + runs-on: ubuntu-latest + needs: + - run-unittests + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install coverage + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-artifacts + path: .coverage-results/ + + - name: Report coverage results + run: | + coverage combine .coverage-results/.coverage* + coverage report --show-missing --fail-under=100 diff --git a/noxfile.py b/noxfile.py index db37c561..003a276d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -132,7 +132,7 @@ def unit(session): default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( From cc46aa68ec184871330d16a6c767f57a4f0eb633 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Thu, 9 Dec 2021 10:32:49 -0800 Subject: [PATCH 045/126] feat: add support for 'error_info' (#315) * feat: Adds support for error_info. * chore: fixes pytype. Co-authored-by: Tres Seaver --- google/api_core/exceptions.py | 71 +++++++++++++++++++++++++++++++---- tests/unit/test_exceptions.py | 62 +++++++++++++++++++++++++----- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 6b1b6f7e..24b65ee0 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -104,6 +104,8 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. + error_info (Union[error_details_pb2.ErrorInfo, None]): An optional object containing error info + (google.rpc.error_details.ErrorInfo). """ code: Union[int, None] = None @@ -122,13 +124,14 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): This may be ``None`` if the exception does not match up to a gRPC error. """ - def __init__(self, message, errors=(), details=(), response=None): + def __init__(self, message, errors=(), details=(), response=None, error_info=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors self._details = details self._response = response + self._error_info = error_info def __str__(self): if self.details: @@ -136,6 +139,42 @@ def __str__(self): else: return "{} {}".format(self.code, self.message) + @property + def reason(self): + """The reason of the error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing reason of the error. + """ + return self._error_info.reason if self._error_info else None + + @property + def domain(self): + """The logical grouping to which the "reason" belongs. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing a logical grouping to which the "reason" belongs. + """ + return self._error_info.domain if self._error_info else None + + @property + def metadata(self): + """Additional structured details about this error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[Dict[str, str], None]: An optional object containing structured details about the error. + """ + return self._error_info.metadata if self._error_info else None + @property def errors(self): """Detailed error information. @@ -433,13 +472,26 @@ def from_http_response(response): errors = payload.get("error", {}).get("errors", ()) # In JSON, details are already formatted in developer-friendly way. details = payload.get("error", {}).get("details", ()) + error_info = list( + filter( + lambda detail: detail.get("@type", "") + == "type.googleapis.com/google.rpc.ErrorInfo", + details, + ) + ) + error_info = error_info[0] if error_info else None message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message + method=response.request.method, url=response.request.url, error=error_message, ) exception = from_http_status( - response.status_code, message, errors=errors, details=details, response=response + response.status_code, + message, + errors=errors, + details=details, + response=response, + error_info=error_info, ) return exception @@ -490,10 +542,10 @@ def _parse_grpc_error_details(rpc_exc): try: status = rpc_status.from_call(rpc_exc) except NotImplementedError: # workaround - return [] + return [], None if not status: - return [] + return [], None possible_errors = [ error_details_pb2.BadRequest, @@ -507,6 +559,7 @@ def _parse_grpc_error_details(rpc_exc): error_details_pb2.Help, error_details_pb2.LocalizedMessage, ] + error_info = None error_details = [] for detail in status.details: matched_detail_cls = list( @@ -519,7 +572,9 @@ def _parse_grpc_error_details(rpc_exc): info = matched_detail_cls[0]() detail.Unpack(info) error_details.append(info) - return error_details + if isinstance(info, error_details_pb2.ErrorInfo): + error_info = info + return error_details, error_info def from_grpc_error(rpc_exc): @@ -535,12 +590,14 @@ def from_grpc_error(rpc_exc): # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): + details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), - details=_parse_grpc_error_details(rpc_exc), + details=details, response=rpc_exc, + error_info=err_info, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 622f58ab..4169ad44 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -275,31 +275,56 @@ def create_bad_request_details(): return status_detail +def create_error_info_details(): + info = error_details_pb2.ErrorInfo( + reason="SERVICE_DISABLED", + domain="googleapis.com", + metadata={ + "consumer": "projects/455411330361", + "service": "translate.googleapis.com", + }, + ) + status_detail = any_pb2.Any() + status_detail.Pack(info) + return status_detail + + def test_error_details_from_rest_response(): bad_request_detail = create_bad_request_details() + error_info_detail = create_error_info_details() status = status_pb2.Status() status.code = 3 status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) status.details.append(bad_request_detail) + status.details.append(error_info_detail) # See JSON schema in https://cloud.google.com/apis/design/errors#http_mapping http_response = make_response( - json.dumps({"error": json.loads(json_format.MessageToJson(status))}).encode( - "utf-8" - ) + json.dumps( + {"error": json.loads(json_format.MessageToJson(status, sort_keys=True))} + ).encode("utf-8") ) exception = exceptions.from_http_response(http_response) - want_error_details = [json.loads(json_format.MessageToJson(bad_request_detail))] + want_error_details = [ + json.loads(json_format.MessageToJson(bad_request_detail)), + json.loads(json_format.MessageToJson(error_info_detail)), + ] assert want_error_details == exception.details + # 404 POST comes from make_response. assert str(exception) == ( "404 POST https://example.com/: 3 INVALID_ARGUMENT:" " One of content, or gcs_content_uri must be set." " [{'@type': 'type.googleapis.com/google.rpc.BadRequest'," - " 'fieldViolations': [{'field': 'document.content'," - " 'description': 'Must have some text content to annotate.'}]}]" + " 'fieldViolations': [{'description': 'Must have some text content to annotate.'," + " 'field': 'document.content'}]}," + " {'@type': 'type.googleapis.com/google.rpc.ErrorInfo'," + " 'domain': 'googleapis.com'," + " 'metadata': {'consumer': 'projects/455411330361'," + " 'service': 'translate.googleapis.com'}," + " 'reason': 'SERVICE_DISABLED'}]" ) @@ -311,6 +336,11 @@ def test_error_details_from_v1_rest_response(): ) exception = exceptions.from_http_response(response) assert exception.details == [] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -320,8 +350,10 @@ def test_error_details_from_grpc_response(): status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) - status_detail = create_bad_request_details() - status.details.append(status_detail) + status_br_detail = create_bad_request_details() + status_ei_detail = create_error_info_details() + status.details.append(status_br_detail) + status.details.append(status_ei_detail) # Actualy error doesn't matter as long as its grpc.Call, # because from_call is mocked. @@ -331,8 +363,13 @@ def test_error_details_from_grpc_response(): exception = exceptions.from_grpc_error(error) bad_request_detail = error_details_pb2.BadRequest() - status_detail.Unpack(bad_request_detail) - assert exception.details == [bad_request_detail] + error_info_detail = error_details_pb2.ErrorInfo() + status_br_detail.Unpack(bad_request_detail) + status_ei_detail.Unpack(error_info_detail) + assert exception.details == [bad_request_detail, error_info_detail] + assert exception.reason == error_info_detail.reason + assert exception.domain == error_info_detail.domain + assert exception.metadata == error_info_detail.metadata @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -351,3 +388,8 @@ def test_error_details_from_grpc_response_unknown_error(): m.return_value = status exception = exceptions.from_grpc_error(error) assert exception.details == [status_detail] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) From 34ebdcc251d4f3d7d496e8e0b78847645a06650b Mon Sep 17 00:00:00 2001 From: Chris Wilson <46912004+sushicw@users.noreply.github.com> Date: Wed, 15 Dec 2021 02:12:56 -0800 Subject: [PATCH 046/126] fix: exclude function target from retry deadline exceeded exception message (#318) * Exclude function target from retry deadline exceeded exception message * apply similar patch in retry_async.py Co-authored-by: Anthonios Partheniou --- google/api_core/retry.py | 4 ++-- google/api_core/retry_async.py | 4 ++-- tests/asyncio/test_retry_async.py | 4 ++++ tests/unit/test_retry.py | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index bd3a4a65..ce496937 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -203,8 +203,8 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if deadline_datetime is not None: if deadline_datetime <= now: raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/google/api_core/retry_async.py b/google/api_core/retry_async.py index 2dfa2f6e..68a25597 100644 --- a/google/api_core/retry_async.py +++ b/google/api_core/retry_async.py @@ -132,8 +132,8 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No # Chains the raising RetryError with the root cause error, # which helps observability and debugability. raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/tests/asyncio/test_retry_async.py b/tests/asyncio/test_retry_async.py index 9e51044b..873caaf1 100644 --- a/tests/asyncio/test_retry_async.py +++ b/tests/asyncio/test_retry_async.py @@ -120,6 +120,10 @@ async def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + @pytest.mark.asyncio async def test_retry_target_bad_sleep_generator(): diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 199ca559..74c5d77c 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -152,6 +152,10 @@ def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + def test_retry_target_bad_sleep_generator(): with pytest.raises(ValueError, match="Sleep generator"): From 6538e0322a82831c82646cf29524bd913cf624b6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:06:17 +0000 Subject: [PATCH 047/126] chore: use python-samples-reviewers (#328) --- .github/.OwlBot.lock.yaml | 2 +- .github/CODEOWNERS | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7519fa3a..f33299dd 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 + digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee818917..1b023b72 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,5 +8,5 @@ # @googleapis/yoshi-python @googleapis/actools-python are the default owners for changes in this repo * @googleapis/yoshi-python @googleapis/actools-python -# @googleapis/python-samples-owners @googleapis/actools-python are the default owners for samples changes -/samples/ @googleapis/python-samples-owners @googleapis/actools-python +# @googleapis/python-samples-reviewers @googleapis/actools-python are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/actools-python From 69a99d8dfd9492d3d449de99845d3f134ce1f40c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 7 Jan 2022 13:48:52 -0500 Subject: [PATCH 048/126] chore: update release_level in repo-metadata.json (#326) * chore: update .repo-metadata.json * revert * remove api_shortname --- .repo-metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index e16c9d27..0f0abd93 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,7 +2,7 @@ "name": "google-api-core", "name_pretty": "Google API client core library", "client_documentation": "https://googleapis.dev/python/google-api-core/latest", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "CORE", "repo": "googleapis/python-api-core", From f9f26969842b456ea372bed941d712b7a9ab7239 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Mon, 10 Jan 2022 09:31:35 -0800 Subject: [PATCH 049/126] feat: iterator for processing JSON responses in REST streaming. (#317) * feat: files for REST server streaming. --- google/api_core/rest_streaming.py | 114 ++++++++++++++++ tests/unit/test_rest_streaming.py | 211 ++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 google/api_core/rest_streaming.py create mode 100644 tests/unit/test_rest_streaming.py diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py new file mode 100644 index 00000000..69f5b41b --- /dev/null +++ b/google/api_core/rest_streaming.py @@ -0,0 +1,114 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for server-side streaming in REST.""" + +from collections import deque +import string +from typing import Deque + +import requests + + +class ResponseIterator: + """Iterator over REST API responses. + + Args: + response (requests.Response): An API response object. + response_message_cls (Callable[proto.Message]): A proto + class expected to be returned from an API. + """ + + def __init__(self, response: requests.Response, response_message_cls): + self._response = response + self._response_message_cls = response_message_cls + # Inner iterator over HTTP response's content. + self._response_itr = self._response.iter_content(decode_unicode=True) + # Contains a list of JSON responses ready to be sent to user. + self._ready_objs: Deque[str] = deque() + # Current JSON response being built. + self._obj = "" + # Keeps track of the nesting level within a JSON object. + self._level = 0 + # Keeps track whether HTTP response is currently sending values + # inside of a string value. + self._in_string = False + # Whether an escape symbol "\" was encountered. + self._escape_next = False + + def cancel(self): + """Cancel existing streaming operation. + """ + self._response.close() + + def _process_chunk(self, chunk: str): + if self._level == 0: + if chunk[0] != "[": + raise ValueError( + "Can only parse array of JSON objects, instead got %s" % chunk + ) + for char in chunk: + if char == "{": + if self._level == 1: + # Level 1 corresponds to the outermost JSON object + # (i.e. the one we care about). + self._obj = "" + if not self._in_string: + self._level += 1 + self._obj += char + elif char == "}": + self._obj += char + if not self._in_string: + self._level -= 1 + if not self._in_string and self._level == 1: + self._ready_objs.append(self._obj) + elif char == '"': + # Helps to deal with an escaped quotes inside of a string. + if not self._escape_next: + self._in_string = not self._in_string + self._obj += char + elif char in string.whitespace: + if self._in_string: + self._obj += char + elif char == "[": + if self._level == 0: + self._level += 1 + else: + self._obj += char + elif char == "]": + if self._level == 1: + self._level -= 1 + else: + self._obj += char + else: + self._obj += char + self._escape_next = not self._escape_next if char == "\\" else False + + def __next__(self): + while not self._ready_objs: + try: + chunk = next(self._response_itr) + self._process_chunk(chunk) + except StopIteration as e: + if self._level > 0: + raise ValueError("Unfinished stream: %s" % self._obj) + raise e + return self._grab() + + def _grab(self): + # Add extra quotes to make json.loads happy. + return self._response_message_cls.from_json(self._ready_objs.popleft()) + + def __iter__(self): + return self diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py new file mode 100644 index 00000000..4be59580 --- /dev/null +++ b/tests/unit/test_rest_streaming.py @@ -0,0 +1,211 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import random +import time +from typing import List +from unittest.mock import patch + +import proto +import pytest +import requests + +from google.api_core import rest_streaming +from google.protobuf import duration_pb2 +from google.protobuf import timestamp_pb2 + + +SEED = int(time.time()) +logging.info(f"Starting rest streaming tests with random seed: {SEED}") +random.seed(SEED) + + +class Genre(proto.Enum): + GENRE_UNSPECIFIED = 0 + CLASSICAL = 1 + JAZZ = 2 + ROCK = 3 + + +class Composer(proto.Message): + given_name = proto.Field(proto.STRING, number=1) + family_name = proto.Field(proto.STRING, number=2) + relateds = proto.RepeatedField(proto.STRING, number=3) + indices = proto.MapField(proto.STRING, proto.STRING, number=4) + + +class Song(proto.Message): + composer = proto.Field(Composer, number=1) + title = proto.Field(proto.STRING, number=2) + lyrics = proto.Field(proto.STRING, number=3) + year = proto.Field(proto.INT32, number=4) + genre = proto.Field(Genre, number=5) + is_five_mins_longer = proto.Field(proto.BOOL, number=6) + score = proto.Field(proto.DOUBLE, number=7) + likes = proto.Field(proto.INT64, number=8) + duration = proto.Field(duration_pb2.Duration, number=9) + date_added = proto.Field(timestamp_pb2.Timestamp, number=10) + + +class EchoResponse(proto.Message): + content = proto.Field(proto.STRING, number=1) + + +class ResponseMock(requests.Response): + class _ResponseItr: + def __init__(self, _response_bytes: bytes, random_split=False): + self._responses_bytes = _response_bytes + self._i = 0 + self._random_split = random_split + + def __next__(self): + if self._i == len(self._responses_bytes): + raise StopIteration + if self._random_split: + n = random.randint(1, len(self._responses_bytes[self._i :])) + else: + n = 1 + x = self._responses_bytes[self._i : self._i + n] + self._i += n + return x.decode("utf-8") + + def __init__( + self, responses: List[proto.Message], response_cls, random_split=False, + ): + super().__init__() + self._responses = responses + self._random_split = random_split + self._response_message_cls = response_cls + + def _parse_responses(self, responses: List[proto.Message]) -> bytes: + # json.dumps returns a string surrounded with quotes that need to be stripped + # in order to be an actual JSON. + json_responses = [ + self._response_message_cls.to_json(r).strip('"') for r in responses + ] + logging.info(f"Sending JSON stream: {json_responses}") + ret_val = "[{}]".format(",".join(json_responses)) + return bytes(ret_val, "utf-8") + + def close(self): + raise NotImplementedError() + + def iter_content(self, *args, **kwargs): + return self._ResponseItr( + self._parse_responses(self._responses), random_split=self._random_split, + ) + + +@pytest.mark.parametrize("random_split", [False]) +def test_next_simple(random_split): + responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=EchoResponse + ) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_nested(random_split): + responses = [ + Song(title="some song", composer=Composer(given_name="some name")), + Song(title="another song", date_added=datetime.datetime(2021, 12, 17)), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_stress(random_split): + n = 50 + responses = [ + Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i)) + for i in range(n) + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_escaped_characters_in_string(random_split): + composer_with_relateds = Composer() + relateds = ["Artist A", "Artist B"] + composer_with_relateds.relateds = relateds + + responses = [ + Song(title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")), + Song( + title='{"this is weird": "totally"}', composer=Composer(given_name="\\{}\\") + ), + Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +def test_next_not_array(): + with patch.object( + ResponseMock, "iter_content", return_value=iter('{"hello": 0}') + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() + + +def test_cancel(): + with patch.object(ResponseMock, "close", return_value=None) as mock_method: + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + itr.cancel() + mock_method.assert_called_once() + + +def test_check_buffer(): + with patch.object( + ResponseMock, + "_parse_responses", + return_value=bytes('[{"content": "hello"}, {', "utf-8"), + ): + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + next(itr) + + +def test_next_html(): + with patch.object( + ResponseMock, "iter_content", return_value=iter("") + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() From beb3e6317faf24ed87680ff73517826d910ca415 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 18:35:46 -0500 Subject: [PATCH 050/126] chore: release 2.4.0 (#316) * chore: release 2.4.0 * chore: delete #318 from log (already released) * Update changelog to include published versions * insert newline in changelog * insert newline in changelog Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- CHANGELOG.md | 21 +++++++++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d47daa..e6177e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.4.0](https://www.github.com/googleapis/python-api-core/compare/v2.3.2...v2.4.0) (2022-01-11) + + +### Features + +* add support for 'error_info' ([#315](https://www.github.com/googleapis/python-api-core/issues/315)) ([cc46aa6](https://www.github.com/googleapis/python-api-core/commit/cc46aa68ec184871330d16a6c767f57a4f0eb633)) +* iterator for processing JSON responses in REST streaming. ([#317](https://www.github.com/googleapis/python-api-core/issues/317)) ([f9f2696](https://www.github.com/googleapis/python-api-core/commit/f9f26969842b456ea372bed941d712b7a9ab7239)) + +## [2.3.2](https://www.github.com/googleapis/python-api-core/compare/v2.3.1...v2.3.2) (2021-12-16) + + +### Bug Fixes + +* address broken wheels in version 2.3.1 + +## [2.3.1](https://www.github.com/googleapis/python-api-core/compare/v2.3.0...v2.3.1) (2021-12-15) + + +### Bug Fixes +* exclude function target from retry deadline exceeded exception message ([#318](https://www.github.com/googleapis/python-api-core/issues/318)) ([34ebdcc](https://www.github.com/googleapis/python-api-core/commit/34ebdcc251d4f3d7d496e8e0b78847645a06650b)) + ## [2.3.0](https://www.github.com/googleapis/python-api-core/compare/v2.2.2...v2.3.0) (2021-11-25) diff --git a/google/api_core/version.py b/google/api_core/version.py index 999199f5..fe11624d 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.3.0" +__version__ = "2.4.0" From adbe63844ed07c1b75e2e6dd97ff865a25750201 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 15:54:21 +0000 Subject: [PATCH 051/126] build: switch to release-please for tagging (#331) --- .github/.OwlBot.lock.yaml | 2 +- .github/release-please.yml | 1 + .github/release-trigger.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .github/release-trigger.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index f33299dd..ff5126c1 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 + digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 diff --git a/.github/release-please.yml b/.github/release-please.yml index 4507ad05..466597e5 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 00000000..d4ca9418 --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true From f267111823545a6c67ef5f10b85cd8c2fab8a612 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 14 Jan 2022 10:41:21 -0500 Subject: [PATCH 052/126] docs: fix typo in library name (#332) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d94f3e88..13d8d3a6 100644 --- a/README.rst +++ b/README.rst @@ -25,4 +25,4 @@ Unsupported Python Versions Python == 2.7, Python == 3.5. The last version of this library compatible with Python 2.7 and 3.5 is -`google-api_core==1.31.1`. +`google-api-core==1.31.1`. From 8169f2bfaaf59042a8b77d2bad9c5a4cc7ed3826 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 12:07:49 -0500 Subject: [PATCH 053/126] chore(python): update release.sh to use keystore (#333) Source-Link: https://github.com/googleapis/synthtool/commit/69fda12e2994f0b595a397e8bb6e3e9f380524eb Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/release.sh | 2 +- .kokoro/release/common.cfg | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c1..eecb84c2 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 0728ce11..a00f93ec 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-api-core python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 586e7649..de4f6f89 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-api-core/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From 74d5ce8a20aa6e270de75fa5ac37c0bb26df7b45 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 17 Jan 2022 12:04:45 -0500 Subject: [PATCH 054/126] chore: move docs check from kokoro to GH actions (#334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: move docs check from kokoro to GH actions * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix typo Co-authored-by: Owl Bot --- .github/sync-repo-settings.yaml | 1 + .github/workflows/docs.yml | 25 ++++++++++ .github/workflows/lint.yml | 30 ++++++++++++ .../{unittest_lint.yml => unittest.yml} | 47 +------------------ 4 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/lint.yml rename .github/workflows/{unittest_lint.yml => unittest.yml} (76%) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 67fcd59f..db2ddd08 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -23,6 +23,7 @@ branchProtectionRules: - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' + - 'docs' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..bd90e5bf --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,25 @@ +name: "Docs" + +on: + pull_request: + branches: + - main +jobs: + run-docs: + name: docs + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs + run: | + nox -s docs docfx + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..c6d94b00 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,30 @@ +name: "Lint" + +on: + pull_request: + branches: + - main +jobs: + run-lint-mypy: + name: lint-mypy + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + run: | + nox -s lint + - name: Run lint_setup_py + run: | + nox -s lint_setup_py + - name: Run mypy + run: | + nox -s mypy diff --git a/.github/workflows/unittest_lint.yml b/.github/workflows/unittest.yml similarity index 76% rename from .github/workflows/unittest_lint.yml rename to .github/workflows/unittest.yml index 302be970..9ab0c395 100644 --- a/.github/workflows/unittest_lint.yml +++ b/.github/workflows/unittest.yml @@ -1,44 +1,11 @@ -name: "Lint / Unit tests / Cover / Mypy" +name: "Unit tests" on: pull_request: branches: - main - jobs: - - run-lint-mypy: - name: lint-mypy - runs-on: ubuntu-latest - - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - - name: Run lint - run: | - nox -s lint - - - name: Run lint_setup_py - run: | - nox -s lint_setup_py - - - name: Run mypy - run: | - nox -s mypy - run-unittests: name: unit${{ matrix.option }}-${{ matrix.python }} runs-on: ubuntu-latest @@ -58,28 +25,22 @@ jobs: python: 3.8 - option: "_wo_grpc" python: 3.9 - steps: - - name: Checkout uses: actions/checkout@v2 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel python -m pip install nox - - name: Run unit tests env: COVERAGE_FILE: .coverage${{ matrix.option }}-${{matrix.python }} run: | nox -s unit${{ matrix.option }}-${{ matrix.python }} - - name: Upload coverage results uses: actions/upload-artifact@v2 with: @@ -91,28 +52,22 @@ jobs: runs-on: ubuntu-latest needs: - run-unittests - steps: - - name: Checkout uses: actions/checkout@v2 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: "3.10" - - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - - name: Download coverage results uses: actions/download-artifact@v2 with: name: coverage-artifacts path: .coverage-results/ - - name: Report coverage results run: | coverage combine .coverage-results/.coverage* From 5e5ad37b8161109d65b0fab43636f7424e570fa3 Mon Sep 17 00:00:00 2001 From: arithmetic1728 <58957152+arithmetic1728@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:55:31 -0800 Subject: [PATCH 055/126] feat: add api_key to client options (#248) * feat: add api_key to client options * update --- google/api_core/client_options.py | 9 +++++++- tests/unit/test_client_options.py | 35 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index be5523df..4a2a84a4 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -66,11 +66,14 @@ class ClientOptions(object): quota_project_id (Optional[str]): A project name that a client's quota belongs to. credentials_file (Optional[str]): A path to a file storing credentials. + ``credentials_file` and ``api_key`` are mutually exclusive. scopes (Optional[Sequence[str]]): OAuth access token override scopes. + api_key (Optional[str]): Google API key. ``credentials_file`` and + ``api_key`` are mutually exclusive. Raises: ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` - are provided. + are provided, or both ``credentials_file`` and ``api_key`` are provided. """ def __init__( @@ -81,17 +84,21 @@ def __init__( quota_project_id=None, credentials_file=None, scopes=None, + api_key=None, ): if client_cert_source and client_encrypted_cert_source: raise ValueError( "client_cert_source and client_encrypted_cert_source are mutually exclusive" ) + if api_key and credentials_file: + raise ValueError("api_key and credentials_file are mutually exclusive") self.api_endpoint = api_endpoint self.client_cert_source = client_cert_source self.client_encrypted_cert_source = client_encrypted_cert_source self.quota_project_id = quota_project_id self.credentials_file = credentials_file self.scopes = scopes + self.api_key = api_key def __repr__(self): return "ClientOptions: " + repr(self.__dict__) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index 38b9ad0a..fbff5457 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -72,6 +72,36 @@ def test_constructor_with_both_cert_sources(): ) +def test_constructor_with_api_key(): + + options = client_options.ClientOptions( + api_endpoint="foo.googleapis.com", + client_cert_source=get_client_cert, + quota_project_id="quote-proj", + api_key="api-key", + scopes=[ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + ], + ) + + assert options.api_endpoint == "foo.googleapis.com" + assert options.client_cert_source() == (b"cert", b"key") + assert options.quota_project_id == "quote-proj" + assert options.api_key == "api-key" + assert options.scopes == [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + ] + + +def test_constructor_with_both_api_key_and_credentials_file(): + with pytest.raises(ValueError): + client_options.ClientOptions( + api_key="api-key", credentials_file="path/to/credentials.json", + ) + + def test_from_dict(): options = client_options.from_dict( { @@ -94,6 +124,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ] + assert options.api_key is None def test_from_dict_bad_argument(): @@ -112,6 +143,6 @@ def test_repr(): assert ( repr(options) - == "ClientOptions: {'api_endpoint': 'foo.googleapis.com', 'client_cert_source': None, 'client_encrypted_cert_source': None}" - or "ClientOptions: {'client_encrypted_cert_source': None, 'client_cert_source': None, 'api_endpoint': 'foo.googleapis.com'}" + == "ClientOptions: {'api_endpoint': 'foo.googleapis.com', 'client_cert_source': None, 'client_encrypted_cert_source': None, 'api_key': None}" + or "ClientOptions: {'client_encrypted_cert_source': None, 'client_cert_source': None, 'api_endpoint': 'foo.googleapis.com', 'api_key': None}" ) From 95e41ff2715cc092d47be1e6244ae835c71e0acd Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 22 Jan 2022 06:04:24 -0500 Subject: [PATCH 056/126] ci(python): run lint / unit tests / docs as GH actions (#336) * ci(python): run lint / unit tests / docs as GH actions Source-Link: https://github.com/googleapis/synthtool/commit/57be0cdb0b94e1669cee0ca38d790de1dfdbcd44 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 * revert changes to unittest gh action * move mypy check into separate workflow * update .sync-repo-settings to reflect changes to gh checks * use python 3.7 for lint check Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 15 ++++++++++++++- .github/sync-repo-settings.yaml | 4 +++- .github/workflows/docs.yml | 27 ++++++++++++++++++++------- .github/workflows/lint.yml | 9 ++------- .github/workflows/mypy.yml | 22 ++++++++++++++++++++++ owlbot.py | 4 ++++ 6 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/mypy.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index eecb84c2..8cb43804 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,16 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index db2ddd08..49c96ed4 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -9,7 +9,8 @@ branchProtectionRules: requiredStatusCheckContexts: - 'cla/google' # No Kokoro: the following are Github actions - - 'lint-mypy' + - 'lint' + - 'mypy' - 'unit-3.6' - 'unit-3.7' - 'unit-3.8' @@ -24,6 +25,7 @@ branchProtectionRules: - 'unit_wo_grpc-3.10' - 'cover' - 'docs' + - 'docfx' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bd90e5bf..f7b8344c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,12 +1,10 @@ -name: "Docs" - on: pull_request: branches: - main +name: docs jobs: - run-docs: - name: docs + docs: runs-on: ubuntu-latest steps: - name: Checkout @@ -14,12 +12,27 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel python -m pip install nox - name: Run docs run: | - nox -s docs docfx - + nox -s docs + docfx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docfx + run: | + nox -s docfx diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c6d94b00..79ef73c2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,12 +1,10 @@ -name: "Lint" - on: pull_request: branches: - main +name: lint jobs: - run-lint-mypy: - name: lint-mypy + lint: runs-on: ubuntu-latest steps: - name: Checkout @@ -25,6 +23,3 @@ jobs: - name: Run lint_setup_py run: | nox -s lint_setup_py - - name: Run mypy - run: | - nox -s mypy diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 00000000..f3212da7 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,22 @@ +on: + pull_request: + branches: + - main +name: mypy +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run mypy + run: | + nox -s mypy diff --git a/owlbot.py b/owlbot.py index 451f7c48..0b7f7d21 100644 --- a/owlbot.py +++ b/owlbot.py @@ -28,6 +28,7 @@ ".flake8", # flake8-import-order, layout ".coveragerc", # layout "CONTRIBUTING.rst", # no systests + ".github/workflows/unittest.yml", # exclude unittest gh action ] templated_files = common.py_library(microgenerator=True, cov_level=100) s.move(templated_files, excludes=excludes) @@ -44,4 +45,7 @@ """, ) +s.replace(".github/workflows/lint.yml", "python-version: \"3.10\"", "python-version: \"3.7\"") + + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From c782f294b50b078f01959627fb82aa4c5efec333 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 2 Feb 2022 13:19:37 -0700 Subject: [PATCH 057/126] fix(deps): remove setuptools from dependencies (#339) * fix(deps): remove setuptools from dependencies Fixes #338 * chore: remove setuptools from constraints --- setup.py | 1 - testing/constraints-3.6.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/setup.py b/setup.py index ddc56004..e400a52f 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ "protobuf >= 3.12.0", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", - "setuptools >= 40.3.0", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 0c2a07b6..bb1eb1e3 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -9,7 +9,6 @@ googleapis-common-protos==1.52.0 protobuf==3.12.0 google-auth==1.25.0 requests==2.18.0 -setuptools==40.3.0 packaging==14.3 grpcio==1.33.2 grpcio-gcp==0.2.2 From 4422ccedd46fb1fd1e99844381681d8aa314c90c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 2 Feb 2022 15:31:14 -0700 Subject: [PATCH 058/126] chore(main): release 2.5.0 (#335) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 17 +++++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6177e0e..6cbede4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.5.0](https://github.com/googleapis/python-api-core/compare/v2.4.0...v2.5.0) (2022-02-02) + + +### Features + +* add api_key to client options ([#248](https://github.com/googleapis/python-api-core/issues/248)) ([5e5ad37](https://github.com/googleapis/python-api-core/commit/5e5ad37b8161109d65b0fab43636f7424e570fa3)) + + +### Bug Fixes + +* **deps:** remove setuptools from dependencies ([#339](https://github.com/googleapis/python-api-core/issues/339)) ([c782f29](https://github.com/googleapis/python-api-core/commit/c782f294b50b078f01959627fb82aa4c5efec333)) + + +### Documentation + +* fix typo in library name ([#332](https://github.com/googleapis/python-api-core/issues/332)) ([f267111](https://github.com/googleapis/python-api-core/commit/f267111823545a6c67ef5f10b85cd8c2fab8a612)) + ## [2.4.0](https://www.github.com/googleapis/python-api-core/compare/v2.3.2...v2.4.0) (2022-01-11) diff --git a/google/api_core/version.py b/google/api_core/version.py index fe11624d..5836d805 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.4.0" +__version__ = "2.5.0" From b558e3d7e2fec3abb5f9ced0e6be232639233458 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 12:26:13 +0000 Subject: [PATCH 059/126] chore(deps): update actions/setup-python action to v3 (#346) Source-Link: https://github.com/googleapis/synthtool/commit/571ee2c3b26182429eddcf115122ee545d7d3787 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 --- .github/.OwlBot.lock.yaml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 8cb43804..d9a55fa4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 + digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f7b8344c..cca4e98b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 79ef73c2..f5331399 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.7" - name: Install nox From 6f614911fa4554838359c54096212ac93692f546 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 2 Mar 2022 17:08:08 +0100 Subject: [PATCH 060/126] chore(deps): update all dependencies to v3 (#347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies to v3 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/workflows/mypy.yml | 4 ++-- .github/workflows/unittest.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index f3212da7..e67dcbc5 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.7" - name: Install nox diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 9ab0c395..1a857a3d 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -27,9 +27,9 @@ jobs: python: 3.9 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} - name: Install nox @@ -54,9 +54,9 @@ jobs: - run-unittests steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install coverage From 021bb7d5bf0a1d8ac58dbf0c738fac309135ba7d Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Wed, 2 Mar 2022 10:28:44 -0800 Subject: [PATCH 061/126] feat: initial support for Extended Operations (#344) Certain APIs with Long-Running Operations deviate from the semantics in https://google.aip.dev/151 and instead define custom operation messages, aka Extended Operations. This change adds a PollingFuture subclass designed to be used with Extended Operations. It is analogous and broadly similar to google.api_core.operation.Operation and subclasses google.api_core.future.polling.PollingFuture. The full description of Extended Operation semantics is beyond the scope of this change. --- google/api_core/extended_operation.py | 206 ++++++++++++++++++++++++++ noxfile.py | 46 +++--- tests/unit/test_extended_operation.py | 182 +++++++++++++++++++++++ 3 files changed, 417 insertions(+), 17 deletions(-) create mode 100644 google/api_core/extended_operation.py create mode 100644 tests/unit/test_extended_operation.py diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py new file mode 100644 index 00000000..209f1213 --- /dev/null +++ b/google/api_core/extended_operation.py @@ -0,0 +1,206 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for extended long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operations using :meth:`ExtendedOperation.result`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + extended_operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + def my_callback(ex_op): + print(f"Operation {ex_op.name} completed") + + extended_operation.add_done_callback(my_callback) + +""" + +import threading + +from google.api_core import exceptions +from google.api_core.future import polling + + +class ExtendedOperation(polling.PollingFuture): + """An ExtendedOperation future for interacting with a Google API Long-Running Operation. + + Args: + extended_operation (proto.Message): The initial operation. + refresh (Callable[[], type(extended_operation)]): A callable that returns + the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel the operation. + retry: Optional(google.api_core.retry.Retry): The retry configuration used + when polling. This can be used to control how often :meth:`done` + is polled. Regardless of the retry's ``deadline``, it will be + overridden by the ``timeout`` argument to :meth:`result`. + + Note: Most long-running API methods use google.api_core.operation.Operation + This class is a wrapper for a subset of methods that use alternative + Long-Running Operation (LRO) semantics. + + Note: there is not a concrete type the extended operation must be. + It MUST have fields that correspond to the following, POSSIBLY WITH DIFFERENT NAMES: + * name: str + * status: Union[str, bool, enum.Enum] + * error_code: int + * error_message: str + """ + + def __init__( + self, extended_operation, refresh, cancel, retry=polling.DEFAULT_RETRY + ): + super().__init__(retry=retry) + self._extended_operation = extended_operation + self._refresh = refresh + self._cancel = cancel + # Note: the extended operation does not give a good way to indicate cancellation. + # We make do with manually tracking cancellation and checking for doneness. + self._cancelled = False + self._completion_lock = threading.Lock() + # Invoke in case the operation came back already complete. + self._handle_refreshed_operation() + + # Note: the following four properties MUST be overridden in a subclass + # if, and only if, the fields in the corresponding extended operation message + # have different names. + # + # E.g. we have an extended operation class that looks like + # + # class MyOperation(proto.Message): + # moniker = proto.Field(proto.STRING, number=1) + # status_msg = proto.Field(proto.STRING, number=2) + # optional http_error_code = proto.Field(proto.INT32, number=3) + # optional http_error_msg = proto.Field(proto.STRING, number=4) + # + # the ExtendedOperation subclass would provide property overrrides that map + # to these (poorly named) fields. + @property + def name(self): + return self._extended_operation.name + + @property + def status(self): + return self._extended_operation.status + + @property + def error_code(self): + return self._extended_operation.error_code + + @property + def error_message(self): + return self._extended_operation.error_message + + def done(self, retry=polling.DEFAULT_RETRY): + self._refresh_and_update(retry) + return self._extended_operation.done + + def cancel(self): + if self.done(): + return False + + self._cancel() + self._cancelled = True + return True + + def cancelled(self): + # TODO(dovs): there is not currently a good way to determine whether the + # operation has been cancelled. + # The best we can do is manually keep track of cancellation + # and check for doneness. + if not self._cancelled: + return False + + self._refresh_and_update() + return self._extended_operation.done + + def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): + if not self._extended_operation.done: + self._extended_operation = self._refresh(retry=retry) + self._handle_refreshed_operation() + + def _handle_refreshed_operation(self): + with self._completion_lock: + if not self._extended_operation.done: + return + + if self.error_code and self.error_message: + exception = exceptions.from_http_status( + status_code=self.error_code, + message=self.error_message, + response=self._extended_operation, + ) + self.set_exception(exception) + elif self.error_code or self.error_message: + exception = exceptions.GoogleAPICallError( + f"Unexpected error {self.error_code}: {self.error_message}" + ) + self.set_exception(exception) + else: + # Extended operations have no payload. + self.set_result(None) + + @classmethod + def make(cls, refresh, cancel, extended_operation, **kwargs): + """ + Return an instantiated ExtendedOperation (or child) that wraps + * a refresh callable + * a cancel callable (can be a no-op) + * an initial result + + .. note:: + It is the caller's responsibility to set up refresh and cancel + with their correct request argument. + The reason for this is that the services that use Extended Operations + have rpcs that look something like the following: + + // service.proto + service MyLongService { + rpc StartLongTask(StartLongTaskRequest) returns (ExtendedOperation) { + option (google.cloud.operation_service) = "CustomOperationService"; + } + } + + service CustomOperationService { + rpc Get(GetOperationRequest) returns (ExtendedOperation) { + option (google.cloud.operation_polling_method) = true; + } + } + + Any info needed for the poll, e.g. a name, path params, etc. + is held in the request, which the initial client method is in a much + better position to make made because the caller made the initial request. + + TL;DR: the caller sets up closures for refresh and cancel that carry + the properly configured requests. + + Args: + refresh (Callable[Optional[Retry]][type(extended_operation)]): A callable that + returns the latest state of the operation. + cancel (Callable[][Any]): A callable that tries to cancel the operation + on a best effort basis. + extended_operation (Any): The initial response of the long running method. + See the docstring for ExtendedOperation.__init__ for requirements on + the type and fields of extended_operation + """ + return cls(extended_operation, refresh, cancel, **kwargs) diff --git a/noxfile.py b/noxfile.py index 003a276d..e02a1280 100644 --- a/noxfile.py +++ b/noxfile.py @@ -92,7 +92,7 @@ def default(session, install_grpc=True): ) # Install all test dependencies, then install this package in-place. - session.install("mock", "pytest", "pytest-cov") + session.install("dataclasses", "mock", "pytest", "pytest-cov", "pytest-xdist") if install_grpc: session.install("-e", ".[grpc]", "-c", constraints_path) else: @@ -102,28 +102,36 @@ def default(session, install_grpc=True): "python", "-m", "py.test", - "--quiet", - "--cov=google.api_core", - "--cov=tests.unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), + *( + # Helpful for running a single test or testfile. + session.posargs + or [ + "--quiet", + "--cov=google.api_core", + "--cov=tests.unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + # Running individual tests with parallelism enabled is usually not helpful. + "-n=auto", + os.path.join("tests", "unit"), + ] + ), ] - pytest_args.extend(session.posargs) # Inject AsyncIO content and proto-plus, if version >= 3.6. # proto-plus is needed for a field mask test in test_protobuf_helpers.py if _greater_or_equal_than_36(session.python): session.install("asyncmock", "pytest-asyncio", "proto-plus") - pytest_args.append("--cov=tests.asyncio") - pytest_args.append(os.path.join("tests", "asyncio")) - session.run(*pytest_args) - else: - # Run py.test against the unit tests. - session.run(*pytest_args) + # Having positional arguments means the user wants to run specific tests. + # Best not to add additional tests to that list. + if not session.posargs: + pytest_args.append("--cov=tests.asyncio") + pytest_args.append(os.path.join("tests", "asyncio")) + + session.run(*pytest_args) @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) @@ -171,7 +179,11 @@ def mypy(session): """Run type-checking.""" session.install(".[grpc, grpcgcp]", "mypy") session.install( - "types-setuptools", "types-requests", "types-protobuf", "types-mock" + "types-setuptools", + "types-requests", + "types-protobuf", + "types-mock", + "types-dataclasses", ) session.run("mypy", "google", "tests") diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py new file mode 100644 index 00000000..7fcebed8 --- /dev/null +++ b/tests/unit/test_extended_operation.py @@ -0,0 +1,182 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import enum +import typing + +import mock +import pytest + +from google.api_core import exceptions +from google.api_core import extended_operation +from google.api_core import retry + +TEST_OPERATION_NAME = "test/extended_operation" + + +@dataclasses.dataclass(frozen=True) +class CustomOperation: + class StatusCode(enum.Enum): + UNKNOWN = 0 + DONE = 1 + PENDING = 2 + + name: str + status: StatusCode + error_code: typing.Optional[int] = None + error_message: typing.Optional[str] = None + + # Note: in generated clients, this property must be generated for each + # extended operation message type. + # The status may be an enum, a string, or a bool. If it's a string or enum, + # its text is compared to the string "DONE". + @property + def done(self): + return self.status.name == "DONE" + + +def make_extended_operation(responses=None): + client_operations_responses = responses or [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ) + ] + + refresh = mock.Mock(spec=["__call__"], side_effect=client_operations_responses) + refresh.responses = client_operations_responses + cancel = mock.Mock(spec=["__call__"]) + extended_operation_future = extended_operation.ExtendedOperation.make( + refresh, cancel, client_operations_responses[0], + ) + + return extended_operation_future, refresh, cancel + + +def test_constructor(): + ex_op, refresh, _ = make_extended_operation() + assert ex_op._extended_operation == refresh.responses[0] + assert ex_op.cancelled() is False + assert ex_op.done() is False + assert ex_op.name == TEST_OPERATION_NAME + assert ex_op.status == CustomOperation.StatusCode.PENDING + assert ex_op.error_code is None + assert ex_op.error_message is None + + +def test_done(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + # Second response indicates that the operation has finished. + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + # Bumper to make sure we stop polling on DONE. + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_message="Gone too far!", + ), + ] + ex_op, refresh, _ = make_extended_operation(responses) + + # Start out not done. + assert not ex_op.done() + assert refresh.call_count == 1 + + # Refresh brings us to the done state. + assert ex_op.done() + assert refresh.call_count == 2 + assert not ex_op.error_message + + # Make sure that subsequent checks are no-ops. + assert ex_op.done() + assert refresh.call_count == 2 + assert not ex_op.error_message + + +def test_cancellation(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + # Second response indicates that the operation was cancelled. + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + ] + ex_op, _, cancel = make_extended_operation(responses) + + assert not ex_op.cancelled() + + assert ex_op.cancel() + assert ex_op.cancelled() + cancel.assert_called_once_with() + + # Cancelling twice should have no effect. + assert not ex_op.cancel() + cancel.assert_called_once_with() + + +def test_done_w_retry(): + # Not sure what's going on here with the coverage, so just ignore it. + test_retry = retry.Retry(predicate=lambda x: True) # pragma: NO COVER + + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + ] + + ex_op, refresh, _ = make_extended_operation(responses) + + ex_op.done(retry=test_retry) + + refresh.assert_called_once_with(retry=test_retry) + + +def test_error(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_code=400, + error_message="Bad request", + ), + ] + + ex_op, _, _ = make_extended_operation(responses) + + # Defaults to CallError when grpc is not installed + with pytest.raises(exceptions.BadRequest): + ex_op.result() + + # Inconsistent result + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_code=2112, + ), + ] + + ex_op, _, _ = make_extended_operation(responses) + + with pytest.raises(exceptions.GoogleAPICallError): + ex_op.result() From a226a7c4ef0bae9a7b2cd03e3ac9aff32eaaff0b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 00:28:36 +0000 Subject: [PATCH 062/126] chore(deps): update actions/checkout action to v3 (#349) Source-Link: https://github.com/googleapis/synthtool/commit/ca879097772aeec2cbb971c3cea8ecc81522b68a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 --- .github/.OwlBot.lock.yaml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index d9a55fa4..480226ac 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 + digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cca4e98b..b46d7305 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f5331399..2fce29f0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: From 04a8b8ba833909602603d73519047a2effeb6a22 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 11:44:12 -0800 Subject: [PATCH 063/126] chore(main): release 2.6.0 (#350) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cbede4c..d8e41ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.6.0](https://github.com/googleapis/python-api-core/compare/v2.5.0...v2.6.0) (2022-03-03) + + +### Features + +* initial support for Extended Operations ([#344](https://github.com/googleapis/python-api-core/issues/344)) ([021bb7d](https://github.com/googleapis/python-api-core/commit/021bb7d5bf0a1d8ac58dbf0c738fac309135ba7d)) + ## [2.5.0](https://github.com/googleapis/python-api-core/compare/v2.4.0...v2.5.0) (2022-02-02) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5836d805..ae34a9fb 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.5.0" +__version__ = "2.6.0" From 7e21e9e34892472a34f9b44175fa761f0e3fd9ed Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 5 Mar 2022 18:37:00 +0200 Subject: [PATCH 064/126] fix: Remove py2 tag from wheel (#343) Co-authored-by: Anthonios Partheniou --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0be0b3ff..8857e9b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [pytype] python_version = 3.6 inputs = From 07747ca9b25601f19ade161d3730eef1da13e2ee Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 11:05:45 -0800 Subject: [PATCH 065/126] chore(main): release 2.6.1 (#352) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e41ac8..b70b67ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) + + +### Bug Fixes + +* Remove py2 tag from wheel ([#343](https://github.com/googleapis/python-api-core/issues/343)) ([7e21e9e](https://github.com/googleapis/python-api-core/commit/7e21e9e34892472a34f9b44175fa761f0e3fd9ed)) + ## [2.6.0](https://github.com/googleapis/python-api-core/compare/v2.5.0...v2.6.0) (2022-03-03) diff --git a/google/api_core/version.py b/google/api_core/version.py index ae34a9fb..410cd066 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.6.0" +__version__ = "2.6.1" From 9abc6f48f23c87b9771dca3c96b4f6af39620a50 Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Mon, 7 Mar 2022 16:09:34 -0800 Subject: [PATCH 066/126] feat: expose extra fields in ExtendedOperation (#351) The operation wrapped by ExtendedOperation may define other fields or methods that the user may wish to use. --- google/api_core/extended_operation.py | 3 +++ tests/unit/test_extended_operation.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py index 209f1213..cabae107 100644 --- a/google/api_core/extended_operation.py +++ b/google/api_core/extended_operation.py @@ -111,6 +111,9 @@ def error_code(self): def error_message(self): return self._extended_operation.error_message + def __getattr__(self, name): + return getattr(self._extended_operation, name) + def done(self, retry=polling.DEFAULT_RETRY): self._refresh_and_update(retry) return self._extended_operation.done diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index 7fcebed8..9521fde8 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -37,6 +37,7 @@ class StatusCode(enum.Enum): status: StatusCode error_code: typing.Optional[int] = None error_message: typing.Optional[str] = None + armor_class: typing.Optional[int] = None # Note: in generated clients, this property must be generated for each # extended operation message type. @@ -180,3 +181,23 @@ def test_error(): with pytest.raises(exceptions.GoogleAPICallError): ex_op.result() + + +def test_pass_through(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.PENDING, + armor_class=10, + ), + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + armor_class=20, + ), + ] + ex_op, _, _ = make_extended_operation(responses) + + assert ex_op.armor_class == 10 + ex_op.result() + assert ex_op.armor_class == 20 From ea41551806f76c6737747f9e4864525b03746682 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 10:15:47 -0800 Subject: [PATCH 067/126] chore(main): release 2.7.0 (#353) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b70b67ed..d7964273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.7.0](https://github.com/googleapis/python-api-core/compare/v2.6.1...v2.7.0) (2022-03-08) + + +### Features + +* expose extra fields in ExtendedOperation ([#351](https://github.com/googleapis/python-api-core/issues/351)) ([9abc6f4](https://github.com/googleapis/python-api-core/commit/9abc6f48f23c87b9771dca3c96b4f6af39620a50)) + ### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) diff --git a/google/api_core/version.py b/google/api_core/version.py index 410cd066..d962613e 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.6.1" +__version__ = "2.7.0" From 0680fb4d3e013fe2de27e0a2ae2cd9896479e596 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Wed, 9 Mar 2022 13:35:53 -0800 Subject: [PATCH 068/126] fix: add more context to error message. (#340) Co-authored-by: Anthonios Partheniou --- google/api_core/path_template.py | 7 +++++-- tests/unit/test_path_template.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index 41fbd4fe..b6284c85 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -270,7 +270,6 @@ def transcode(http_options, **request_kwargs): ] path_args = {field: get_field(request_kwargs, field) for field in path_fields} request["uri"] = expand(uri_template, **path_args) - # Remove fields used in uri path from request leftovers = copy.deepcopy(request_kwargs) for path_field in path_fields: @@ -297,4 +296,8 @@ def transcode(http_options, **request_kwargs): request["method"] = http_option["method"] return request - raise ValueError("Request obj does not match any template") + raise ValueError( + "Request {} does not match any URL path template in available HttpRule's {}".format( + request_kwargs, [opt["uri"] for opt in http_options] + ) + ) diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index 2c5216e0..c12b35fc 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -362,6 +362,7 @@ def test_transcode_with_additional_bindings( [[["get", "/v1/{name}", ""]], {"name": "first/last"}], [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], + [[["post", "/v1/{first_name}", "data"]], {"last_name": "last"}], ], ) def test_transcode_fails(http_options, request_kwargs): @@ -385,5 +386,4 @@ def helper_test_transcode(http_options_list, expected_result_list): } if expected_result_list[2]: expected_result["body"] = expected_result_list[2] - return (http_options, expected_result) From c89f55d10df50c222d05ea046e07e05ef7d17725 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 13:37:46 -0800 Subject: [PATCH 069/126] chore(main): release 2.7.1 (#354) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7964273..0610df01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) + + +### Bug Fixes + +* add more context to error message. ([#340](https://github.com/googleapis/python-api-core/issues/340)) ([0680fb4](https://github.com/googleapis/python-api-core/commit/0680fb4d3e013fe2de27e0a2ae2cd9896479e596)) + ## [2.7.0](https://github.com/googleapis/python-api-core/compare/v2.6.1...v2.7.0) (2022-03-08) diff --git a/google/api_core/version.py b/google/api_core/version.py index d962613e..a97caa56 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.0" +__version__ = "2.7.1" From 112049e79f5a5b0a989d85d438a1bd29485f46f7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Mar 2022 11:35:14 -0400 Subject: [PATCH 070/126] fix: allow grpc without grpcio-status (#355) --- google/api_core/exceptions.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 24b65ee0..38fe6e78 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -24,15 +24,23 @@ import http.client from typing import Dict from typing import Union +import warnings from google.rpc import error_details_pb2 try: import grpc - from grpc_status import rpc_status + + try: + from grpc_status import rpc_status + except ImportError: # pragma: NO COVER + warnings.warn( + "Please install grpcio-status to obtain helpful grpc error messages.", + ImportWarning, + ) + rpc_status = None except ImportError: # pragma: NO COVER grpc = None - rpc_status = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta From 778d27d18ce17e7c0b6dbc1fb56ddf093e278a4e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 06:18:58 -0400 Subject: [PATCH 071/126] chore(python): use black==22.3.0 (#362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): use black==22.3.0 Source-Link: https://github.com/googleapis/synthtool/commit/6fab84af09f2cf89a031fd8671d1def6b2931b11 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe * ci: use black 22.3.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 13 +++- google/api_core/datetime_helpers.py | 4 +- google/api_core/exceptions.py | 6 +- google/api_core/grpc_helpers.py | 4 +- .../abstract_operations_client.py | 68 +++++++++++++++---- .../api_core/operations_v1/transports/base.py | 6 +- google/api_core/path_template.py | 40 +++++------ google/api_core/rest_streaming.py | 3 +- noxfile.py | 7 +- tests/asyncio/gapic/test_method_async.py | 3 +- .../test_operations_rest_client.py | 48 +++++++++---- tests/unit/test_client_options.py | 3 +- tests/unit/test_extended_operation.py | 4 +- tests/unit/test_page_iterator.py | 3 +- tests/unit/test_rest_streaming.py | 8 ++- 16 files changed, 152 insertions(+), 70 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 480226ac..87dd0061 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 + digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe diff --git a/docs/conf.py b/docs/conf.py index 09f0c2b6..9a80171b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -314,7 +314,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (root_doc, "google-api-core", "google-api-core Documentation", [author], 1,) + ( + root_doc, + "google-api-core", + "google-api-core Documentation", + [author], + 1, + ) ] # If true, show URL addresses after external links. @@ -355,7 +361,10 @@ intersphinx_mapping = { "python": ("https://python.readthedocs.org/en/latest/", None), "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), + "google.api_core": ( + "https://googleapis.dev/python/google-api-core/latest/", + None, + ), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py index 78268efc..9470863a 100644 --- a/google/api_core/datetime_helpers.py +++ b/google/api_core/datetime_helpers.py @@ -151,7 +151,7 @@ def from_rfc3339(value): micros = 0 else: scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) + nanos = int(fraction) * (10**scale) micros = nanos // 1000 return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) @@ -245,7 +245,7 @@ def from_rfc3339(cls, stamp): nanos = 0 else: scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) + nanos = int(fraction) * (10**scale) return cls( bare.year, bare.month, diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 38fe6e78..bd2f8562 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -202,7 +202,7 @@ def details(self): Returns: Sequence[Any]: A list of structured objects from error_details.proto - """ + """ return list(self._details) @property @@ -490,7 +490,9 @@ def from_http_response(response): error_info = error_info[0] if error_info else None message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message, + method=response.request.method, + url=response.request.url, + error=error_message, ) exception = from_http_status( diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 594df987..e86ddde5 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -246,7 +246,9 @@ def _create_composite_credentials( # Create the metadata plugin for inserting the authorization header. metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( - credentials, request, default_host=default_host, + credentials, + request, + default_host=default_host, ) # Create a set of grpc.CallCredentials using the metadata plugin. diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py index 631094e7..e02bc199 100644 --- a/google/api_core/operations_v1/abstract_operations_client.py +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -49,7 +49,8 @@ class AbstractOperationsClientMeta(type): _transport_registry["rest"] = OperationsRestTransport def get_transport_class( - cls, label: Optional[str] = None, + cls, + label: Optional[str] = None, ) -> Type[OperationsTransport]: """Returns an appropriate transport class. @@ -165,7 +166,9 @@ def transport(self) -> OperationsTransport: return self._transport @staticmethod - def common_billing_account_path(billing_account: str,) -> str: + def common_billing_account_path( + billing_account: str, + ) -> str: """Returns a fully-qualified billing_account string.""" return "billingAccounts/{billing_account}".format( billing_account=billing_account, @@ -178,9 +181,13 @@ def parse_common_billing_account_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_folder_path(folder: str,) -> str: + def common_folder_path( + folder: str, + ) -> str: """Returns a fully-qualified folder string.""" - return "folders/{folder}".format(folder=folder,) + return "folders/{folder}".format( + folder=folder, + ) @staticmethod def parse_common_folder_path(path: str) -> Dict[str, str]: @@ -189,9 +196,13 @@ def parse_common_folder_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_organization_path(organization: str,) -> str: + def common_organization_path( + organization: str, + ) -> str: """Returns a fully-qualified organization string.""" - return "organizations/{organization}".format(organization=organization,) + return "organizations/{organization}".format( + organization=organization, + ) @staticmethod def parse_common_organization_path(path: str) -> Dict[str, str]: @@ -200,9 +211,13 @@ def parse_common_organization_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_project_path(project: str,) -> str: + def common_project_path( + project: str, + ) -> str: """Returns a fully-qualified project string.""" - return "projects/{project}".format(project=project,) + return "projects/{project}".format( + project=project, + ) @staticmethod def parse_common_project_path(path: str) -> Dict[str, str]: @@ -211,10 +226,14 @@ def parse_common_project_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_location_path(project: str, location: str,) -> str: + def common_location_path( + project: str, + location: str, + ) -> str: """Returns a fully-qualified location string.""" return "projects/{project}/locations/{location}".format( - project=project, location=location, + project=project, + location=location, ) @staticmethod @@ -406,12 +425,20 @@ def list_operations( ) # Send the request. - response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) # This method is paged; wrap the response in a pager, which provides # an `__iter__` convenience method. response = pagers.ListOperationsPager( - method=rpc, request=request, response=response, metadata=metadata, + method=rpc, + request=request, + response=response, + metadata=metadata, ) # Done; return the response. @@ -459,7 +486,12 @@ def get_operation( ) # Send the request. - response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) # Done; return the response. return response @@ -506,7 +538,10 @@ def delete_operation( # Send the request. rpc( - request, retry=retry, timeout=timeout, metadata=metadata, + request, + retry=retry, + timeout=timeout, + metadata=metadata, ) def cancel_operation( @@ -560,5 +595,8 @@ def cancel_operation( # Send the request. rpc( - request, retry=retry, timeout=timeout, metadata=metadata, + request, + retry=retry, + timeout=timeout, + metadata=metadata, ) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py index 460e6460..f9070567 100644 --- a/google/api_core/operations_v1/transports/base.py +++ b/google/api_core/operations_v1/transports/base.py @@ -183,9 +183,9 @@ def _prep_wrapped_messages(self, client_info): def close(self): """Closes resources associated with the transport. - .. warning:: - Only call this method if the transport is NOT shared - with other clients - this may cause errors in other clients! + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! """ raise NotImplementedError() diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index b6284c85..a99a4c81 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -239,26 +239,26 @@ def validate(tmpl, path): def transcode(http_options, **request_kwargs): """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, - https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 - - Args: - http_options (list(dict)): A list of dicts which consist of these keys, - 'method' (str): The http method - 'uri' (str): The path template - 'body' (str): The body field name (optional) - (This is a simplified representation of the proto option `google.api.http`) - - request_kwargs (dict) : A dict representing the request object - - Returns: - dict: The transcoded request with these keys, - 'method' (str) : The http method - 'uri' (str) : The expanded uri - 'body' (dict) : A dict representing the body (optional) - 'query_params' (dict) : A dict mapping query parameter variables and values - - Raises: - ValueError: If the request does not match the given template. + https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 + + Args: + http_options (list(dict)): A list of dicts which consist of these keys, + 'method' (str): The http method + 'uri' (str): The path template + 'body' (str): The body field name (optional) + (This is a simplified representation of the proto option `google.api.http`) + + request_kwargs (dict) : A dict representing the request object + + Returns: + dict: The transcoded request with these keys, + 'method' (str) : The http method + 'uri' (str) : The expanded uri + 'body' (dict) : A dict representing the body (optional) + 'query_params' (dict) : A dict mapping query parameter variables and values + + Raises: + ValueError: If the request does not match the given template. """ for http_option in http_options: request = {} diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py index 69f5b41b..f91381c1 100644 --- a/google/api_core/rest_streaming.py +++ b/google/api_core/rest_streaming.py @@ -48,8 +48,7 @@ def __init__(self, response: requests.Response, response_message_cls): self._escape_next = False def cancel(self): - """Cancel existing streaming operation. - """ + """Cancel existing streaming operation.""" self._response.close() def _process_chunk(self, chunk: str): diff --git a/noxfile.py b/noxfile.py index e02a1280..c9333219 100644 --- a/noxfile.py +++ b/noxfile.py @@ -21,7 +21,7 @@ import nox # pytype: disable=import-error -BLACK_VERSION = "black==19.10b0" +BLACK_VERSION = "black==22.3.0" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] # Black and flake8 clash on the syntax for ignoring flake8's F401 in this file. BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"] @@ -64,7 +64,10 @@ def lint(session): session.install("flake8", "flake8-import-order", BLACK_VERSION) session.install(".") session.run( - "black", "--check", *BLACK_EXCLUDES, *BLACK_PATHS, + "black", + "--check", + *BLACK_EXCLUDES, + *BLACK_PATHS, ) session.run("flake8", "google", "tests") diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index 1410747d..11847da7 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -162,7 +162,8 @@ async def test_wrap_method_with_default_retry_and_timeout_using_sentinel(unused_ ) result = await wrapped_method( - retry=gapic_v1.method_async.DEFAULT, timeout=gapic_v1.method_async.DEFAULT, + retry=gapic_v1.method_async.DEFAULT, + timeout=gapic_v1.method_async.DEFAULT, ) assert result == 42 diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py index dddf6b71..625539e2 100644 --- a/tests/unit/operations_v1/test_operations_rest_client.py +++ b/tests/unit/operations_v1/test_operations_rest_client.py @@ -399,7 +399,9 @@ def test_operations_client_client_options_scopes( client_class, transport_class, transport_name ): # Check the case scopes are provided. - options = client_options.ClientOptions(scopes=["1", "2"],) + options = client_options.ClientOptions( + scopes=["1", "2"], + ) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options) @@ -513,10 +515,12 @@ def test_list_operations_rest_pager(): next_page_token="abc", ), operations_pb2.ListOperationsResponse( - operations=[], next_page_token="def", + operations=[], + next_page_token="def", ), operations_pb2.ListOperationsResponse( - operations=[operations_pb2.Operation()], next_page_token="ghi", + operations=[operations_pb2.Operation()], + next_page_token="ghi", ), operations_pb2.ListOperationsResponse( operations=[operations_pb2.Operation(), operations_pb2.Operation()], @@ -553,7 +557,9 @@ def test_get_operation_rest( with mock.patch.object(Session, "request") as req: # Designate an appropriate value for the returned response. return_value = operations_pb2.Operation( - name="operations/sample1", done=True, error=status_pb2.Status(code=411), + name="operations/sample1", + done=True, + error=status_pb2.Status(code=411), ) # Wrap the value into a proper Response obj @@ -679,7 +685,8 @@ def test_credentials_transport_error(): ) with pytest.raises(ValueError): AbstractOperationsClient( - credentials=ga_credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, ) # It is an error to provide a credentials file and a transport instance. @@ -698,7 +705,8 @@ def test_credentials_transport_error(): ) with pytest.raises(ValueError): AbstractOperationsClient( - client_options={"scopes": ["1", "2"]}, transport=transport, + client_options={"scopes": ["1", "2"]}, + transport=transport, ) @@ -765,7 +773,8 @@ def test_operations_base_transport_with_credentials_file(): Transport.return_value = None load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) transports.OperationsTransport( - credentials_file="credentials.json", quota_project_id="octopus", + credentials_file="credentials.json", + quota_project_id="octopus", ) load_creds.assert_called_once_with( "credentials.json", @@ -792,7 +801,9 @@ def test_operations_auth_adc(): adc.return_value = (ga_credentials.AnonymousCredentials(), None) AbstractOperationsClient() adc.assert_called_once_with( - scopes=None, default_scopes=(), quota_project_id=None, + scopes=None, + default_scopes=(), + quota_project_id=None, ) @@ -849,7 +860,9 @@ def test_parse_common_billing_account_path(): def test_common_folder_path(): folder = "whelk" - expected = "folders/{folder}".format(folder=folder,) + expected = "folders/{folder}".format( + folder=folder, + ) actual = AbstractOperationsClient.common_folder_path(folder) assert expected == actual @@ -867,7 +880,9 @@ def test_parse_common_folder_path(): def test_common_organization_path(): organization = "oyster" - expected = "organizations/{organization}".format(organization=organization,) + expected = "organizations/{organization}".format( + organization=organization, + ) actual = AbstractOperationsClient.common_organization_path(organization) assert expected == actual @@ -885,7 +900,9 @@ def test_parse_common_organization_path(): def test_common_project_path(): project = "cuttlefish" - expected = "projects/{project}".format(project=project,) + expected = "projects/{project}".format( + project=project, + ) actual = AbstractOperationsClient.common_project_path(project) assert expected == actual @@ -905,7 +922,8 @@ def test_common_location_path(): project = "winkle" location = "nautilus" expected = "projects/{project}/locations/{location}".format( - project=project, location=location, + project=project, + location=location, ) actual = AbstractOperationsClient.common_location_path(project, location) assert expected == actual @@ -930,7 +948,8 @@ def test_client_withDEFAULT_CLIENT_INFO(): transports.OperationsTransport, "_prep_wrapped_messages" ) as prep: AbstractOperationsClient( - credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, ) prep.assert_called_once_with(client_info) @@ -939,6 +958,7 @@ def test_client_withDEFAULT_CLIENT_INFO(): ) as prep: transport_class = AbstractOperationsClient.get_transport_class() transport_class( - credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, ) prep.assert_called_once_with(client_info) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index fbff5457..334b8c1f 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -98,7 +98,8 @@ def test_constructor_with_api_key(): def test_constructor_with_both_api_key_and_credentials_file(): with pytest.raises(ValueError): client_options.ClientOptions( - api_key="api-key", credentials_file="path/to/credentials.json", + api_key="api-key", + credentials_file="path/to/credentials.json", ) diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index 9521fde8..ad62b778 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -59,7 +59,9 @@ def make_extended_operation(responses=None): refresh.responses = client_operations_responses cancel = mock.Mock(spec=["__call__"]) extended_operation_future = extended_operation.ExtendedOperation.make( - refresh, cancel, client_operations_responses[0], + refresh, + cancel, + client_operations_responses[0], ) return extended_operation_future, refresh, cancel diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py index a44e998b..cf43aedf 100644 --- a/tests/unit/test_page_iterator.py +++ b/tests/unit/test_page_iterator.py @@ -505,7 +505,8 @@ def api_request(*args, **kw): assert list(next(items_iter)) == [ dict(name=str(i)) for i in range( - ipage * page_size, min((ipage + 1) * page_size, n_results), + ipage * page_size, + min((ipage + 1) * page_size, n_results), ) ] else: diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py index 4be59580..1b34a04f 100644 --- a/tests/unit/test_rest_streaming.py +++ b/tests/unit/test_rest_streaming.py @@ -83,7 +83,10 @@ def __next__(self): return x.decode("utf-8") def __init__( - self, responses: List[proto.Message], response_cls, random_split=False, + self, + responses: List[proto.Message], + response_cls, + random_split=False, ): super().__init__() self._responses = responses @@ -105,7 +108,8 @@ def close(self): def iter_content(self, *args, **kwargs): return self._ResponseItr( - self._parse_responses(self._responses), random_split=self._random_split, + self._parse_responses(self._responses), + random_split=self._random_split, ) From 7186e36b72dafc821629455254999c914cdca43f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 00:12:27 +0000 Subject: [PATCH 072/126] chore(python): update .pre-commit-config.yaml to use black==22.3.0 (#363) Source-Link: https://github.com/googleapis/synthtool/commit/7804ade3daae0d66649bee8df6c55484c6580b8d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d --- .github/.OwlBot.lock.yaml | 3 ++- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 87dd0061..22cc254a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe + digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d +# created: 2022-03-30T23:44:26.560599165Z diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62eb5a77..46d23716 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 From 45b0b52730938cdb77d3e99dc602d3dcf4917403 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 21:59:34 -0400 Subject: [PATCH 073/126] chore(python): Enable size-label bot (#364) Source-Link: https://github.com/googleapis/synthtool/commit/06e82790dd719a165ad32b8a06f8f6ec3e3cae0f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-label.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 .github/auto-label.yaml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 22cc254a..58a0b153 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d -# created: 2022-03-30T23:44:26.560599165Z + digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce +# created: 2022-04-01T01:42:03.609279246Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml new file mode 100644 index 00000000..09c8d735 --- /dev/null +++ b/.github/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From caba5f22b35305531626bafc4e0041619208ca26 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 06:44:25 -0400 Subject: [PATCH 074/126] chore(python): add license header to auto-label.yaml (#365) Source-Link: https://github.com/googleapis/synthtool/commit/eb78c980b52c7c6746d2edb77d9cf7aaa99a2aab Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-label.yaml | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 58a0b153..bc893c97 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce -# created: 2022-04-01T01:42:03.609279246Z + digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 +# created: 2022-04-06T10:30:21.687684602Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml index 09c8d735..41bff0b5 100644 --- a/.github/auto-label.yaml +++ b/.github/auto-label.yaml @@ -1,2 +1,15 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requestsize: enabled: true From 523dbd0b10d37ffcf83fa751f0bad313f162abf1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 13 Apr 2022 12:22:14 -0400 Subject: [PATCH 075/126] fix: remove dependency on pkg_resources (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #360 🦕 --- google/api_core/client_info.py | 8 ++++---- google/api_core/grpc_helpers.py | 9 --------- google/api_core/operations_v1/transports/base.py | 15 +++++---------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index e093ffda..3e4376c9 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -21,8 +21,6 @@ import platform from typing import Union -import pkg_resources - from google.api_core import version as api_core_version _PY_VERSION = platform.python_version() @@ -31,8 +29,10 @@ _GRPC_VERSION: Union[str, None] try: - _GRPC_VERSION = pkg_resources.get_distribution("grpcio").version -except pkg_resources.DistributionNotFound: # pragma: NO COVER + import grpc + + _GRPC_VERSION = grpc.__version__ +except ImportError: # pragma: NO COVER _GRPC_VERSION = None diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index e86ddde5..db16f6fc 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -18,7 +18,6 @@ import functools import grpc -import pkg_resources from google.api_core import exceptions import google.auth @@ -33,14 +32,6 @@ except ImportError: HAS_GRPC_GCP = False -try: - # google.auth.__version__ was added in 1.26.0 - _GOOGLE_AUTH_VERSION = google.auth.__version__ -except AttributeError: - try: # try pkg_resources if it is available - _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version - except pkg_resources.DistributionNotFound: # pragma: NO COVER - _GOOGLE_AUTH_VERSION = None # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py index f9070567..e19bc3e8 100644 --- a/google/api_core/operations_v1/transports/base.py +++ b/google/api_core/operations_v1/transports/base.py @@ -16,26 +16,21 @@ import abc from typing import Awaitable, Callable, Optional, Sequence, Union -import pkg_resources - import google.api_core # type: ignore from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore +from google.api_core import version import google.auth # type: ignore from google.auth import credentials as ga_credentials # type: ignore from google.longrunning import operations_pb2 from google.oauth2 import service_account # type: ignore from google.protobuf import empty_pb2 # type: ignore -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google.api_core.operations_v1", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=version.__version__, +) class OperationsTransport(abc.ABC): From 319c02638682563edc1deb24c5e8ba28d15facab Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:26:39 -0400 Subject: [PATCH 076/126] chore(main): release 2.7.2 (#358) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0610df01..5ac245f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) + + +### Bug Fixes + +* allow grpc without grpcio-status ([#355](https://github.com/googleapis/python-api-core/issues/355)) ([112049e](https://github.com/googleapis/python-api-core/commit/112049e79f5a5b0a989d85d438a1bd29485f46f7)) +* remove dependency on pkg_resources ([#361](https://github.com/googleapis/python-api-core/issues/361)) ([523dbd0](https://github.com/googleapis/python-api-core/commit/523dbd0b10d37ffcf83fa751f0bad313f162abf1)) + ### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) diff --git a/google/api_core/version.py b/google/api_core/version.py index a97caa56..52126b8e 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.1" +__version__ = "2.7.2" From 12f76670f0f0e329be19964a7c1a678c1fe41534 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 Apr 2022 17:55:06 +0200 Subject: [PATCH 077/126] chore(deps): update all dependencies (#367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/workflows/unittest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 1a857a3d..77bfc2cc 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -42,7 +42,7 @@ jobs: run: | nox -s unit${{ matrix.option }}-${{ matrix.python }} - name: Upload coverage results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: coverage-artifacts path: .coverage${{ matrix.option }}-${{ matrix.python }} @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - name: Download coverage results - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: coverage-artifacts path: .coverage-results/ From 44978730b9f7fc1e2dcc6a182f282794807f3eae Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:26:27 +0000 Subject: [PATCH 078/126] chore(python): use ubuntu 22.04 in docs image (#368) Source-Link: https://github.com/googleapis/synthtool/commit/f15cc72fb401b4861cedebb10af74afe428fb1f8 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/docker/docs/Dockerfile | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index bc893c97..64f82d6b 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 -# created: 2022-04-06T10:30:21.687684602Z + digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd +# created: 2022-04-21T15:43:16.246106921Z diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 4e1b1fb8..238b87b9 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ubuntu:20.04 +from ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive @@ -60,8 +60,24 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb +###################### Install python 3.8.11 + +# Download python 3.8.11 +RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz + +# Extract files +RUN tar -xvf Python-3.8.11.tgz + +# Install python 3.8.11 +RUN ./Python-3.8.11/configure --enable-optimizations +RUN make altinstall + +###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.8 /tmp/get-pip.py \ + && python3 /tmp/get-pip.py \ && rm /tmp/get-pip.py +# Test pip +RUN python3 -m pip + CMD ["python3.8"] From 022add16266f9c07f0f88eea13472cc2e0bfc991 Mon Sep 17 00:00:00 2001 From: Dorian Kind Date: Tue, 26 Apr 2022 12:44:33 +0200 Subject: [PATCH 079/126] fix: Avoid AttributeError if grpcio-status is not installed (#370) --- google/api_core/exceptions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index bd2f8562..aaba8791 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -599,7 +599,9 @@ def from_grpc_error(rpc_exc): """ # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. - if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): + if ( + grpc is not None and isinstance(rpc_exc, grpc.Call) + ) or _is_informative_grpc_error(rpc_exc): details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), From 64802e104bf47c32a212e00494d4bf0bbcb2fe33 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 26 Apr 2022 18:26:17 -0400 Subject: [PATCH 080/126] test: use `not` instead of `is False` in assert (#366) * test: use == instead of is when comparing equality * use not instead of == --- tests/unit/test_extended_operation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index ad62b778..c551bfa8 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -70,8 +70,8 @@ def make_extended_operation(responses=None): def test_constructor(): ex_op, refresh, _ = make_extended_operation() assert ex_op._extended_operation == refresh.responses[0] - assert ex_op.cancelled() is False - assert ex_op.done() is False + assert not ex_op.cancelled() + assert not ex_op.done() assert ex_op.name == TEST_OPERATION_NAME assert ex_op.status == CustomOperation.StatusCode.PENDING assert ex_op.error_code is None From 1ce2e5409e5b97a146390fcaaa385fe80087d571 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 29 Apr 2022 13:32:45 -0400 Subject: [PATCH 081/126] test: fix KeyError in test_rest_streaming.py (#373) --- tests/unit/test_rest_streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py index 1b34a04f..a44c83c0 100644 --- a/tests/unit/test_rest_streaming.py +++ b/tests/unit/test_rest_streaming.py @@ -28,6 +28,7 @@ from google.protobuf import timestamp_pb2 +__protobuf__ = proto.module(package=__name__) SEED = int(time.time()) logging.info(f"Starting rest streaming tests with random seed: {SEED}") random.seed(SEED) From 52366f235bc1b94895d245188c7d71f0d4d1246a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 05:24:26 -0400 Subject: [PATCH 082/126] chore(main): release 2.7.3 (#371) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ac245f8..bfebfa51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) + + +### Bug Fixes + +* Avoid AttributeError if grpcio-status is not installed ([#370](https://github.com/googleapis/python-api-core/issues/370)) ([022add1](https://github.com/googleapis/python-api-core/commit/022add16266f9c07f0f88eea13472cc2e0bfc991)) + ### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 52126b8e..138e01b3 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.2" +__version__ = "2.7.3" From 364776ca5256e6202f8eaa0b589c18aaf7ba0680 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 11:44:43 -0400 Subject: [PATCH 083/126] chore: [autoapprove] update readme_gen.py to include autoescape True (#374) Source-Link: https://github.com/googleapis/synthtool/commit/6b4d5a6407d740beb4158b302194a62a4108a8a6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- scripts/readme-gen/readme_gen.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 64f82d6b..b631901e 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd -# created: 2022-04-21T15:43:16.246106921Z + digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 +# created: 2022-05-05T15:17:27.599381182Z diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py index d309d6e9..91b59676 100644 --- a/scripts/readme-gen/readme_gen.py +++ b/scripts/readme-gen/readme_gen.py @@ -28,7 +28,10 @@ jinja_env = jinja2.Environment( trim_blocks=True, loader=jinja2.FileSystemLoader( - os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + os.path.abspath(os.path.join(os.path.dirname(__file__), "templates")) + ), + autoescape=True, +) README_TMPL = jinja_env.get_template('README.tmpl.rst') From cddc802449b222877aae81c0968e5b6ed87ee14d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 23:04:25 +0000 Subject: [PATCH 084/126] chore(python): auto approve template changes (#376) Source-Link: https://github.com/googleapis/synthtool/commit/453a5d9c9a55d1969240a37d36cec626d20a9024 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-approve.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/auto-approve.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b631901e..757c9dca 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 -# created: 2022-05-05T15:17:27.599381182Z + digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 +# created: 2022-05-05T22:08:23.383410683Z diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml new file mode 100644 index 00000000..311ebbb8 --- /dev/null +++ b/.github/auto-approve.yml @@ -0,0 +1,3 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve +processes: + - "OwlBotTemplateChanges" From c97c4980125a86f384cdf12720df7bb1a2adf9d2 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Wed, 18 May 2022 12:03:14 -0700 Subject: [PATCH 085/126] feat: adds support for audience in client_options (#379) feat: adds support for audience in client_options. --- google/api_core/client_options.py | 7 +++++++ tests/unit/test_client_options.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index 4a2a84a4..ee9f28a9 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -70,6 +70,11 @@ class ClientOptions(object): scopes (Optional[Sequence[str]]): OAuth access token override scopes. api_key (Optional[str]): Google API key. ``credentials_file`` and ``api_key`` are mutually exclusive. + api_audience (Optional[str]): The intended audience for the API calls + to the service that will be set when using certain 3rd party + authentication flows. Audience is typically a resource identifier. + If not set, the service endpoint value will be used as a default. + An example of a valid ``api_audience`` is: "https://language.googleapis.com". Raises: ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` @@ -85,6 +90,7 @@ def __init__( credentials_file=None, scopes=None, api_key=None, + api_audience=None, ): if client_cert_source and client_encrypted_cert_source: raise ValueError( @@ -99,6 +105,7 @@ def __init__( self.credentials_file = credentials_file self.scopes = scopes self.api_key = api_key + self.api_audience = api_audience def __repr__(self): return "ClientOptions: " + repr(self.__dict__) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index 334b8c1f..d56a1b3a 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -36,6 +36,7 @@ def test_constructor(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ], + api_audience="foo2.googleapis.com", ) assert options.api_endpoint == "foo.googleapis.com" @@ -46,6 +47,7 @@ def test_constructor(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ] + assert options.api_audience == "foo2.googleapis.com" def test_constructor_with_encrypted_cert_source(): @@ -114,6 +116,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ], + "api_audience": "foo2.googleapis.com", } ) @@ -126,6 +129,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform.read-only", ] assert options.api_key is None + assert options.api_audience == "foo2.googleapis.com" def test_from_dict_bad_argument(): From eed844f211ad8c6ab2c4cb0d6f089e1f11999f71 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 18 May 2022 14:45:58 -0700 Subject: [PATCH 086/126] chore(main): release 2.8.0 (#381) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfebfa51..7ed6c616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.8.0](https://github.com/googleapis/python-api-core/compare/v2.7.3...v2.8.0) (2022-05-18) + + +### Features + +* adds support for audience in client_options ([#379](https://github.com/googleapis/python-api-core/issues/379)) ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) +* adds support for audience in client_options. ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) + ### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) diff --git a/google/api_core/version.py b/google/api_core/version.py index 138e01b3..0a9aecb3 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.3" +__version__ = "2.8.0" From d84d66c2a4107f5f9a20c53e870a27fb1250ea3d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 26 May 2022 16:26:18 -0400 Subject: [PATCH 087/126] fix(deps): require protobuf>= 3.15.0, <4.0.0dev (#385) Also adds upper limits for extras. fix(deps): require googleapis-common-protos >= 1.56.2 --- setup.py | 8 ++++---- testing/constraints-3.6.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index e400a52f..70b26c92 100644 --- a/setup.py +++ b/setup.py @@ -29,15 +29,15 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "googleapis-common-protos >= 1.52.0, < 2.0dev", - "protobuf >= 3.12.0", + "googleapis-common-protos >= 1.56.2, < 2.0dev", + "protobuf >= 3.15.0, <4.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], - "grpcgcp": "grpcio-gcp >= 0.2.2", - "grpcio-gcp": "grpcio-gcp >= 0.2.2", + "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", + "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index bb1eb1e3..014e2b2a 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,8 +5,8 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -googleapis-common-protos==1.52.0 -protobuf==3.12.0 +googleapis-common-protos==1.56.2 +protobuf==3.15.0 google-auth==1.25.0 requests==2.18.0 packaging==14.3 From 9be727df7fb19d7264c81eab9cc8b6de63f6117e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 16:29:43 -0400 Subject: [PATCH 088/126] chore(main): release 2.8.1 (#386) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed6c616..c84458ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) + + +### Bug Fixes + +* **deps:** require googleapis-common-protos >= 1.56.2 ([d84d66c](https://github.com/googleapis/python-api-core/commit/d84d66c2a4107f5f9a20c53e870a27fb1250ea3d)) +* **deps:** require protobuf>= 3.15.0, <4.0.0dev ([#385](https://github.com/googleapis/python-api-core/issues/385)) ([d84d66c](https://github.com/googleapis/python-api-core/commit/d84d66c2a4107f5f9a20c53e870a27fb1250ea3d)) + ## [2.8.0](https://github.com/googleapis/python-api-core/compare/v2.7.3...v2.8.0) (2022-05-18) diff --git a/google/api_core/version.py b/google/api_core/version.py index 0a9aecb3..7e8f51a6 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.0" +__version__ = "2.8.1" From 0c4668df73ac3989170ea5677f906f147a9560d0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 27 May 2022 12:27:54 -0400 Subject: [PATCH 089/126] chore: allow releases from older version branches (#388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: allow releases from older version branches * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/release-please.yml | 9 +++++++++ owlbot.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/release-please.yml b/.github/release-please.yml index 466597e5..29601ad4 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1,2 +1,11 @@ releaseType: python handleGHRelease: true +# NOTE: this section is generated by synthtool.languages.python +# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py +branches: +- branch: v1 + handleGHRelease: true + releaseType: python +- branch: v0 + handleGHRelease: true + releaseType: python diff --git a/owlbot.py b/owlbot.py index 0b7f7d21..9d58f4a1 100644 --- a/owlbot.py +++ b/owlbot.py @@ -16,6 +16,7 @@ import synthtool as s from synthtool import gcp +from synthtool.languages import python common = gcp.CommonTemplates() @@ -47,5 +48,6 @@ s.replace(".github/workflows/lint.yml", "python-version: \"3.10\"", "python-version: \"3.7\"") +python.configure_previous_major_version_branches() s.shell.run(["nox", "-s", "blacken"], hide_output=False) From ac266e935bc4e7c6dff250384407e7a60d8dba90 Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Wed, 1 Jun 2022 21:31:15 -0400 Subject: [PATCH 090/126] docs: fix changelog header to consistent size (#394) --- CHANGELOG.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c84458ba..227226de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ [1]: https://pypi.org/project/google-api-core/#history -### [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) +## [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) ### Bug Fixes @@ -20,14 +20,14 @@ * adds support for audience in client_options ([#379](https://github.com/googleapis/python-api-core/issues/379)) ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) * adds support for audience in client_options. ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) -### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) +## [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) ### Bug Fixes * Avoid AttributeError if grpcio-status is not installed ([#370](https://github.com/googleapis/python-api-core/issues/370)) ([022add1](https://github.com/googleapis/python-api-core/commit/022add16266f9c07f0f88eea13472cc2e0bfc991)) -### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) +## [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) ### Bug Fixes @@ -35,7 +35,7 @@ * allow grpc without grpcio-status ([#355](https://github.com/googleapis/python-api-core/issues/355)) ([112049e](https://github.com/googleapis/python-api-core/commit/112049e79f5a5b0a989d85d438a1bd29485f46f7)) * remove dependency on pkg_resources ([#361](https://github.com/googleapis/python-api-core/issues/361)) ([523dbd0](https://github.com/googleapis/python-api-core/commit/523dbd0b10d37ffcf83fa751f0bad313f162abf1)) -### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) +## [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) ### Bug Fixes @@ -49,7 +49,7 @@ * expose extra fields in ExtendedOperation ([#351](https://github.com/googleapis/python-api-core/issues/351)) ([9abc6f4](https://github.com/googleapis/python-api-core/commit/9abc6f48f23c87b9771dca3c96b4f6af39620a50)) -### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) +## [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) ### Bug Fixes @@ -113,14 +113,14 @@ * handle bare 'grpc.Call' in 'from_grpc_error' ([#298](https://www.github.com/googleapis/python-api-core/issues/298)) ([060b339](https://www.github.com/googleapis/python-api-core/commit/060b339e3af296dd1772bfc1b4a0d2b4264cae1f)) -### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) +## [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) ### Bug Fixes * make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) -### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) +## [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) ### Bug Fixes @@ -134,7 +134,7 @@ * add 'GoogleAPICallError.error_details' property ([#286](https://www.github.com/googleapis/python-api-core/issues/286)) ([ef6f0fc](https://www.github.com/googleapis/python-api-core/commit/ef6f0fcfdfe771172056e35e3c990998b3b00416)) -### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) +## [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) ### Bug Fixes @@ -150,7 +150,7 @@ * Add helper function to format query_params for rest transport. ([#275](https://www.github.com/googleapis/python-api-core/issues/275)) ([1c5eb4d](https://www.github.com/googleapis/python-api-core/commit/1c5eb4df93d78e791082d9282330ebf0faacd222)) * add support for Python 3.10 ([#284](https://www.github.com/googleapis/python-api-core/issues/284)) ([a422a5d](https://www.github.com/googleapis/python-api-core/commit/a422a5d72cb6f363d57e7a4effe421ba8e049cde)) -### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) +## [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) ### Bug Fixes @@ -179,7 +179,7 @@ * strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) -### [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) +## [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) ### Bug Fixes @@ -242,7 +242,7 @@ * Add support for `rest/` token in `x-goog-api-client` header ([#189](https://www.github.com/googleapis/python-api-core/issues/189)) ([15aca6b](https://www.github.com/googleapis/python-api-core/commit/15aca6b288b2ec5ce0251e442e1dfa7f52e1b124)) * retry google.auth TransportError and requests ConnectionError ([#178](https://www.github.com/googleapis/python-api-core/issues/178)) ([6ae04a8](https://www.github.com/googleapis/python-api-core/commit/6ae04a8d134fffe13f06081e15f9723c1b2ea334)) -### [1.26.3](https://www.github.com/googleapis/python-api-core/compare/v1.26.2...v1.26.3) (2021-03-25) +## [1.26.3](https://www.github.com/googleapis/python-api-core/compare/v1.26.2...v1.26.3) (2021-03-25) ### Bug Fixes @@ -254,14 +254,14 @@ * update python contributing guide ([#147](https://www.github.com/googleapis/python-api-core/issues/147)) ([1d76b57](https://www.github.com/googleapis/python-api-core/commit/1d76b57d1f218f7885f85dc7c052bad1ad3857ac)) -### [1.26.2](https://www.github.com/googleapis/python-api-core/compare/v1.26.1...v1.26.2) (2021-03-23) +## [1.26.2](https://www.github.com/googleapis/python-api-core/compare/v1.26.1...v1.26.2) (2021-03-23) ### Bug Fixes * save empty IAM policy bindings ([#155](https://www.github.com/googleapis/python-api-core/issues/155)) ([536c2ca](https://www.github.com/googleapis/python-api-core/commit/536c2cad814b8fa8cd346a3d7bd5f6b9889c4a6f)) -### [1.26.1](https://www.github.com/googleapis/python-api-core/compare/v1.26.0...v1.26.1) (2021-02-12) +## [1.26.1](https://www.github.com/googleapis/python-api-core/compare/v1.26.0...v1.26.1) (2021-02-12) ### Bug Fixes @@ -275,7 +275,7 @@ * allow default_host and default_scopes to be passed to create_channel ([#134](https://www.github.com/googleapis/python-api-core/issues/134)) ([94c76e0](https://www.github.com/googleapis/python-api-core/commit/94c76e0873e5b2f42331d5b1ad286c1e63b61395)) -### [1.25.1](https://www.github.com/googleapis/python-api-core/compare/v1.25.0...v1.25.1) (2021-01-25) +## [1.25.1](https://www.github.com/googleapis/python-api-core/compare/v1.25.0...v1.25.1) (2021-01-25) ### Bug Fixes @@ -299,7 +299,7 @@ * **python:** document adding Python 3.9 support, dropping 3.5 support ([#120](https://www.github.com/googleapis/python-api-core/issues/120)) ([b51b7f5](https://www.github.com/googleapis/python-api-core/commit/b51b7f587042fe9340371c1b5c8e9adf8001c43a)), closes [#787](https://www.github.com/googleapis/python-api-core/issues/787) -### [1.24.1](https://www.github.com/googleapis/python-api-core/compare/v1.24.0...v1.24.1) (2020-12-16) +## [1.24.1](https://www.github.com/googleapis/python-api-core/compare/v1.24.0...v1.24.1) (2020-12-16) ### Bug Fixes @@ -332,28 +332,28 @@ * harden install to use full paths, and windows separators on windows ([#88](https://www.github.com/googleapis/python-api-core/issues/88)) ([db8e636](https://www.github.com/googleapis/python-api-core/commit/db8e636f545a8872f959e3f403cfec30ffed6c34)) * update out-of-date comment in exceptions.py ([#93](https://www.github.com/googleapis/python-api-core/issues/93)) ([70ebe42](https://www.github.com/googleapis/python-api-core/commit/70ebe42601b3d088b3421233ef7d8245229b7265)) -### [1.22.4](https://www.github.com/googleapis/python-api-core/compare/v1.22.3...v1.22.4) (2020-10-05) +## [1.22.4](https://www.github.com/googleapis/python-api-core/compare/v1.22.3...v1.22.4) (2020-10-05) ### Bug Fixes * use version.py instead of pkg_resources.get_distribution ([#80](https://www.github.com/googleapis/python-api-core/issues/80)) ([d480d97](https://www.github.com/googleapis/python-api-core/commit/d480d97e41cd6705325b3b649360553a83c23f47)) -### [1.22.3](https://www.github.com/googleapis/python-api-core/compare/v1.22.2...v1.22.3) (2020-10-02) +## [1.22.3](https://www.github.com/googleapis/python-api-core/compare/v1.22.2...v1.22.3) (2020-10-02) ### Bug Fixes * **deps:** require six >= 1.13.0 ([#78](https://www.github.com/googleapis/python-api-core/issues/78)) ([a7a8b98](https://www.github.com/googleapis/python-api-core/commit/a7a8b98602a3eb277fdc607ac69f3bcb147f3351)), closes [/github.com/benjaminp/six/blob/c0be8815d13df45b6ae471c4c436cce8c192245d/CHANGES#L30-L31](https://www.github.com/googleapis//github.com/benjaminp/six/blob/c0be8815d13df45b6ae471c4c436cce8c192245d/CHANGES/issues/L30-L31) -### [1.22.2](https://www.github.com/googleapis/python-api-core/compare/v1.22.1...v1.22.2) (2020-09-03) +## [1.22.2](https://www.github.com/googleapis/python-api-core/compare/v1.22.1...v1.22.2) (2020-09-03) ### Bug Fixes * only add quota project id if supported ([#75](https://www.github.com/googleapis/python-api-core/issues/75)) ([8f8ee78](https://www.github.com/googleapis/python-api-core/commit/8f8ee7879e4f834f3c676e535ffc41b5b9b2de62)) -### [1.22.1](https://www.github.com/googleapis/python-api-core/compare/v1.22.0...v1.22.1) (2020-08-12) +## [1.22.1](https://www.github.com/googleapis/python-api-core/compare/v1.22.0...v1.22.1) (2020-08-12) ### Documentation @@ -384,7 +384,7 @@ * allow credentials files to be passed for channel creation ([#50](https://www.github.com/googleapis/python-api-core/issues/50)) ([ded92d0](https://www.github.com/googleapis/python-api-core/commit/ded92d0acdcde4295d0e5df05fda0d83783a3991)) -### [1.20.1](https://www.github.com/googleapis/python-api-core/compare/v1.20.0...v1.20.1) (2020-06-16) +## [1.20.1](https://www.github.com/googleapis/python-api-core/compare/v1.20.0...v1.20.1) (2020-06-16) ### Bug Fixes @@ -398,7 +398,7 @@ * allow disabling response stream pre-fetch ([#30](https://www.github.com/googleapis/python-api-core/issues/30)) ([74e0b0f](https://www.github.com/googleapis/python-api-core/commit/74e0b0f8387207933c120af15b2bb5d175dd8f84)), closes [#25](https://www.github.com/googleapis/python-api-core/issues/25) -### [1.19.1](https://www.github.com/googleapis/python-api-core/compare/v1.19.0...v1.19.1) (2020-06-06) +## [1.19.1](https://www.github.com/googleapis/python-api-core/compare/v1.19.0...v1.19.1) (2020-06-06) ### Bug Fixes From 0eb727f92314db3c4383754514f75a49ba02e27b Mon Sep 17 00:00:00 2001 From: Mariatta Wijaya Date: Wed, 1 Jun 2022 18:36:11 -0700 Subject: [PATCH 091/126] docs: Fix typo in the BackgroundConsumer docstring (#395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `UNAVAILABLE` instead of `UNVAILABLE` Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-api-core/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- google/api_core/bidi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 4b4963f7..57f5f9dd 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -364,7 +364,7 @@ class ResumableBidiRpc(BidiRpc): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -589,7 +589,7 @@ class BackgroundConsumer(object): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') From e92045b7a34b8d0a6374d6b1f67c1c6095ad33c6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 3 Jun 2022 14:36:25 -0400 Subject: [PATCH 092/126] chore: test minimum dependencies in python 3.7 (#397) * chore: test minimum dependencies in python 3.7 * remove duplicate entry --- testing/constraints-3.7.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index e69de29b..36643dbc 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -0,0 +1,15 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 +googleapis-common-protos==1.56.2 +protobuf==3.15.0 +google-auth==1.25.0 +requests==2.18.0 +packaging==14.3 +grpcio==1.33.2 +grpcio-gcp==0.2.2 +grpcio-status==1.33.2 From 5da6733a475c436efc11b14889af73b3a0e20379 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 13 Jun 2022 14:55:32 -0400 Subject: [PATCH 093/126] fix: drop support for grpc-gcp (#401) --- .github/sync-repo-settings.yaml | 5 -- .github/workflows/unittest.yml | 2 +- google/api_core/grpc_helpers.py | 17 +---- google/api_core/grpc_helpers_async.py | 3 - noxfile.py | 19 +----- setup.py | 2 - testing/constraints-3.6.txt | 2 - testing/constraints-3.7.txt | 1 - tests/asyncio/test_grpc_helpers_async.py | 5 +- tests/unit/test_grpc_helpers.py | 80 ++++-------------------- 10 files changed, 20 insertions(+), 116 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 49c96ed4..b06a42ac 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -16,11 +16,6 @@ branchProtectionRules: - 'unit-3.8' - 'unit-3.9' - 'unit-3.10' - - 'unit_grpc_gcp-3.6' - - 'unit_grpc_gcp-3.7' - - 'unit_grpc_gcp-3.8' - - 'unit_grpc_gcp-3.9' - - 'unit_grpc_gcp-3.10' - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 77bfc2cc..0c2e37f6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - option: ["", "_grpc_gcp", "_wo_grpc"] + option: ["", "_wo_grpc"] python: - "3.6" - "3.7" diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index db16f6fc..47d27726 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -25,13 +25,6 @@ import google.auth.transport.grpc import google.auth.transport.requests -try: - import grpc_gcp - - HAS_GRPC_GCP = True -except ImportError: - HAS_GRPC_GCP = False - # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) @@ -282,8 +275,7 @@ def create_channel( default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". - kwargs: Additional key-word args passed to - :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + kwargs: Additional key-word args passed to :func:`grpc.secure_channel`. Returns: grpc.Channel: The created channel. @@ -302,12 +294,7 @@ def create_channel( default_host=default_host, ) - if HAS_GRPC_GCP: - # If grpc_gcp module is available use grpc_gcp.secure_channel, - # otherwise, use grpc.secure_channel to create grpc channel. - return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) - else: - return grpc.secure_channel(target, composite_credentials, **kwargs) + return grpc.secure_channel(target, composite_credentials, **kwargs) _MethodCall = collections.namedtuple( diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 452e7871..63a3ad18 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -27,9 +27,6 @@ from google.api_core import exceptions, grpc_helpers -# TODO(lidiz) Support gRPC GCP wrapper -HAS_GRPC_GCP = False - # NOTE(lidiz) Alternatively, we can hack "__getattribute__" to perform # automatic patching for us. But that means the overhead of creating an # extra Python function spreads to every single send and receive. diff --git a/noxfile.py b/noxfile.py index c9333219..9b71610c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,7 +32,6 @@ # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", - "unit_grpc_gcp", "unit_wo_grpc", "cover", "pytype", @@ -143,18 +142,6 @@ def unit(session): default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) -def unit_grpc_gcp(session): - """Run the unit test suite with grpcio-gcp installed.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - # Install grpcio-gcp - session.install("-e", ".[grpcgcp]", "-c", constraints_path) - - default(session) - - @nox.session(python=["3.6", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" @@ -173,14 +160,14 @@ def lint_setup_py(session): @nox.session(python="3.6") def pytype(session): """Run type-checking.""" - session.install(".[grpc, grpcgcp]", "pytype >= 2019.3.21") + session.install(".[grpc]", "pytype >= 2019.3.21") session.run("pytype") @nox.session(python=DEFAULT_PYTHON_VERSION) def mypy(session): """Run type-checking.""" - session.install(".[grpc, grpcgcp]", "mypy") + session.install(".[grpc]", "mypy") session.install( "types-setuptools", "types-requests", @@ -207,7 +194,7 @@ def cover(session): def docs(session): """Build the docs for this library.""" - session.install("-e", ".[grpc, grpcgcp]") + session.install("-e", ".[grpc]") session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) diff --git a/setup.py b/setup.py index 70b26c92..49bb1034 100644 --- a/setup.py +++ b/setup.py @@ -36,8 +36,6 @@ ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], - "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", - "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 014e2b2a..c3e6ad74 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,6 +11,4 @@ google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 -grpcio-gcp==0.2.2 -grpcio-gcp==0.2.2 grpcio-status==1.33.2 diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 36643dbc..c3e6ad74 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -11,5 +11,4 @@ google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 -grpcio-gcp==0.2.2 grpcio-status==1.33.2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index 3681a40d..2d0a1bcd 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -565,11 +565,8 @@ def test_create_channel_with_credentials_file_and_default_scopes( grpc_secure_channel.assert_called_once_with(target, composite_creds) -@pytest.mark.skipif( - grpc_helpers_async.HAS_GRPC_GCP, reason="grpc_gcp module not available" -) @mock.patch("grpc.aio.secure_channel") -def test_create_channel_without_grpc_gcp(grpc_secure_channel): +def test_create_channel(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index ca969e48..649072f0 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -365,10 +365,7 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c default.assert_called_once_with(scopes=None, default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True) @@ -400,10 +397,7 @@ def test_create_channel_implicit_with_default_host( mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host ) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -426,10 +420,7 @@ def test_create_channel_implicit_with_ssl_creds( composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -451,10 +442,7 @@ def test_create_channel_implicit_with_scopes( default.assert_called_once_with(scopes=["one", "two"], default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -476,10 +464,7 @@ def test_create_channel_implicit_with_default_scopes( default.assert_called_once_with(scopes=None, default_scopes=["three", "four"]) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) def test_create_channel_explicit_with_duplicate_credentials(): @@ -507,10 +492,7 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -530,10 +512,7 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -557,10 +536,7 @@ def test_create_channel_explicit_default_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -582,10 +558,7 @@ def test_create_channel_explicit_with_quota_project( credentials.with_quota_project.assert_called_once_with("project-foo") assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -610,10 +583,7 @@ def test_create_channel_with_credentials_file( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -641,10 +611,7 @@ def test_create_channel_with_credentials_file_and_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -672,32 +639,11 @@ def test_create_channel_with_credentials_file_and_default_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) -@pytest.mark.skipif( - not grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available" -) -@mock.patch("grpc_gcp.secure_channel") -def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): - target = "example.com:443" - scopes = ["test_scope"] - - credentials = mock.create_autospec(google.auth.credentials.Scoped, instance=True) - credentials.requires_scopes = True - - grpc_helpers.create_channel(target, credentials=credentials, scopes=scopes) - grpc_gcp_secure_channel.assert_called() - - credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) - - -@pytest.mark.skipif(grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available") @mock.patch("grpc.secure_channel") -def test_create_channel_without_grpc_gcp(grpc_secure_channel): +def test_create_channel(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From 8f73d2ee2d3af2201f877aa7e2f7361147759dc7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 13 Jun 2022 18:10:35 -0400 Subject: [PATCH 094/126] fix(deps): allow protobuf < 5.0.0 (#400) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 49bb1034..3c40548d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.15.0, <4.0.0dev", + "protobuf >= 3.15.0, <5.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] From 5b5e77563229687c901d77b5fdecc18168b535e6 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:39:20 -0400 Subject: [PATCH 095/126] chore(main): release 2.8.2 (#396) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 227226de..16e22ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.8.2](https://github.com/googleapis/python-api-core/compare/v2.8.1...v2.8.2) (2022-06-13) + + +### Bug Fixes + +* **deps:** allow protobuf < 5.0.0 ([#400](https://github.com/googleapis/python-api-core/issues/400)) ([8f73d2e](https://github.com/googleapis/python-api-core/commit/8f73d2ee2d3af2201f877aa7e2f7361147759dc7)) +* drop support for grpc-gcp ([#401](https://github.com/googleapis/python-api-core/issues/401)) ([5da6733](https://github.com/googleapis/python-api-core/commit/5da6733a475c436efc11b14889af73b3a0e20379)) + + +### Documentation + +* fix changelog header to consistent size ([#394](https://github.com/googleapis/python-api-core/issues/394)) ([ac266e9](https://github.com/googleapis/python-api-core/commit/ac266e935bc4e7c6dff250384407e7a60d8dba90)) +* Fix typo in the BackgroundConsumer docstring ([#395](https://github.com/googleapis/python-api-core/issues/395)) ([0eb727f](https://github.com/googleapis/python-api-core/commit/0eb727f92314db3c4383754514f75a49ba02e27b)) + ## [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index 7e8f51a6..839a77a1 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.1" +__version__ = "2.8.2" From 7ddb8c00e6be7ab6905a9a802ad1c3063fbfa46c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:44:37 -0400 Subject: [PATCH 096/126] fix: require python 3.7+ (#410) * chore(python): drop python 3.6 Source-Link: https://github.com/googleapis/synthtool/commit/4f89b13af10d086458f9b379e56a614f9d6dab7b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c * require python 3.7+ in setup.py * remove python 3.6 sample configs * exclude templated README * remove python 3.6 from noxfile * remove python 3.6 from remaining files * remove testing/constraints-3.6.txt Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 4 +- .github/sync-repo-settings.yaml | 2 - .github/workflows/unittest.yml | 1 - .kokoro/continuous/prerelease-deps.cfg | 7 ++++ .kokoro/presubmit/prerelease-deps.cfg | 7 ++++ .kokoro/samples/python3.6/common.cfg | 40 ------------------- .kokoro/samples/python3.6/continuous.cfg | 7 ---- .kokoro/samples/python3.6/periodic-head.cfg | 11 ----- .kokoro/samples/python3.6/periodic.cfg | 6 --- .kokoro/samples/python3.6/presubmit.cfg | 6 --- .kokoro/test-samples-impl.sh | 4 +- CONTRIBUTING.rst | 16 +------- README.rst | 7 +++- google/api_core/grpc_helpers_async.py | 2 +- mypy.ini | 2 +- noxfile.py | 19 +++++---- owlbot.py | 1 + .../templates/install_deps.tmpl.rst | 2 +- setup.cfg | 2 +- setup.py | 3 +- testing/constraints-3.6.txt | 14 ------- 21 files changed, 39 insertions(+), 124 deletions(-) create mode 100644 .kokoro/continuous/prerelease-deps.cfg create mode 100644 .kokoro/presubmit/prerelease-deps.cfg delete mode 100644 .kokoro/samples/python3.6/common.cfg delete mode 100644 .kokoro/samples/python3.6/continuous.cfg delete mode 100644 .kokoro/samples/python3.6/periodic-head.cfg delete mode 100644 .kokoro/samples/python3.6/periodic.cfg delete mode 100644 .kokoro/samples/python3.6/presubmit.cfg delete mode 100644 testing/constraints-3.6.txt diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 757c9dca..1ce60852 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 -# created: 2022-05-05T22:08:23.383410683Z + digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c +# created: 2022-07-05T18:31:20.838186805Z diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index b06a42ac..e2d70f90 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -11,12 +11,10 @@ branchProtectionRules: # No Kokoro: the following are Github actions - 'lint' - 'mypy' - - 'unit-3.6' - 'unit-3.7' - 'unit-3.8' - 'unit-3.9' - 'unit-3.10' - - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' - 'docs' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 0c2e37f6..4c1e8c4e 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -13,7 +13,6 @@ jobs: matrix: option: ["", "_wo_grpc"] python: - - "3.6" - "3.7" - "3.8" - "3.9" diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index 3bb6b3a6..00000000 --- a/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-api-core/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af14..00000000 --- a/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index a18c0cfc..00000000 --- a/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-api-core/.kokoro/test-samples-against-head.sh" -} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e59..00000000 --- a/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d975..00000000 --- a/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9c..2c6500ca 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6b375f03..dddeddb9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -197,13 +197,11 @@ Supported Python Versions We support: -- `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ -.. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ @@ -215,18 +213,6 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-api-core/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.6. -Reasons for this include: - -- Encouraging use of newest versions of Python 3 -- Taking the lead of `prominent`_ open-source `projects`_ -- `Unicode literal support`_ which allows for a cleaner codebase that - works in both Python 2 and Python 3 - -.. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django -.. _projects: http://flask.pocoo.org/docs/0.10/python3/ -.. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ - ********** Versioning ********** diff --git a/README.rst b/README.rst index 13d8d3a6..58ae26cb 100644 --- a/README.rst +++ b/README.rst @@ -16,13 +16,16 @@ common helpers used by all Google API clients. For more information, see the Supported Python Versions ------------------------- -Python >= 3.6 +Python >= 3.7 Unsupported Python Versions --------------------------- -Python == 2.7, Python == 3.5. +Python == 2.7, Python == 3.5, Python == 3.6. The last version of this library compatible with Python 2.7 and 3.5 is `google-api-core==1.31.1`. + +The last version of this library compatible with Python 3.6 is +`google-api-core==2.8.2`. diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 63a3ad18..5a5bf2a6 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""AsyncIO helpers for :mod:`grpc` supporting 3.6+. +"""AsyncIO helpers for :mod:`grpc` supporting 3.7+. Please combine more detailed docstring in grpc_helpers.py to use following functions. This module is implementing the same surface with AsyncIO semantics. diff --git a/mypy.ini b/mypy.ini index 5c111571..ce33582a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,4 @@ [mypy] -python_version = 3.6 +python_version = 3.7 namespace_packages = True ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index 9b71610c..d0db9914 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,14 +43,14 @@ ] -def _greater_or_equal_than_36(version_string): +def _greater_or_equal_than_37(version_string): tokens = version_string.split(".") for i, token in enumerate(tokens): try: tokens[i] = int(token) except ValueError: pass - return tokens >= [3, 6] + return tokens >= [3, 7] @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -122,9 +122,9 @@ def default(session, install_grpc=True): ), ] - # Inject AsyncIO content and proto-plus, if version >= 3.6. + # Inject AsyncIO content and proto-plus, if version >= 3.7. # proto-plus is needed for a field mask test in test_protobuf_helpers.py - if _greater_or_equal_than_36(session.python): + if _greater_or_equal_than_37(session.python): session.install("asyncmock", "pytest-asyncio", "proto-plus") # Having positional arguments means the user wants to run specific tests. @@ -136,19 +136,19 @@ def default(session, install_grpc=True): session.run(*pytest_args) -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["3.6", "3.10"]) +@nox.session(python=["3.8", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" default(session, install_grpc=False) -@nox.session(python="3.6") +@nox.session(python="3.8") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" @@ -156,8 +156,7 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -# No 3.7 because pytype supports up to 3.6 only. -@nox.session(python="3.6") +@nox.session(python="3.8") def pytype(session): """Run type-checking.""" session.install(".[grpc]", "pytype >= 2019.3.21") @@ -178,7 +177,7 @@ def mypy(session): session.run("mypy", "google", "tests") -@nox.session(python="3.6") +@nox.session(python="3.8") def cover(session): """Run the final coverage report. diff --git a/owlbot.py b/owlbot.py index 9d58f4a1..ab4f4f0a 100644 --- a/owlbot.py +++ b/owlbot.py @@ -30,6 +30,7 @@ ".coveragerc", # layout "CONTRIBUTING.rst", # no systests ".github/workflows/unittest.yml", # exclude unittest gh action + "README.rst", ] templated_files = common.py_library(microgenerator=True, cov_level=100) s.move(templated_files, excludes=excludes) diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d6498..6f069c6c 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/setup.cfg b/setup.cfg index 8857e9b0..f7b5a3bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [pytype] -python_version = 3.6 +python_version = 3.7 inputs = google/ exclude = diff --git a/setup.py b/setup.py index 3c40548d..4919e5f8 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -93,7 +92,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6", + python_requires=">=3.7", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index c3e6ad74..00000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,14 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -googleapis-common-protos==1.56.2 -protobuf==3.15.0 -google-auth==1.25.0 -requests==2.18.0 -packaging==14.3 -grpcio==1.33.2 -grpcio-status==1.33.2 From cb0329ab323a415ff9d2b854f352c7b6b49e87e9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 10 Aug 2022 03:52:06 +0200 Subject: [PATCH 097/126] chore(deps): update actions/setup-python action to v4 (#419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update actions/setup-python action to v4 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- .github/workflows/mypy.yml | 2 +- .github/workflows/unittest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e67dcbc5..f08164f6 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install nox diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 4c1e8c4e..c3027cee 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install nox @@ -55,7 +55,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install coverage From 7472e6d10e8e7ae858be8171b477c972773f88da Mon Sep 17 00:00:00 2001 From: sai-sunder-s <4540365+sai-sunder-s@users.noreply.github.com> Date: Tue, 9 Aug 2022 18:56:45 -0700 Subject: [PATCH 098/126] doc: Update gcloud command for ADC (#406) ADC login command for gcloud is `gcloud auth application-default auth login` Co-authored-by: Anthonios Partheniou --- docs/auth.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.rst b/docs/auth.rst index a9b296d4..3dcc5fd3 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -165,7 +165,7 @@ possible to call Google Cloud APIs with a user account via getting started with the ``google-cloud-*`` library. The simplest way to use credentials from a user account is via -Application Default Credentials using ``gcloud auth login`` +Application Default Credentials using ``gcloud auth application-default login`` (as mentioned above) and :func:`google.auth.default`: .. code:: python From 0dde0f9797ccbaa8a320baa7093e0577b8220905 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:14:05 -0400 Subject: [PATCH 099/126] chore: remove 'pip install' statements from python_library templates [autoapprove] (#424) Source-Link: https://github.com/googleapis/synthtool/commit/1f37ce74cbc4897f35c9ba5c40393b102da913b1 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/lint.yml | 2 +- .kokoro/publish-docs.sh | 4 +- .kokoro/release.sh | 5 +- .kokoro/requirements.in | 8 + .kokoro/requirements.txt | 464 +++++++++++++++++++++++++++++++++++++ 7 files changed, 479 insertions(+), 12 deletions(-) create mode 100644 .kokoro/requirements.in create mode 100644 .kokoro/requirements.txt diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 1ce60852..1c14d7f6 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c -# created: 2022-07-05T18:31:20.838186805Z + digest: sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 +# created: 2022-08-24T15:24:05.205983455Z diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b46d7305..7092a139 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2fce29f0..eae860a2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install nox diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 8acb14e8..1c4d6237 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -21,14 +21,12 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3 -m pip install --user --upgrade --quiet nox +python3 -m pip install --require-hashes -r .kokoro/requirements.txt python3 -m nox --version # build docs nox -s docs -python3 -m pip install --user gcp-docuploader - # create metadata python3 -m docuploader create-metadata \ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ diff --git a/.kokoro/release.sh b/.kokoro/release.sh index a00f93ec..09d841ed 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -16,12 +16,9 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install gcp-releasetool +python3 -m pip install --require-hashes -r .kokoro/requirements.txt python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script -# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. -python3 -m pip install --upgrade twine wheel setuptools - # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in new file mode 100644 index 00000000..7718391a --- /dev/null +++ b/.kokoro/requirements.in @@ -0,0 +1,8 @@ +gcp-docuploader +gcp-releasetool +importlib-metadata +typing-extensions +twine +wheel +setuptools +nox \ No newline at end of file diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt new file mode 100644 index 00000000..c4b824f2 --- /dev/null +++ b/.kokoro/requirements.txt @@ -0,0 +1,464 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==2.0.0 \ + --hash=sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20 \ + --hash=sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e + # via nox +attrs==22.1.0 \ + --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ + --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c + # via gcp-releasetool +bleach==5.0.1 \ + --hash=sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a \ + --hash=sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c + # via readme-renderer +cachetools==5.2.0 \ + --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ + --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db + # via google-auth +certifi==2022.6.15 \ + --hash=sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d \ + --hash=sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412 + # via requests +cffi==1.15.1 \ + --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ + --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ + --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ + --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ + --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ + --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ + --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ + --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ + --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ + --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ + --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ + --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ + --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ + --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ + --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ + --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ + --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ + --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ + --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ + --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ + --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ + --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ + --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ + --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ + --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ + --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ + --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ + --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ + --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ + --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ + --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ + --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ + --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ + --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ + --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ + --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ + --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ + --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ + --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ + --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ + --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ + --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ + --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ + --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ + --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ + --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ + --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ + --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ + --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ + --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ + --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ + --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ + --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ + --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ + --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ + --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ + --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ + --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ + --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ + --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ + --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ + --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ + --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ + --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via cryptography +charset-normalizer==2.1.1 \ + --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ + --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f + # via requests +click==8.0.4 \ + --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ + --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb + # via + # gcp-docuploader + # gcp-releasetool +colorlog==6.6.0 \ + --hash=sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8 \ + --hash=sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e + # via + # gcp-docuploader + # nox +commonmark==0.9.1 \ + --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ + --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 + # via rich +cryptography==37.0.4 \ + --hash=sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59 \ + --hash=sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596 \ + --hash=sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3 \ + --hash=sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5 \ + --hash=sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab \ + --hash=sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884 \ + --hash=sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82 \ + --hash=sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b \ + --hash=sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441 \ + --hash=sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa \ + --hash=sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d \ + --hash=sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b \ + --hash=sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a \ + --hash=sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6 \ + --hash=sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157 \ + --hash=sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280 \ + --hash=sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282 \ + --hash=sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67 \ + --hash=sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8 \ + --hash=sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046 \ + --hash=sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327 \ + --hash=sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9 + # via + # gcp-releasetool + # secretstorage +distlib==0.3.5 \ + --hash=sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe \ + --hash=sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c + # via virtualenv +docutils==0.19 \ + --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ + --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc + # via readme-renderer +filelock==3.8.0 \ + --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ + --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 + # via virtualenv +gcp-docuploader==0.6.3 \ + --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ + --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b + # via -r requirements.in +gcp-releasetool==1.8.6 \ + --hash=sha256:42e51ab8e2e789bc8e22a03c09352962cd3452951c801a2230d564816630304a \ + --hash=sha256:a3518b79d1b243c494eac392a01c7fd65187fd6d52602dcab9b529bc934d4da1 + # via -r requirements.in +google-api-core==2.8.2 \ + --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ + --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.11.0 \ + --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ + --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb + # via + # gcp-releasetool + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.3.2 \ + --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ + --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a + # via google-cloud-storage +google-cloud-storage==2.5.0 \ + --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ + --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 + # via gcp-docuploader +google-crc32c==1.3.0 \ + --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ + --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ + --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \ + --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \ + --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \ + --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \ + --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \ + --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \ + --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \ + --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \ + --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \ + --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \ + --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \ + --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \ + --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \ + --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \ + --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \ + --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \ + --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \ + --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \ + --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \ + --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \ + --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \ + --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \ + --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \ + --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \ + --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \ + --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \ + --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \ + --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \ + --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \ + --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \ + --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \ + --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \ + --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \ + --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \ + --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \ + --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \ + --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \ + --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \ + --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ + --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ + --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 + # via google-resumable-media +google-resumable-media==2.3.3 \ + --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ + --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 + # via google-cloud-storage +googleapis-common-protos==1.56.4 \ + --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ + --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 + # via google-api-core +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +importlib-metadata==4.12.0 \ + --hash=sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670 \ + --hash=sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23 + # via + # -r requirements.in + # twine +jeepney==0.8.0 \ + --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ + --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 + # via + # keyring + # secretstorage +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via gcp-releasetool +keyring==23.8.2 \ + --hash=sha256:0d9973f8891850f1ade5f26aafd06bb16865fbbae3fc56b0defb6a14a2624003 \ + --hash=sha256:10d2a8639663fe2090705a00b8c47c687cacdf97598ea9c11456679fa974473a + # via + # gcp-releasetool + # twine +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via jinja2 +nox==2022.8.7 \ + --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ + --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c + # via -r requirements.in +packaging==21.3 \ + --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ + --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 + # via + # gcp-releasetool + # nox +pkginfo==1.8.3 \ + --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ + --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c + # via twine +platformdirs==2.5.2 \ + --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ + --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 + # via virtualenv +protobuf==3.20.1 \ + --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ + --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ + --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ + --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ + --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ + --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ + --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ + --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ + --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ + --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ + --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ + --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ + --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ + --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ + --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ + --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ + --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ + --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ + --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ + --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ + --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ + --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ + --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ + --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 + # via + # gcp-docuploader + # gcp-releasetool + # google-api-core +py==1.11.0 \ + --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ + --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 + # via nox +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 \ + --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \ + --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74 + # via google-auth +pycparser==2.21 \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via cffi +pygments==2.13.0 \ + --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \ + --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42 + # via + # readme-renderer + # rich +pyjwt==2.4.0 \ + --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ + --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba + # via gcp-releasetool +pyparsing==3.0.9 \ + --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ + --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc + # via packaging +pyperclip==1.8.2 \ + --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 + # via gcp-releasetool +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via gcp-releasetool +readme-renderer==37.0 \ + --hash=sha256:07b7ea234e03e58f77cc222e206e6abb8f4c0435becce5104794ee591f9301c5 \ + --hash=sha256:9fa416704703e509eeb900696751c908ddeb2011319d93700d8f18baff887a69 + # via twine +requests==2.28.1 \ + --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ + --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 + # via + # gcp-releasetool + # google-api-core + # google-cloud-storage + # requests-toolbelt + # twine +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==12.5.1 \ + --hash=sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb \ + --hash=sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca + # via twine +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +secretstorage==3.3.3 \ + --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ + --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 + # via keyring +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # bleach + # gcp-docuploader + # google-auth + # python-dateutil +twine==4.0.1 \ + --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ + --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 + # via -r requirements.in +typing-extensions==4.3.0 \ + --hash=sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02 \ + --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6 + # via -r requirements.in +urllib3==1.26.12 \ + --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ + --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 + # via + # requests + # twine +virtualenv==20.16.3 \ + --hash=sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1 \ + --hash=sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9 + # via nox +webencodings==0.5.1 \ + --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ + --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 + # via bleach +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 + # via -r requirements.in +zipp==3.8.1 \ + --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \ + --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.2.0 \ + --hash=sha256:7f4bc85450898a09f76ebf28b72fa25bc7111f6c7d665d514a60bba9c75ef2a9 \ + --hash=sha256:a3ca5857c89f82f5c9410e8508cb32f4872a3bafd4aa7ae122a24ca33bccc750 + # via -r requirements.in From d1026e7912732384caf2c2c04da45e2ce6a8af7f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 24 Aug 2022 19:33:52 -0400 Subject: [PATCH 100/126] chore(python): exclude path in renovate.json [autoapprove] (#425) Source-Link: https://github.com/googleapis/synthtool/commit/69fabaee9eca28af7ecaa02c86895e606fbbebd6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- renovate.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 1c14d7f6..c6acdf3f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 -# created: 2022-08-24T15:24:05.205983455Z + digest: sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 +# created: 2022-08-24T17:07:22.006876712Z diff --git a/renovate.json b/renovate.json index c21036d3..566a70f3 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From 7371d025d2cdcdabb2ff670706868b34a13c4990 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 08:52:17 -0400 Subject: [PATCH 101/126] chore(python): update dependency distlib (#429) Source-Link: https://github.com/googleapis/synthtool/commit/c4dd5953003d13b239f872d329c3146586bb417e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index c6acdf3f..23e106b6 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 -# created: 2022-08-24T17:07:22.006876712Z + digest: sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 +# created: 2022-08-29T17:28:30.441852797Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index c4b824f2..4b29ef24 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -136,9 +136,9 @@ cryptography==37.0.4 \ # via # gcp-releasetool # secretstorage -distlib==0.3.5 \ - --hash=sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe \ - --hash=sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c +distlib==0.3.6 \ + --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \ + --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e # via virtualenv docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ From c3ad8ea67447e3d8a1154d7a9221e116f60d425a Mon Sep 17 00:00:00 2001 From: Vadym Matsishevskyi <25311427+vam-google@users.noreply.github.com> Date: Tue, 30 Aug 2022 11:02:04 -0700 Subject: [PATCH 102/126] feat: Make grpc transcode logic work in terms of protobuf python objects (#428) * feat: Make grpc transcode logic work in terms of protobuf python objects (for context: [gRPC Transcoding](https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44)) Previously it worked on dictionaries only, but that causes problems. In GAPIC the dictionaries are created through the same logic as JSON (there is no better built-in way), thus applying [protobuf json mapping](https://developers.google.com/protocol-buffers/docs/proto3#json) conversion logic in the process. Unfortunately converting a protobuf object to a dictionary and to JSON, although similar, are not the same thing. Specifically the `Timestamp`, `Duration`, `FieldMask`, `uint64`, `int64`, and `*Value` protobuf messages are converted to strings for JSON (instead of being properly converted to dicts for most of those types, and `int64/uint64` converted to `int` respectively). As a result a rountrip that GAPIC was relying on (protobuf object -> dict -> transcode -> protobuf object) did not work properly. For example, when converted to dictionary, every int64 field would be converted to `string` (because it is what proto-JSON mapping spec requires), but later, when we need to rebuild a message from a transcoded dictionary that would fail with the following error: ``` TypeError: '0' has type str, but expected one of: int ``` Note, `*Rules` thing from proto-plus does not help, becuase the type may happen inside common native protobuf stub messsages (like `google.type.Money`), fields of which are outside of scope of the proto-plus custom conversion logic. Also, this change greatly simplifies the procedure of transcodding, eliminating multiple conversion steps (to and from dictionaries multiple times) making the whole logic significanly more efficient (python gapics are nutoriously known to be slow due to proto-plus stuff, so efficiency is important) and robust (JSON conversion logic does not interfere anymore with pure protobuf objects grpc transcoding) * reformat code using black * reformat code according to flake8 Co-authored-by: Anthonios Partheniou --- google/api_core/path_template.py | 60 ++++-- tests/unit/test_path_template.py | 308 ++++++++++++++++++++++++++++--- 2 files changed, 325 insertions(+), 43 deletions(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index a99a4c81..2639459a 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -176,7 +176,7 @@ def get_field(request, field): """Get the value of a field from a given dictionary. Args: - request (dict): A dictionary object. + request (dict | Message): A dictionary or a Message object. field (str): The key to the request in dot notation. Returns: @@ -184,10 +184,12 @@ def get_field(request, field): """ parts = field.split(".") value = request + for part in parts: if not isinstance(value, dict): - return - value = value.get(part) + value = getattr(value, part, None) + else: + value = value.get(part) if isinstance(value, dict): return return value @@ -197,19 +199,27 @@ def delete_field(request, field): """Delete the value of a field from a given dictionary. Args: - request (dict): A dictionary object. + request (dict | Message): A dictionary object or a Message. field (str): The key to the request in dot notation. """ parts = deque(field.split(".")) while len(parts) > 1: - if not isinstance(request, dict): - return part = parts.popleft() - request = request.get(part) + if not isinstance(request, dict): + if hasattr(request, part): + request = getattr(request, part, None) + else: + return + else: + request = request.get(part) part = parts.popleft() if not isinstance(request, dict): - return - request.pop(part, None) + if hasattr(request, part): + request.ClearField(part) + else: + return + else: + request.pop(part, None) def validate(tmpl, path): @@ -237,7 +247,7 @@ def validate(tmpl, path): return True if re.match(pattern, path) is not None else False -def transcode(http_options, **request_kwargs): +def transcode(http_options, message=None, **request_kwargs): """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 @@ -248,18 +258,20 @@ def transcode(http_options, **request_kwargs): 'body' (str): The body field name (optional) (This is a simplified representation of the proto option `google.api.http`) + message (Message) : A request object (optional) request_kwargs (dict) : A dict representing the request object Returns: dict: The transcoded request with these keys, 'method' (str) : The http method 'uri' (str) : The expanded uri - 'body' (dict) : A dict representing the body (optional) - 'query_params' (dict) : A dict mapping query parameter variables and values + 'body' (dict | Message) : A dict or a Message representing the body (optional) + 'query_params' (dict | Message) : A dict or Message mapping query parameter variables and values Raises: ValueError: If the request does not match the given template. """ + transcoded_value = message or request_kwargs for http_option in http_options: request = {} @@ -268,27 +280,35 @@ def transcode(http_options, **request_kwargs): path_fields = [ match.group("name") for match in _VARIABLE_RE.finditer(uri_template) ] - path_args = {field: get_field(request_kwargs, field) for field in path_fields} + path_args = {field: get_field(transcoded_value, field) for field in path_fields} request["uri"] = expand(uri_template, **path_args) - # Remove fields used in uri path from request - leftovers = copy.deepcopy(request_kwargs) - for path_field in path_fields: - delete_field(leftovers, path_field) if not validate(uri_template, request["uri"]) or not all(path_args.values()): continue + # Remove fields used in uri path from request + leftovers = copy.deepcopy(transcoded_value) + for path_field in path_fields: + delete_field(leftovers, path_field) + # Assign body and query params body = http_option.get("body") if body: if body == "*": request["body"] = leftovers - request["query_params"] = {} + if message: + request["query_params"] = message.__class__() + else: + request["query_params"] = {} else: try: - request["body"] = leftovers.pop(body) - except KeyError: + if message: + request["body"] = getattr(leftovers, body) + delete_field(leftovers, body) + else: + request["body"] = leftovers.pop(body) + except (KeyError, AttributeError): continue request["query_params"] = leftovers else: diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index c12b35fc..73d351c0 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -17,6 +17,7 @@ import mock import pytest +from google.api import auth_pb2 from google.api_core import path_template @@ -171,113 +172,264 @@ def test__replace_variable_with_pattern(): @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ [ [["get", "/v1/no/template", ""]], + None, {"foo": "bar"}, ["get", "/v1/no/template", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/no/template", ""]], + auth_pb2.AuthenticationRule(selector="bar"), + {}, + [ + "get", + "/v1/no/template", + None, + auth_pb2.AuthenticationRule(selector="bar"), + ], + ], # Single templates [ [["get", "/v1/{field}", ""]], + None, {"field": "parent"}, ["get", "/v1/parent", {}, {}], ], + [ + [["get", "/v1/{selector}", ""]], + auth_pb2.AuthenticationRule(selector="parent"), + {}, + ["get", "/v1/parent", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field.sub}", ""]], + None, {"field": {"sub": "parent"}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="parent"), + ), + {}, + [ + "get", + "/v1/parent", + None, + auth_pb2.AuthenticationRule( + selector="bar", oauth=auth_pb2.OAuthRequirements() + ), + ], + ], ], ) -def test_transcode_base_case(http_options, request_kwargs, expected_result): +def test_transcode_base_case(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ [ [["get", "/v1/{field.subfield}", ""]], + None, {"field": {"subfield": "parent"}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="parent"), + ), + {}, + [ + "get", + "/v1/parent", + None, + auth_pb2.AuthenticationRule( + selector="bar", oauth=auth_pb2.OAuthRequirements() + ), + ], + ], [ [["get", "/v1/{field.subfield.subsubfield}", ""]], + None, {"field": {"subfield": {"subsubfield": "parent"}}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {"subfield": {}}, "foo": "bar"}], ], [ [["get", "/v1/{field.subfield1}/{field.subfield2}", ""]], + None, {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, ["get", "/v1/parent/child", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{selector}/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="parent", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="child"), + ), + {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, + [ + "get", + "/v1/parent/child", + None, + auth_pb2.AuthenticationRule(oauth=auth_pb2.OAuthRequirements()), + ], + ], ], ) -def test_transcode_subfields(http_options, request_kwargs, expected_result): +def test_transcode_subfields(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Single segment wildcard [ [["get", "/v1/{field=*}", ""]], + None, {"field": "parent"}, ["get", "/v1/parent", {}, {}], ], + [ + [["get", "/v1/{selector=*}", ""]], + auth_pb2.AuthenticationRule(selector="parent"), + {}, + ["get", "/v1/parent", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/*/b/*}", ""]], + None, {"field": "a/parent/b/child", "foo": "bar"}, ["get", "/v1/a/parent/b/child", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/*/b/*}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/b/child", allow_without_credential=True + ), + {}, + [ + "get", + "/v1/a/parent/b/child", + None, + auth_pb2.AuthenticationRule(allow_without_credential=True), + ], + ], # Double segment wildcard [ [["get", "/v1/{field=**}", ""]], + None, {"field": "parent/p1"}, ["get", "/v1/parent/p1", {}, {}], ], + [ + [["get", "/v1/{selector=**}", ""]], + auth_pb2.AuthenticationRule(selector="parent/p1"), + {}, + ["get", "/v1/parent/p1", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/**/b/**}", ""]], + None, {"field": "a/parent/p1/b/child/c1", "foo": "bar"}, ["get", "/v1/a/parent/p1/b/child/c1", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/**/b/**}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/p1/b/child/c1", allow_without_credential=True + ), + {}, + [ + "get", + "/v1/a/parent/p1/b/child/c1", + None, + auth_pb2.AuthenticationRule(allow_without_credential=True), + ], + ], # Combined single and double segment wildcard [ [["get", "/v1/{field=a/*/b/**}", ""]], + None, {"field": "a/parent/b/child/c1"}, ["get", "/v1/a/parent/b/child/c1", {}, {}], ], + [ + [["get", "/v1/{selector=a/*/b/**}", ""]], + auth_pb2.AuthenticationRule(selector="a/parent/b/child/c1"), + {}, + ["get", "/v1/a/parent/b/child/c1", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/**/b/*}/v2/{name}", ""]], + None, {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, ["get", "/v1/a/parent/p1/b/child/v2/first", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/**/b/*}/v2/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/p1/b/child", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first"), + ), + {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, + [ + "get", + "/v1/a/parent/p1/b/child/v2/first", + None, + auth_pb2.AuthenticationRule(oauth=auth_pb2.OAuthRequirements()), + ], + ], ], ) -def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): +def test_transcode_with_wildcard( + http_options, message, request_kwargs, expected_result +): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Single field body [ [["post", "/v1/no/template", "data"]], + None, {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, ["post", "/v1/no/template", {"id": 1, "info": "some info"}, {"foo": "bar"}], ], + [ + [["post", "/v1/no/template", "oauth"]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="child"), + ), + {}, + [ + "post", + "/v1/no/template", + auth_pb2.OAuthRequirements(canonical_scopes="child"), + auth_pb2.AuthenticationRule(selector="bar"), + ], + ], [ [["post", "/v1/{field=a/*}/b/{name=**}", "data"]], + None, { "field": "a/parent", "name": "first/last", @@ -291,9 +443,29 @@ def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): {"foo": "bar"}, ], ], + [ + [["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "oauth"]], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + requirements=[auth_pb2.AuthRequirement(provider_id="p")], + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + {}, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.OAuthRequirements(), + auth_pb2.AuthenticationRule( + requirements=[auth_pb2.AuthRequirement(provider_id="p")], + allow_without_credential=True, + ), + ], + ], # Wildcard body [ [["post", "/v1/{field=a/*}/b/{name=**}", "*"]], + None, { "field": "a/parent", "name": "first/last", @@ -307,16 +479,38 @@ def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): {}, ], ], + [ + [["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "*"]], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + auth_pb2.AuthenticationRule(), + ], + ], ], ) -def test_transcode_with_body(http_options, request_kwargs, expected_result): +def test_transcode_with_body(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Additional bindings [ @@ -324,6 +518,7 @@ def test_transcode_with_body(http_options, request_kwargs, expected_result): ["post", "/v1/{field=a/*}/b/{name=**}", "extra_data"], ["post", "/v1/{field=a/*}/b/{name=**}", "*"], ], + None, { "field": "a/parent", "name": "first/last", @@ -337,38 +532,105 @@ def test_transcode_with_body(http_options, request_kwargs, expected_result): {}, ], ], + [ + [ + [ + "post", + "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", + "extra_data", + ], + ["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "*"], + ], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + {}, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + auth_pb2.AuthenticationRule(), + ], + ], [ [ ["get", "/v1/{field=a/*}/b/{name=**}", ""], ["get", "/v1/{field=a/*}/b/first/last", ""], ], + None, {"field": "a/parent", "foo": "bar"}, ["get", "/v1/a/parent/b/first/last", {}, {"foo": "bar"}], ], + [ + [ + ["get", "/v1/{selector=a/*}/b/{oauth.allow_without_credential=**}", ""], + ["get", "/v1/{selector=a/*}/b/first/last", ""], + ], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(), + ), + {}, + [ + "get", + "/v1/a/parent/b/first/last", + None, + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + ], + ], ], ) def test_transcode_with_additional_bindings( - http_options, request_kwargs, expected_result + http_options, message, request_kwargs, expected_result ): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs", + "http_options, message, request_kwargs", [ - [[["get", "/v1/{name}", ""]], {"foo": "bar"}], - [[["get", "/v1/{name}", ""]], {"name": "first/last"}], - [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], - [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], - [[["post", "/v1/{first_name}", "data"]], {"last_name": "last"}], + [[["get", "/v1/{name}", ""]], None, {"foo": "bar"}], + [[["get", "/v1/{selector}", ""]], auth_pb2.AuthenticationRule(), {}], + [[["get", "/v1/{name}", ""]], auth_pb2.AuthenticationRule(), {}], + [[["get", "/v1/{name}", ""]], None, {"name": "first/last"}], + [ + [["get", "/v1/{selector}", ""]], + auth_pb2.AuthenticationRule(selector="first/last"), + {}, + ], + [[["get", "/v1/{name=mr/*/*}", ""]], None, {"name": "first/last"}], + [ + [["get", "/v1/{selector=mr/*/*}", ""]], + auth_pb2.AuthenticationRule(selector="first/last"), + {}, + ], + [[["post", "/v1/{name}", "data"]], None, {"name": "first/last"}], + [ + [["post", "/v1/{selector}", "data"]], + auth_pb2.AuthenticationRule(selector="first"), + {}, + ], + [[["post", "/v1/{first_name}", "data"]], None, {"last_name": "last"}], + [ + [["post", "/v1/{first_name}", ""]], + auth_pb2.AuthenticationRule(selector="first"), + {}, + ], ], ) -def test_transcode_fails(http_options, request_kwargs): +def test_transcode_fails(http_options, message, request_kwargs): http_options, _ = helper_test_transcode(http_options, range(4)) with pytest.raises(ValueError): - path_template.transcode(http_options, **request_kwargs) + path_template.transcode(http_options, message, **request_kwargs) def helper_test_transcode(http_options_list, expected_result_list): From 29aa9efc8c6e7193bdbf00f0e6d8cb5840ad6b6e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 18:28:12 +0000 Subject: [PATCH 103/126] chore: fix path to requirements.txt in release script [autoapprove] (#430) Source-Link: https://github.com/googleapis/synthtool/commit/fdba3ed145bdb2f4f3eff434d4284b1d03b80d34 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 --- .github/.OwlBot.lock.yaml | 3 +-- .kokoro/release.sh | 2 +- .kokoro/requirements.txt | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 23e106b6..0d9eb2af 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 -# created: 2022-08-29T17:28:30.441852797Z + digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 09d841ed..697f7e6d 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -16,7 +16,7 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install --require-hashes -r .kokoro/requirements.txt +python3 -m pip install --require-hashes -r github/python-api-core/.kokoro/requirements.txt python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script # Disable buffering, so that the logs stream through. diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 4b29ef24..92b2f727 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -100,9 +100,9 @@ click==8.0.4 \ # via # gcp-docuploader # gcp-releasetool -colorlog==6.6.0 \ - --hash=sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8 \ - --hash=sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e +colorlog==6.7.0 \ + --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ + --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 # via # gcp-docuploader # nox @@ -152,9 +152,9 @@ gcp-docuploader==0.6.3 \ --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b # via -r requirements.in -gcp-releasetool==1.8.6 \ - --hash=sha256:42e51ab8e2e789bc8e22a03c09352962cd3452951c801a2230d564816630304a \ - --hash=sha256:a3518b79d1b243c494eac392a01c7fd65187fd6d52602dcab9b529bc934d4da1 +gcp-releasetool==1.8.7 \ + --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ + --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d # via -r requirements.in google-api-core==2.8.2 \ --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ @@ -251,9 +251,9 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.8.2 \ - --hash=sha256:0d9973f8891850f1ade5f26aafd06bb16865fbbae3fc56b0defb6a14a2624003 \ - --hash=sha256:10d2a8639663fe2090705a00b8c47c687cacdf97598ea9c11456679fa974473a +keyring==23.9.0 \ + --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \ + --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db # via # gcp-releasetool # twine @@ -440,9 +440,9 @@ urllib3==1.26.12 \ # via # requests # twine -virtualenv==20.16.3 \ - --hash=sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1 \ - --hash=sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9 +virtualenv==20.16.4 \ + --hash=sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782 \ + --hash=sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22 # via nox webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ From 8c19609d6244930bd91fd5f40ef9b5b65584c4a5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Sep 2022 16:22:35 -0400 Subject: [PATCH 104/126] fix: restore support for grpcio-gcp (#418) docs: add a note that grpcio-gcp is only supported in environments with protobuf < 4.x.x docs: raise DeprecationWarning when 'grpcio-gcp' is used fix(deps): require protobuf >= 3.20.1 --- .github/sync-repo-settings.yaml | 4 ++ .github/workflows/unittest.yml | 2 +- google/api_core/grpc_helpers.py | 28 ++++++++++- noxfile.py | 15 ++++++ setup.py | 4 +- testing/constraints-3.7.txt | 3 +- testing/constraints-3.8.txt | 2 + tests/unit/test_grpc_helpers.py | 88 ++++++++++++++++++++++++++++----- 8 files changed, 129 insertions(+), 17 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index e2d70f90..6d2c2a0e 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -11,6 +11,10 @@ branchProtectionRules: # No Kokoro: the following are Github actions - 'lint' - 'mypy' + - 'unit_grpc_gcp-3.7' + - 'unit_grpc_gcp-3.8' + - 'unit_grpc_gcp-3.9' + - 'unit_grpc_gcp-3.10' - 'unit-3.7' - 'unit-3.8' - 'unit-3.9' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index c3027cee..cd1d4d60 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - option: ["", "_wo_grpc"] + option: ["", "_grpc_gcp", "_wo_grpc"] python: - "3.7" - "3.8" diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 47d27726..bf04ae4c 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -16,6 +16,7 @@ import collections import functools +import warnings import grpc @@ -24,6 +25,27 @@ import google.auth.credentials import google.auth.transport.grpc import google.auth.transport.requests +import google.protobuf + +PROTOBUF_VERSION = google.protobuf.__version__ + +# The grpcio-gcp package only has support for protobuf < 4 +if PROTOBUF_VERSION[0:2] == "3.": + try: + import grpc_gcp + + warnings.warn( + """Support for grpcio-gcp is deprecated. This feature will be + removed from `google-api-core` after January 1, 2024. If you need to + continue to use this feature, please pin to a specific version of + `google-api-core`.""", + DeprecationWarning, + ) + HAS_GRPC_GCP = True + except ImportError: + HAS_GRPC_GCP = False +else: + HAS_GRPC_GCP = False # The list of gRPC Callable interfaces that return iterators. @@ -275,7 +297,9 @@ def create_channel( default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". - kwargs: Additional key-word args passed to :func:`grpc.secure_channel`. + kwargs: Additional key-word args passed to + :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + Note: `grpc_gcp` is only supported in environments with protobuf < 4.0.0. Returns: grpc.Channel: The created channel. @@ -294,6 +318,8 @@ def create_channel( default_host=default_host, ) + if HAS_GRPC_GCP: + return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) return grpc.secure_channel(target, composite_credentials, **kwargs) diff --git a/noxfile.py b/noxfile.py index d0db9914..2d8f1e02 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,6 +32,7 @@ # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", + "unit_grpc_gcp", "unit_wo_grpc", "cover", "pytype", @@ -142,6 +143,20 @@ def unit(session): default(session) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) +def unit_grpc_gcp(session): + """Run the unit test suite with grpcio-gcp installed.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + # Install grpcio-gcp + session.install("-e", ".[grpcgcp]", "-c", constraints_path) + # Install protobuf < 4.0.0 + session.install("protobuf<4.0.0") + + default(session) + + @nox.session(python=["3.8", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" diff --git a/setup.py b/setup.py index 4919e5f8..2dd2a0cd 100644 --- a/setup.py +++ b/setup.py @@ -30,12 +30,14 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.15.0, <5.0.0dev", + "protobuf >= 3.20.1, <5.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], + "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", + "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index c3e6ad74..fe671145 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,9 +6,10 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 googleapis-common-protos==1.56.2 -protobuf==3.15.0 +protobuf==3.20.1 google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 grpcio-status==1.33.2 +grpcio-gcp==0.2.2 diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index e69de29b..8d760bbd 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -0,0 +1,2 @@ +googleapis-common-protos==1.56.3 +protobuf==4.21.5 \ No newline at end of file diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 649072f0..8b9fd9f1 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -365,7 +365,10 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c default.assert_called_once_with(scopes=None, default_scopes=None) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True) @@ -397,7 +400,10 @@ def test_create_channel_implicit_with_default_host( mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host ) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -420,7 +426,11 @@ def test_create_channel_implicit_with_ssl_creds( composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -442,7 +452,10 @@ def test_create_channel_implicit_with_scopes( default.assert_called_once_with(scopes=["one", "two"], default_scopes=None) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -464,7 +477,10 @@ def test_create_channel_implicit_with_default_scopes( default.assert_called_once_with(scopes=None, default_scopes=["three", "four"]) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) def test_create_channel_explicit_with_duplicate_credentials(): @@ -492,7 +508,11 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -512,7 +532,11 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -536,7 +560,11 @@ def test_create_channel_explicit_default_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -558,7 +586,11 @@ def test_create_channel_explicit_with_quota_project( credentials.with_quota_project.assert_called_once_with("project-foo") assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -583,7 +615,11 @@ def test_create_channel_with_credentials_file( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -611,7 +647,11 @@ def test_create_channel_with_credentials_file_and_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -639,11 +679,33 @@ def test_create_channel_with_credentials_file_and_default_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) + + +@pytest.mark.skipif( + not grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available" +) +@mock.patch("grpc_gcp.secure_channel") +def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): + target = "example.com:443" + scopes = ["test_scope"] + + credentials = mock.create_autospec(google.auth.credentials.Scoped, instance=True) + credentials.requires_scopes = True + + grpc_helpers.create_channel(target, credentials=credentials, scopes=scopes) + grpc_gcp_secure_channel.assert_called() + + credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) +@pytest.mark.skipif(grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available") @mock.patch("grpc.secure_channel") -def test_create_channel(grpc_secure_channel): +def test_create_channel_without_grpc_gcp(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From 922f46852ca4420a4ac217db8b814561735a4415 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:31:54 -0400 Subject: [PATCH 105/126] chore(main): release 2.9.0 (#422) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 13 +++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e22ab8..90826869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.9.0](https://github.com/googleapis/python-api-core/compare/v2.8.2...v2.9.0) (2022-09-01) + + +### Features + +* Make grpc transcode logic work in terms of protobuf python objects ([#428](https://github.com/googleapis/python-api-core/issues/428)) ([c3ad8ea](https://github.com/googleapis/python-api-core/commit/c3ad8ea67447e3d8a1154d7a9221e116f60d425a)) + + +### Bug Fixes + +* Require python 3.7+ ([#410](https://github.com/googleapis/python-api-core/issues/410)) ([7ddb8c0](https://github.com/googleapis/python-api-core/commit/7ddb8c00e6be7ab6905a9a802ad1c3063fbfa46c)) +* Restore support for grpcio-gcp ([#418](https://github.com/googleapis/python-api-core/issues/418)) ([8c19609](https://github.com/googleapis/python-api-core/commit/8c19609d6244930bd91fd5f40ef9b5b65584c4a5)) + ## [2.8.2](https://github.com/googleapis/python-api-core/compare/v2.8.1...v2.8.2) (2022-06-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 839a77a1..b2a8c553 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.2" +__version__ = "2.9.0" From 9066ed49dbdb47b2245507eb2c25acd97984be54 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 19:27:59 -0400 Subject: [PATCH 106/126] chore(python): update .kokoro/requirements.txt (#431) Source-Link: https://github.com/googleapis/synthtool/commit/703554a14c7479542335b62fa69279f93a9e38ec Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/requirements.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0d9eb2af..2fa0f7c4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 + digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 92b2f727..385f2d4d 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -241,6 +241,10 @@ importlib-metadata==4.12.0 \ # via # -r requirements.in # twine +jaraco-classes==3.2.2 \ + --hash=sha256:6745f113b0b588239ceb49532aa09c3ebb947433ce311ef2f8e3ad64ebb74594 \ + --hash=sha256:e6ef6fd3fcf4579a7a019d87d1e56a883f4e4c35cfe925f86731abc58804e647 + # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 @@ -299,6 +303,10 @@ markupsafe==2.1.1 \ --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 # via jinja2 +more-itertools==8.14.0 \ + --hash=sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2 \ + --hash=sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750 + # via jaraco-classes nox==2022.8.7 \ --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c From 83678e94e1081f9087b19c43f26fad4774184d66 Mon Sep 17 00:00:00 2001 From: Victor Chudnovsky Date: Thu, 1 Sep 2022 17:27:04 -0700 Subject: [PATCH 107/126] feat: add 'strict' to flatten_query_params to lower-case bools (#433) * feat: add 'strict' to flatten_query_params to lower-case bools * pylint Co-authored-by: Anthonios Partheniou --- google/api_core/rest_helpers.py | 49 +++++++++++++++++++++------------ tests/unit/test_rest_helpers.py | 19 ++++++++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/google/api_core/rest_helpers.py b/google/api_core/rest_helpers.py index 23fb614f..a78822f1 100644 --- a/google/api_core/rest_helpers.py +++ b/google/api_core/rest_helpers.py @@ -18,8 +18,8 @@ import operator -def flatten_query_params(obj): - """Flatten a nested dict into a list of (name,value) tuples. +def flatten_query_params(obj, strict=False): + """Flatten a dict into a list of (name,value) tuples. The result is suitable for setting query params on an http request. @@ -28,9 +28,10 @@ def flatten_query_params(obj): >>> obj = {'a': ... {'b': ... {'c': ['x', 'y', 'z']} }, - ... 'd': 'uvw', } - >>> flatten_query_params(obj) - [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw')] + ... 'd': 'uvw', + ... 'e': True, } + >>> flatten_query_params(obj, strict=True) + [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw'), ('e', 'true')] Note that, as described in https://github.com/googleapis/googleapis/blob/48d9fb8c8e287c472af500221c6450ecd45d7d39/google/api/http.proto#L117, @@ -38,7 +39,9 @@ def flatten_query_params(obj): This is enforced in this function. Args: - obj: a nested dictionary (from json), or None + obj: a possibly nested dictionary (from json), or None + strict: a bool, defaulting to False, to enforce that all values in the + result tuples be strings and, if boolean, lower-cased. Returns: a list of tuples, with each tuple having a (possibly) multi-part name and a scalar value. @@ -51,17 +54,17 @@ def flatten_query_params(obj): if obj is not None and not isinstance(obj, dict): raise TypeError("flatten_query_params must be called with dict object") - return _flatten(obj, key_path=[]) + return _flatten(obj, key_path=[], strict=strict) -def _flatten(obj, key_path): +def _flatten(obj, key_path, strict=False): if obj is None: return [] if isinstance(obj, dict): - return _flatten_dict(obj, key_path=key_path) + return _flatten_dict(obj, key_path=key_path, strict=strict) if isinstance(obj, list): - return _flatten_list(obj, key_path=key_path) - return _flatten_value(obj, key_path=key_path) + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) def _is_primitive_value(obj): @@ -74,21 +77,33 @@ def _is_primitive_value(obj): return True -def _flatten_value(obj, key_path): - return [(".".join(key_path), obj)] +def _flatten_value(obj, key_path, strict=False): + return [(".".join(key_path), _canonicalize(obj, strict=strict))] -def _flatten_dict(obj, key_path): - items = (_flatten(value, key_path=key_path + [key]) for key, value in obj.items()) +def _flatten_dict(obj, key_path, strict=False): + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) return functools.reduce(operator.concat, items, []) -def _flatten_list(elems, key_path): +def _flatten_list(elems, key_path, strict=False): # Only lists of scalar values are supported. # The name (key_path) is repeated for each value. items = ( - _flatten_value(elem, key_path=key_path) + _flatten_value(elem, key_path=key_path, strict=strict) for elem in elems if _is_primitive_value(elem) ) return functools.reduce(operator.concat, items, []) + + +def _canonicalize(obj, strict=False): + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj diff --git a/tests/unit/test_rest_helpers.py b/tests/unit/test_rest_helpers.py index 5932fa55..ff1a43f0 100644 --- a/tests/unit/test_rest_helpers.py +++ b/tests/unit/test_rest_helpers.py @@ -36,9 +36,26 @@ def test_flatten_empty_dict(): def test_flatten_simple_dict(): - assert rest_helpers.flatten_query_params({"a": "abc", "b": "def"}) == [ + obj = {"a": "abc", "b": "def", "c": True, "d": False, "e": 10, "f": -3.76} + assert rest_helpers.flatten_query_params(obj) == [ ("a", "abc"), ("b", "def"), + ("c", True), + ("d", False), + ("e", 10), + ("f", -3.76), + ] + + +def test_flatten_simple_dict_strict(): + obj = {"a": "abc", "b": "def", "c": True, "d": False, "e": 10, "f": -3.76} + assert rest_helpers.flatten_query_params(obj, strict=True) == [ + ("a", "abc"), + ("b", "def"), + ("c", "true"), + ("d", "false"), + ("e", "10"), + ("f", "-3.76"), ] From cea67dafecc71481b0c362763da354a541966c79 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 17:43:34 -0700 Subject: [PATCH 108/126] chore(main): release 2.10.0 (#434) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90826869..268cb4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.10.0](https://github.com/googleapis/python-api-core/compare/v2.9.0...v2.10.0) (2022-09-02) + + +### Features + +* Add 'strict' to flatten_query_params to lower-case bools ([#433](https://github.com/googleapis/python-api-core/issues/433)) ([83678e9](https://github.com/googleapis/python-api-core/commit/83678e94e1081f9087b19c43f26fad4774184d66)) + ## [2.9.0](https://github.com/googleapis/python-api-core/compare/v2.8.2...v2.9.0) (2022-09-01) diff --git a/google/api_core/version.py b/google/api_core/version.py index b2a8c553..13e710fc 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.9.0" +__version__ = "2.10.0" From fe617c205918a3e4dfddeb06123e70540898032e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 18:44:27 +0000 Subject: [PATCH 109/126] chore(python): exclude setup.py in renovate config (#436) Source-Link: https://github.com/googleapis/synthtool/commit/56da63e80c384a871356d1ea6640802017f213b4 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2fa0f7c4..b8dcb4a4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b + digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 diff --git a/renovate.json b/renovate.json index 566a70f3..39b2a0ec 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From 538df80ed6d21f43b512a73853935f7a7b9bdf52 Mon Sep 17 00:00:00 2001 From: Vadym Matsishevskyi <25311427+vam-google@users.noreply.github.com> Date: Wed, 14 Sep 2022 14:47:55 +0000 Subject: [PATCH 110/126] fix: Improve transcoding error message (#442) * fix: Improve transcodding error message This fixes https://github.com/googleapis/python-api-core/issues/441 and https://github.com/googleapis/python-api-core/issues/440. Also, with this change we stop outputting the whole request message in transcodding erro message to prevent leaking any confidential information from a request message in a form of an error in a log message. * reformat code with black --- google/api_core/path_template.py | 35 ++++++++++++++++++++++++++------ tests/unit/test_path_template.py | 3 ++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index 2639459a..b8ebb2af 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -272,15 +272,19 @@ def transcode(http_options, message=None, **request_kwargs): ValueError: If the request does not match the given template. """ transcoded_value = message or request_kwargs + bindings = [] for http_option in http_options: request = {} # Assign path uri_template = http_option["uri"] - path_fields = [ - match.group("name") for match in _VARIABLE_RE.finditer(uri_template) + fields = [ + (m.group("name"), m.group("template")) + for m in _VARIABLE_RE.finditer(uri_template) ] - path_args = {field: get_field(transcoded_value, field) for field in path_fields} + bindings.append((uri_template, fields)) + + path_args = {field: get_field(transcoded_value, field) for field, _ in fields} request["uri"] = expand(uri_template, **path_args) if not validate(uri_template, request["uri"]) or not all(path_args.values()): @@ -288,7 +292,7 @@ def transcode(http_options, message=None, **request_kwargs): # Remove fields used in uri path from request leftovers = copy.deepcopy(transcoded_value) - for path_field in path_fields: + for path_field, _ in fields: delete_field(leftovers, path_field) # Assign body and query params @@ -316,8 +320,27 @@ def transcode(http_options, message=None, **request_kwargs): request["method"] = http_option["method"] return request + bindings_description = [ + '\n\tURI: "{}"' + "\n\tRequired request fields:\n\t\t{}".format( + uri, + "\n\t\t".join( + [ + 'field: "{}", pattern: "{}"'.format(n, p if p else "*") + for n, p in fields + ] + ), + ) + for uri, fields in bindings + ] + raise ValueError( - "Request {} does not match any URL path template in available HttpRule's {}".format( - request_kwargs, [opt["uri"] for opt in http_options] + "Invalid request." + "\nSome of the fields of the request message are either not initialized or " + "initialized with an invalid value." + "\nPlease make sure your request matches at least one accepted HTTP binding." + "\nTo match a binding the request message must have all the required fields " + "initialized with values matching their patterns as listed below:{}".format( + "\n".join(bindings_description) ) ) diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index 73d351c0..808b36f3 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -629,8 +629,9 @@ def test_transcode_with_additional_bindings( ) def test_transcode_fails(http_options, message, request_kwargs): http_options, _ = helper_test_transcode(http_options, range(4)) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: path_template.transcode(http_options, message, **request_kwargs) + assert str(exc_info.value).count("URI") == len(http_options) def helper_test_transcode(http_options_list, expected_result_list): From b1f995739b6a9f76eb6a4b7c6427d63a11ce352c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 11:16:34 -0400 Subject: [PATCH 111/126] chore(main): release 2.10.1 (#443) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 268cb4d6..b417e98e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.10.1](https://github.com/googleapis/python-api-core/compare/v2.10.0...v2.10.1) (2022-09-14) + + +### Bug Fixes + +* Improve transcoding error message ([#442](https://github.com/googleapis/python-api-core/issues/442)) ([538df80](https://github.com/googleapis/python-api-core/commit/538df80ed6d21f43b512a73853935f7a7b9bdf52)) + ## [2.10.0](https://github.com/googleapis/python-api-core/compare/v2.9.0...v2.10.0) (2022-09-02) diff --git a/google/api_core/version.py b/google/api_core/version.py index 13e710fc..8c486101 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.10.0" +__version__ = "2.10.1" From 4420055de88eecc03322130449a2dd90bf925a88 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 29 Sep 2022 15:29:31 -0400 Subject: [PATCH 112/126] fix(deps): require protobuf >= 3.20.2 (#455) * chore: exclude requirements.txt file from renovate-bot Source-Link: https://github.com/googleapis/synthtool/commit/f58d3135a2fab20e225d98741dbc06d57459b816 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 * update constraints files * fix(deps): require protobuf 3.20.2 Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/requirements.txt | 49 ++++++++++++++++++------------------- setup.py | 2 +- testing/constraints-3.7.txt | 2 +- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b8dcb4a4..3815c983 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 + digest: sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 385f2d4d..d15994ba 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -325,31 +325,30 @@ platformdirs==2.5.2 \ --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 # via virtualenv -protobuf==3.20.1 \ - --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ - --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ - --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ - --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ - --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ - --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ - --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ - --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ - --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ - --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ - --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ - --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ - --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ - --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ - --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ - --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ - --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ - --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ - --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ - --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ - --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ - --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ - --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ - --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 +protobuf==3.20.2 \ + --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ + --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ + --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ + --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ + --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ + --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ + --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ + --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ + --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ + --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ + --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ + --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ + --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ + --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ + --hash=sha256:9f876a69ca55aed879b43c295a328970306e8e80a263ec91cf6e9189243c613b \ + --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ + --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ + --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ + --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ + --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ + --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ + --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ + --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 # via # gcp-docuploader # gcp-releasetool diff --git a/setup.py b/setup.py index 2dd2a0cd..9dadc489 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.20.1, <5.0.0dev", + "protobuf >= 3.20.2, <5.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index fe671145..eab4c2e7 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,7 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 googleapis-common-protos==1.56.2 -protobuf==3.20.1 +protobuf==3.20.2 google-auth==1.25.0 requests==2.18.0 packaging==14.3 From e949364ce3a2c4c3cdb2658054d4793aa942d999 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 7 Oct 2022 20:36:48 -0400 Subject: [PATCH 113/126] fix(deps): allow protobuf 3.19.5 (#459) * fix(deps): allow protobuf 3.19.5 * explicitly exclude protobuf 4.21.0 * chore: use a compatible version of protobuf for testing --- setup.py | 2 +- testing/constraints-3.7.txt | 2 +- testing/constraints-3.8.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9dadc489..1e746c8e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.20.2, <5.0.0dev", + "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index eab4c2e7..d58c05b0 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,7 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 googleapis-common-protos==1.56.2 -protobuf==3.20.2 +protobuf==3.19.5 google-auth==1.25.0 requests==2.18.0 packaging==14.3 diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index 8d760bbd..1b5bb58e 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -1,2 +1,2 @@ googleapis-common-protos==1.56.3 -protobuf==4.21.5 \ No newline at end of file +protobuf==4.21.6 \ No newline at end of file From ddcc2499fa756f98268eb552edba6f25ddbe0c26 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 20:41:09 -0400 Subject: [PATCH 114/126] chore(main): release 2.10.2 (#456) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b417e98e..bf80f36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.10.2](https://github.com/googleapis/python-api-core/compare/v2.10.1...v2.10.2) (2022-10-08) + + +### Bug Fixes + +* **deps:** Allow protobuf 3.19.5 ([#459](https://github.com/googleapis/python-api-core/issues/459)) ([e949364](https://github.com/googleapis/python-api-core/commit/e949364ce3a2c4c3cdb2658054d4793aa942d999)) + ## [2.10.1](https://github.com/googleapis/python-api-core/compare/v2.10.0...v2.10.1) (2022-09-14) diff --git a/google/api_core/version.py b/google/api_core/version.py index 8c486101..ad66f162 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.10.1" +__version__ = "2.10.2" From 434253de16d9efdf984ddb64c409706cda1d5f82 Mon Sep 17 00:00:00 2001 From: Vadym Matsishevskyi <25311427+vam-google@users.noreply.github.com> Date: Thu, 10 Nov 2022 06:15:00 -0800 Subject: [PATCH 115/126] fix: Major refactoring of Polling, Retry and Timeout logic (#462) * fix: Major refactoring and fix for Polling, Retry and Timeout logic This is in response to https://freeman.vc/notes/aws-vs-gcp-reliability-is-wildly-different, which triggered an investigation of the whole Polling/Retry/Timeout behavior in Python GAPIC clients and revealed many fundamental flaws in its implementaiton. To properly describe the refactoring this PR does we need to stick to a rigorous terminology, as vague definitions of retries, timeouts, polling and related concepts seems to be the main source of the present bugs and overal confusion among both groups: users of the library and creators of the library. Please check the documentation of the `google.api_core.retry.Retry` class the `google.api_core.future.polling.Polling.result()` method for the proper definitions and context. Note, the overall semantics around Polling, Retry and Timeout remains quite confusing even after refactoring (although it is now more or less rigorously defined), but it was clean as I could make it while still maintaining backward compatibility of the whole library. The quick summary of the changes in this PR: 1) Properly define and fix the application of Deadline and Timeout concepts. Please check the updated documentation for the `google.api_core.retry.Retry` class for the actual definitions. Originally the `deadline` has been used to represent timeouts conflating the two concepts. As result this PR replaces `deadline` arguments with `timeout` ones in as backward-compatible manner as possible (i.e. backward compatible in all practical applications). 2) Properly define RPC Timeout, Retry Timeout and Pollint Timeout and how a generic Timeout concept (aka Logical Timeout) is mapped to one of those depending on the context. Please check `google.api_core.retry.Retry` class documentation for details. 3) Properly define and fix the application of Retry and Polling concepts. Please check the updated documentation for `google.api_core.future.polling.PollingFuture.result()` for details. 4) Separate `retry` and `polling` configurations for Polling future, as these are two different concepts (although both operating on `Retry` class). Originally both retry and polling configurations were controlled by a single `retry` parameter, merging configuration regarding how "rpc error responses" and how "operation not completed" responses are supposed to be handled. 5) For the following config properties - `Retry (including `Retry Timeout`), `Polling` (including `Polling Timeout`) and `RPC Timeout` - fix and properly define how each of the above properties gets configured and which config gets precedence in case of a conflict (check `PollingFuture.result()` method documentation for details). Each of those properties can be specified as follows: directly provided by the user for each call, specified during gapic generation time from config values in `grpc_service_config.json` file (for Retry and RPC Timeout) and `gapic.yaml` file (for Polling), or be provided as a hard-coded basic default values in python-api-core library itself. 6) Fix the per-call polling config propagation logic (the polling/retry configs supplied to `PollingFuture.result()` used to be ignored for actual call). 7) Deprecate the usage of `deadline` terminology in the whole library and backward-compatibly replace it with timeout. This is essential as what has been called "deadline" in this library was actually "timeout" as it is defined in `google.api_core.retry.Retry` class documentation. 8) Deprecate `ExponentialTimeout`, `ConstantTimeout` and related logic as those are outdated concepts and are not consistent with the other GAPIC Languages. Replace it with `TimeToDeadlineTimeout` to be consistent with how the rest of the languages do it. 9) Deprecate `google.api_core.operations_v1.config` as it is an outdated concept and self-inconsistent (as all gapic clients provide configuraiton in code). The configs are directly provided in code instead. 10) Switch randomized delay calculation from `delay` being treated as expected value for randomized_delay to `delay` being treated as maximum value for `randomized_delay` (i.e. the new expected valud for `randomized_delay` is `delay / 2`). See the `exponential_sleep_generator()` method implementation for details. This is needed to make Python implementation of retries and polling exponential backoff consistent with the rest of GAPIC languages. Also fix the uncontrollable growth of `delay` value (since it is a subject of exponential growth, the `delay` value was quickly reaching "infinity" value, and the whole thing was not failing simply due to python being a very forgiving language which forgives multiplying "infinity" by a number (`inf * number = inf`) binstead of simply overflowing to a (most likely) negative number). 11) Fix url construction in `OperationsRestTransport`. Without this fix the polling logic for REST transport was completely broken (is not affecting Compute client, as that one has custom LRO). 12) Las but not least: change the default values for Polling logic to be the following: `initial=1.0` (same as before), `maximum=20.0` (was `60`), `multiplier=1.5` (was `2.0`), `timeout=900` (was `120`, but due to timeout resolution logic was actually None (i.e. infinity)). This, in conjunction with changed calculation of randomized delay (i.e. its expected value now being `delay / 2`) overall makes polling logic much less aggressive in terms of increasing delays between each polling iteration, making LRO return much earlier for users on average, but still keeping a healthy balance between strain put on both client and server by polling and responsiveness of LROs for user. *The design doc summarising all the changes and reasons for them is in progress. * fix ci failures (mainly sphinx errors) * remove unused code * fix typo * Pin pytest version to <7.2.0 * reformat code * address pr feedback * address PR feedback * address pr feedback * Update google/api_core/future/polling.py Co-authored-by: Victor Chudnovsky * Apply documentation suggestions from code review Co-authored-by: Victor Chudnovsky * Address PR feedback Co-authored-by: Victor Chudnovsky --- .github/workflows/lint.yml | 2 +- .github/workflows/mypy.yml | 2 +- google/api_core/extended_operation.py | 28 ++- google/api_core/future/async_future.py | 2 +- google/api_core/future/polling.py | 200 +++++++++++++++--- google/api_core/gapic_v1/config.py | 9 + google/api_core/gapic_v1/method.py | 69 ++---- google/api_core/grpc_helpers.py | 4 +- google/api_core/operation.py | 22 +- .../operations_v1/operations_async_client.py | 39 ++-- .../operations_v1/operations_client.py | 39 ++-- .../operations_v1/operations_client_config.py | 1 + .../api_core/operations_v1/transports/rest.py | 44 +++- google/api_core/retry.py | 166 +++++++++++---- google/api_core/retry_async.py | 58 +++-- google/api_core/timeout.py | 68 +++++- noxfile.py | 19 +- tests/asyncio/gapic/test_method_async.py | 35 --- .../test_operations_async_client.py | 2 +- tests/asyncio/test_grpc_helpers_async.py | 4 +- tests/asyncio/test_operation_async.py | 2 +- tests/asyncio/test_retry_async.py | 13 +- tests/unit/future/test_polling.py | 74 ++++--- tests/unit/gapic/test_method.py | 52 ----- .../operations_v1/test_operations_client.py | 7 +- .../test_operations_rest_client.py | 37 ++-- tests/unit/test_bidi.py | 2 +- tests/unit/test_client_info.py | 6 +- tests/unit/test_exceptions.py | 2 +- tests/unit/test_general_helpers.py | 13 ++ tests/unit/test_grpc_helpers.py | 28 +-- tests/unit/test_operation.py | 2 +- tests/unit/test_retry.py | 29 ++- tests/unit/test_timeout.py | 125 +++++++++-- 34 files changed, 789 insertions(+), 416 deletions(-) create mode 100644 tests/unit/test_general_helpers.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eae860a2..d2aee5b7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index f08164f6..a505525d 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py index cabae107..092eba4e 100644 --- a/google/api_core/extended_operation.py +++ b/google/api_core/extended_operation.py @@ -50,10 +50,13 @@ class ExtendedOperation(polling.PollingFuture): refresh (Callable[[], type(extended_operation)]): A callable that returns the latest state of the operation. cancel (Callable[[], None]): A callable that tries to cancel the operation. - retry: Optional(google.api_core.retry.Retry): The retry configuration used - when polling. This can be used to control how often :meth:`done` - is polled. Regardless of the retry's ``deadline``, it will be - overridden by the ``timeout`` argument to :meth:`result`. + polling Optional(google.api_core.retry.Retry): The configuration used + for polling. This can be used to control how often :meth:`done` + is polled. If the ``timeout`` argument to :meth:`result` is + specified it will override the ``polling.timeout`` property. + retry Optional(google.api_core.retry.Retry): DEPRECATED use ``polling`` + instead. If specified it will override ``polling`` parameter to + maintain backward compatibility. Note: Most long-running API methods use google.api_core.operation.Operation This class is a wrapper for a subset of methods that use alternative @@ -68,9 +71,14 @@ class ExtendedOperation(polling.PollingFuture): """ def __init__( - self, extended_operation, refresh, cancel, retry=polling.DEFAULT_RETRY + self, + extended_operation, + refresh, + cancel, + polling=polling.DEFAULT_POLLING, + **kwargs, ): - super().__init__(retry=retry) + super().__init__(polling=polling, **kwargs) self._extended_operation = extended_operation self._refresh = refresh self._cancel = cancel @@ -114,7 +122,7 @@ def error_message(self): def __getattr__(self, name): return getattr(self._extended_operation, name) - def done(self, retry=polling.DEFAULT_RETRY): + def done(self, retry=None): self._refresh_and_update(retry) return self._extended_operation.done @@ -137,9 +145,11 @@ def cancelled(self): self._refresh_and_update() return self._extended_operation.done - def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): + def _refresh_and_update(self, retry=None): if not self._extended_operation.done: - self._extended_operation = self._refresh(retry=retry) + self._extended_operation = ( + self._refresh(retry=retry) if retry else self._refresh() + ) self._handle_refreshed_operation() def _handle_refreshed_operation(self): diff --git a/google/api_core/future/async_future.py b/google/api_core/future/async_future.py index 88c183f9..325ee9cd 100644 --- a/google/api_core/future/async_future.py +++ b/google/api_core/future/async_future.py @@ -95,7 +95,7 @@ async def _blocking_poll(self, timeout=None): if self._future.done(): return - retry_ = self._retry.with_deadline(timeout) + retry_ = self._retry.with_timeout(timeout) try: await retry_(self._done_or_raise)() diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 02e680f6..6e6aa5d4 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -18,7 +18,7 @@ import concurrent.futures from google.api_core import exceptions -from google.api_core import retry +from google.api_core import retry as retries from google.api_core.future import _helpers from google.api_core.future import base @@ -29,14 +29,37 @@ class _OperationNotComplete(Exception): pass -RETRY_PREDICATE = retry.if_exception_type( +# DEPRECATED as it conflates RPC retry and polling concepts into one. +# Use POLLING_PREDICATE instead to configure polling. +RETRY_PREDICATE = retries.if_exception_type( _OperationNotComplete, exceptions.TooManyRequests, exceptions.InternalServerError, exceptions.BadGateway, exceptions.ServiceUnavailable, ) -DEFAULT_RETRY = retry.Retry(predicate=RETRY_PREDICATE) + +# DEPRECATED: use DEFAULT_POLLING to configure LRO polling logic. Construct +# Retry object using its default values as a baseline for any custom retry logic +# (not to be confused with polling logic). +DEFAULT_RETRY = retries.Retry(predicate=RETRY_PREDICATE) + +# POLLING_PREDICATE is supposed to poll only on _OperationNotComplete. +# Any RPC-specific errors (like ServiceUnavailable) will be handled +# by retry logic (not to be confused with polling logic) which is triggered for +# every polling RPC independently of polling logic but within its context. +POLLING_PREDICATE = retries.if_exception_type( + _OperationNotComplete, +) + +# Default polling configuration +DEFAULT_POLLING = retries.Retry( + predicate=POLLING_PREDICATE, + initial=1.0, # seconds + maximum=20.0, # seconds + multiplier=1.5, + timeout=900, # seconds +) class PollingFuture(base.Future): @@ -45,21 +68,29 @@ class PollingFuture(base.Future): The :meth:`done` method should be implemented by subclasses. The polling behavior will repeatedly call ``done`` until it returns True. + The actuall polling logic is encapsulated in :meth:`result` method. See + documentation for that method for details on how polling works. + .. note:: Privacy here is intended to prevent the final class from overexposing, not to prevent subclasses from accessing methods. Args: - retry (google.api_core.retry.Retry): The retry configuration used - when polling. This can be used to control how often :meth:`done` - is polled. Regardless of the retry's ``deadline``, it will be - overridden by the ``timeout`` argument to :meth:`result`. + polling (google.api_core.retry.Retry): The configuration used for polling. + This parameter controls how often :meth:`done` is polled. If the + ``timeout`` argument is specified in :meth:`result` method it will + override the ``polling.timeout`` property. + retry (google.api_core.retry.Retry): DEPRECATED use ``polling`` instead. + If set, it will override ``polling`` paremeter for backward + compatibility. """ - def __init__(self, retry=DEFAULT_RETRY): + _DEFAULT_VALUE = object() + + def __init__(self, polling=DEFAULT_POLLING, **kwargs): super(PollingFuture, self).__init__() - self._retry = retry + self._polling = kwargs.get("retry", polling) self._result = None self._exception = None self._result_set = False @@ -69,11 +100,13 @@ def __init__(self, retry=DEFAULT_RETRY): self._done_callbacks = [] @abc.abstractmethod - def done(self, retry=DEFAULT_RETRY): + def done(self, retry=None): """Checks to see if the operation is complete. Args: - retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. + retry (google.api_core.retry.Retry): (Optional) How to retry the + polling RPC (to not be confused with polling configuration. See + the documentation for :meth:`result` for details). Returns: bool: True if the operation is complete, False otherwise. @@ -81,45 +114,136 @@ def done(self, retry=DEFAULT_RETRY): # pylint: disable=redundant-returns-doc, missing-raises-doc raise NotImplementedError() - def _done_or_raise(self, retry=DEFAULT_RETRY): + def _done_or_raise(self, retry=None): """Check if the future is done and raise if it's not.""" - kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} - - if not self.done(**kwargs): + if not self.done(retry=retry): raise _OperationNotComplete() def running(self): """True if the operation is currently running.""" return not self.done() - def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): - """Poll and wait for the Future to be resolved. + def _blocking_poll(self, timeout=_DEFAULT_VALUE, retry=None, polling=None): + """Poll and wait for the Future to be resolved.""" - Args: - timeout (int): - How long (in seconds) to wait for the operation to complete. - If None, wait indefinitely. - """ if self._result_set: return - retry_ = self._retry.with_deadline(timeout) + polling = polling or self._polling + if timeout is not PollingFuture._DEFAULT_VALUE: + polling = polling.with_timeout(timeout) try: - kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} - retry_(self._done_or_raise)(**kwargs) + polling(self._done_or_raise)(retry=retry) except exceptions.RetryError: raise concurrent.futures.TimeoutError( - "Operation did not complete within the designated " "timeout." + f"Operation did not complete within the designated timeout of " + f"{polling.timeout} seconds." ) - def result(self, timeout=None, retry=DEFAULT_RETRY): - """Get the result of the operation, blocking if necessary. + def result(self, timeout=_DEFAULT_VALUE, retry=None, polling=None): + """Get the result of the operation. + + This method will poll for operation status periodically, blocking if + necessary. If you just want to make sure that this method does not block + for more than X seconds and you do not care about the nitty-gritty of + how this method operates, just call it with ``result(timeout=X)``. The + other parameters are for advanced use only. + + Every call to this method is controlled by the following three + parameters, each of which has a specific, distinct role, even though all three + may look very similar: ``timeout``, ``retry`` and ``polling``. In most + cases users do not need to specify any custom values for any of these + parameters and may simply rely on default ones instead. + + If you choose to specify custom parameters, please make sure you've + read the documentation below carefully. + + First, please check :class:`google.api_core.retry.Retry` + class documentation for the proper definition of timeout and deadline + terms and for the definition the three different types of timeouts. + This class operates in terms of Retry Timeout and Polling Timeout. It + does not let customizing RPC timeout and the user is expected to rely on + default behavior for it. + + The roles of each argument of this method are as follows: + + ``timeout`` (int): (Optional) The Polling Timeout as defined in + :class:`google.api_core.retry.Retry`. If the operation does not complete + within this timeout an exception will be thrown. This parameter affects + neither Retry Timeout nor RPC Timeout. + + ``retry`` (google.api_core.retry.Retry): (Optional) How to retry the + polling RPC. The ``retry.timeout`` property of this parameter is the + Retry Timeout as defined in :class:`google.api_core.retry.Retry`. + This parameter defines ONLY how the polling RPC call is retried + (i.e. what to do if the RPC we used for polling returned an error). It + does NOT define how the polling is done (i.e. how frequently and for + how long to call the polling RPC); use the ``polling`` parameter for that. + If a polling RPC throws and error and retrying it fails, the whole + future fails with the corresponding exception. If you want to tune which + server response error codes are not fatal for operation polling, use this + parameter to control that (``retry.predicate`` in particular). + + ``polling`` (google.api_core.retry.Retry): (Optional) How often and + for how long to call the polling RPC periodically (i.e. what to do if + a polling rpc returned successfully but its returned result indicates + that the long running operation is not completed yet, so we need to + check it again at some point in future). This parameter does NOT define + how to retry each individual polling RPC in case of an error; use the + ``retry`` parameter for that. The ``polling.timeout`` of this parameter + is Polling Timeout as defined in as defined in + :class:`google.api_core.retry.Retry`. + + For each of the arguments, there are also default values in place, which + will be used if a user does not specify their own. The default values + for the three parameters are not to be confused with the default values + for the corresponding arguments in this method (those serve as "not set" + markers for the resolution logic). + + If ``timeout`` is provided (i.e.``timeout is not _DEFAULT VALUE``; note + the ``None`` value means "infinite timeout"), it will be used to control + the actual Polling Timeout. Otherwise, the ``polling.timeout`` value + will be used instead (see below for how the ``polling`` config itself + gets resolved). In other words, this parameter effectively overrides + the ``polling.timeout`` value if specified. This is so to preserve + backward compatibility. + + If ``retry`` is provided (i.e. ``retry is not None``) it will be used to + control retry behavior for the polling RPC and the ``retry.timeout`` + will determine the Retry Timeout. If not provided, the + polling RPC will be called with whichever default retry config was + specified for the polling RPC at the moment of the construction of the + polling RPC's client. For example, if the polling RPC is + ``operations_client.get_operation()``, the ``retry`` parameter will be + controlling its retry behavior (not polling behavior) and, if not + specified, that specific method (``operations_client.get_operation()``) + will be retried according to the default retry config provided during + creation of ``operations_client`` client instead. This argument exists + mainly for backward compatibility; users are very unlikely to ever need + to set this parameter explicitly. + + If ``polling`` is provided (i.e. ``polling is not None``), it will be used + to controll the overall polling behavior and ``polling.timeout`` will + controll Polling Timeout unless it is overridden by ``timeout`` parameter + as described above. If not provided, the``polling`` parameter specified + during construction of this future (the ``polling`` argument in the + constructor) will be used instead. Note: since the ``timeout`` argument may + override ``polling.timeout`` value, this parameter should be viewed as + coupled with the ``timeout`` parameter as described above. Args: - timeout (int): - How long (in seconds) to wait for the operation to complete. - If None, wait indefinitely. + timeout (int): (Optional) How long (in seconds) to wait for the + operation to complete. If None, wait indefinitely. + retry (google.api_core.retry.Retry): (Optional) How to retry the + polling RPC. This defines ONLY how the polling RPC call is + retried (i.e. what to do if the RPC we used for polling returned + an error). It does NOT define how the polling is done (i.e. how + frequently and for how long to call the polling RPC). + polling (google.api_core.retry.Retry): (Optional) How often and + for how long to call polling RPC periodically. This parameter + does NOT define how to retry each individual polling RPC call + (use the ``retry`` parameter for that). Returns: google.protobuf.Message: The Operation's result. @@ -128,8 +252,8 @@ def result(self, timeout=None, retry=DEFAULT_RETRY): google.api_core.GoogleAPICallError: If the operation errors or if the timeout is reached before the operation completes. """ - kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} - self._blocking_poll(timeout=timeout, **kwargs) + + self._blocking_poll(timeout=timeout, retry=retry, polling=polling) if self._exception is not None: # pylint: disable=raising-bad-type @@ -138,12 +262,18 @@ def result(self, timeout=None, retry=DEFAULT_RETRY): return self._result - def exception(self, timeout=None): + def exception(self, timeout=_DEFAULT_VALUE): """Get the exception from the operation, blocking if necessary. + See the documentation for the :meth:`result` method for details on how + this method operates, as both ``result`` and this method rely on the + exact same polling logic. The only difference is that this method does + not accept ``retry`` and ``polling`` arguments but relies on the default ones + instead. + Args: timeout (int): How long to wait for the operation to complete. - If None, wait indefinitely. + If None, wait indefinitely. Returns: Optional[google.api_core.GoogleAPICallError]: The operation's diff --git a/google/api_core/gapic_v1/config.py b/google/api_core/gapic_v1/config.py index 9c722871..36b50d9f 100644 --- a/google/api_core/gapic_v1/config.py +++ b/google/api_core/gapic_v1/config.py @@ -33,6 +33,9 @@ def _exception_class_for_grpc_status_name(name): """Returns the Google API exception class for a gRPC error code name. + DEPRECATED: use ``exceptions.exception_class_for_grpc_status`` method + directly instead. + Args: name (str): The name of the gRPC status code, for example, ``UNAVAILABLE``. @@ -47,6 +50,8 @@ def _exception_class_for_grpc_status_name(name): def _retry_from_retry_config(retry_params, retry_codes, retry_impl=retry.Retry): """Creates a Retry object given a gapic retry configuration. + DEPRECATED: instantiate retry and timeout classes directly instead. + Args: retry_params (dict): The retry parameter values, for example:: @@ -81,6 +86,8 @@ def _retry_from_retry_config(retry_params, retry_codes, retry_impl=retry.Retry): def _timeout_from_retry_config(retry_params): """Creates a ExponentialTimeout object given a gapic retry configuration. + DEPRECATED: instantiate retry and timeout classes directly instead. + Args: retry_params (dict): The retry parameter values, for example:: @@ -113,6 +120,8 @@ def parse_method_configs(interface_config, retry_impl=retry.Retry): """Creates default retry and timeout objects for each method in a gapic interface config. + DEPRECATED: instantiate retry and timeout classes directly instead. + Args: interface_config (Mapping): The interface config section of the full gapic library config. For example, If the full configuration has diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py index 73c8d4bc..0c1624a3 100644 --- a/google/api_core/gapic_v1/method.py +++ b/google/api_core/gapic_v1/method.py @@ -22,8 +22,8 @@ import functools from google.api_core import grpc_helpers -from google.api_core import timeout from google.api_core.gapic_v1 import client_info +from google.api_core.timeout import TimeToDeadlineTimeout USE_DEFAULT_METADATA = object() @@ -52,55 +52,14 @@ def _apply_decorators(func, decorators): ``decorators`` may contain items that are ``None`` or ``False`` which will be ignored. """ - decorators = filter(_is_not_none_or_false, reversed(decorators)) + filtered_decorators = filter(_is_not_none_or_false, reversed(decorators)) - for decorator in decorators: + for decorator in filtered_decorators: func = decorator(func) return func -def _determine_timeout(default_timeout, specified_timeout, retry): - """Determines how timeout should be applied to a wrapped method. - - Args: - default_timeout (Optional[Timeout]): The default timeout specified - at method creation time. - specified_timeout (Optional[Timeout]): The timeout specified at - invocation time. If :attr:`DEFAULT`, this will be set to - the ``default_timeout``. - retry (Optional[Retry]): The retry specified at invocation time. - - Returns: - Optional[Timeout]: The timeout to apply to the method or ``None``. - """ - # If timeout is specified as a number instead of a Timeout instance, - # convert it to a ConstantTimeout. - if isinstance(specified_timeout, (int, float)): - specified_timeout = timeout.ConstantTimeout(specified_timeout) - if isinstance(default_timeout, (int, float)): - default_timeout = timeout.ConstantTimeout(default_timeout) - - if specified_timeout is DEFAULT: - specified_timeout = default_timeout - - if specified_timeout is default_timeout: - # If timeout is the default and the default timeout is exponential and - # a non-default retry is specified, make sure the timeout's deadline - # matches the retry's. This handles the case where the user leaves - # the timeout default but specifies a lower deadline via the retry. - if ( - retry - and retry is not DEFAULT - and isinstance(default_timeout, timeout.ExponentialTimeout) - ): - return default_timeout.with_deadline(retry._deadline) - else: - return default_timeout - - return specified_timeout - - class _GapicCallable(object): """Callable that applies retry, timeout, and metadata logic. @@ -108,9 +67,11 @@ class _GapicCallable(object): target (Callable): The low-level RPC method. retry (google.api_core.retry.Retry): The default retry for the callable. If ``None``, this callable will not retry by default - timeout (google.api_core.timeout.Timeout): The default timeout - for the callable. If ``None``, this callable will not specify - a timeout argument to the low-level RPC method by default. + timeout (google.api_core.timeout.Timeout): The default timeout for the + callable (i.e. duration of time within which an RPC must terminate + after its start, not to be confused with deadline). If ``None``, + this callable will not specify a timeout argument to the low-level + RPC method. metadata (Sequence[Tuple[str, str]]): Additional metadata that is provided to the RPC method on every invocation. This is merged with any metadata specified during invocation. If ``None``, no @@ -125,18 +86,16 @@ def __init__(self, target, retry, timeout, metadata=None): def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs): """Invoke the low-level RPC with retry, timeout, and metadata.""" - timeout = _determine_timeout( - self._timeout, - timeout, - # Use only the invocation-specified retry only for this, as we only - # want to adjust the timeout deadline if the *user* specified - # a different retry. - retry, - ) if retry is DEFAULT: retry = self._retry + if timeout is DEFAULT: + timeout = self._timeout + + if isinstance(timeout, (int, float)): + timeout = TimeToDeadlineTimeout(timeout=timeout) + # Apply all applicable decorators. wrapped_func = _apply_decorators(self._target, [retry, timeout]) diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index bf04ae4c..102dc0a0 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -30,7 +30,7 @@ PROTOBUF_VERSION = google.protobuf.__version__ # The grpcio-gcp package only has support for protobuf < 4 -if PROTOBUF_VERSION[0:2] == "3.": +if PROTOBUF_VERSION[0:2] == "3.": # pragma: NO COVER try: import grpc_gcp @@ -318,7 +318,7 @@ def create_channel( default_host=default_host, ) - if HAS_GRPC_GCP: + if HAS_GRPC_GCP: # pragma: NO COVER return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) return grpc.secure_channel(target, composite_credentials, **kwargs) diff --git a/google/api_core/operation.py b/google/api_core/operation.py index b17f753b..90cbdc99 100644 --- a/google/api_core/operation.py +++ b/google/api_core/operation.py @@ -61,10 +61,13 @@ class Operation(polling.PollingFuture): result. metadata_type (func:`type`): The protobuf type for the operation's metadata. - retry (google.api_core.retry.Retry): The retry configuration used - when polling. This can be used to control how often :meth:`done` - is polled. Regardless of the retry's ``deadline``, it will be - overridden by the ``timeout`` argument to :meth:`result`. + polling (google.api_core.retry.Retry): The configuration used for polling. + This parameter controls how often :meth:`done` is polled. If the + ``timeout`` argument is specified in the :meth:`result` method, it will + override the ``polling.timeout`` property. + retry (google.api_core.retry.Retry): DEPRECATED: use ``polling`` instead. + If specified it will override ``polling`` parameter to maintain + backward compatibility. """ def __init__( @@ -74,9 +77,10 @@ def __init__( cancel, result_type, metadata_type=None, - retry=polling.DEFAULT_RETRY, + polling=polling.DEFAULT_POLLING, + **kwargs ): - super(Operation, self).__init__(retry=retry) + super(Operation, self).__init__(polling=polling, **kwargs) self._operation = operation self._refresh = refresh self._cancel = cancel @@ -146,7 +150,7 @@ def _set_result_from_operation(self): ) self.set_exception(exception) - def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): + def _refresh_and_update(self, retry=None): """Refresh the operation and update the result if needed. Args: @@ -155,10 +159,10 @@ def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): # If the currently cached operation is done, no need to make another # RPC as it will not change once done. if not self._operation.done: - self._operation = self._refresh(retry=retry) + self._operation = self._refresh(retry=retry) if retry else self._refresh() self._set_result_from_operation() - def done(self, retry=polling.DEFAULT_RETRY): + def done(self, retry=None): """Checks to see if the operation is complete. Args: diff --git a/google/api_core/operations_v1/operations_async_client.py b/google/api_core/operations_v1/operations_async_client.py index 5a5e5562..81c4513c 100644 --- a/google/api_core/operations_v1/operations_async_client.py +++ b/google/api_core/operations_v1/operations_async_client.py @@ -24,8 +24,10 @@ import functools +from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1, page_iterator_async -from google.api_core.operations_v1 import operations_client_config +from google.api_core import retry as retries +from google.api_core import timeout as timeouts from google.longrunning import operations_pb2 @@ -41,39 +43,44 @@ class OperationsAsyncClient: the default configuration is used. """ - def __init__(self, channel, client_config=operations_client_config.config): + def __init__(self, channel, client_config=None): # Create the gRPC client stub with gRPC AsyncIO channel. self.operations_stub = operations_pb2.OperationsStub(channel) - # Create all wrapped methods using the interface configuration. - # The interface config contains all of the default settings for retry - # and timeout for each RPC method. - interfaces = client_config["interfaces"] - interface_config = interfaces["google.longrunning.Operations"] - method_configs = gapic_v1.config_async.parse_method_configs(interface_config) + default_retry = retries.Retry( + initial=0.1, # seconds + maximum=60.0, # seconds + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + timeout=600.0, # seconds + ) + default_timeout = timeouts.TimeToDeadlineTimeout(timeout=600.0) self._get_operation = gapic_v1.method_async.wrap_method( self.operations_stub.GetOperation, - default_retry=method_configs["GetOperation"].retry, - default_timeout=method_configs["GetOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._list_operations = gapic_v1.method_async.wrap_method( self.operations_stub.ListOperations, - default_retry=method_configs["ListOperations"].retry, - default_timeout=method_configs["ListOperations"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._cancel_operation = gapic_v1.method_async.wrap_method( self.operations_stub.CancelOperation, - default_retry=method_configs["CancelOperation"].retry, - default_timeout=method_configs["CancelOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._delete_operation = gapic_v1.method_async.wrap_method( self.operations_stub.DeleteOperation, - default_retry=method_configs["DeleteOperation"].retry, - default_timeout=method_configs["DeleteOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) async def get_operation( diff --git a/google/api_core/operations_v1/operations_client.py b/google/api_core/operations_v1/operations_client.py index e48eac01..3ddd3c47 100644 --- a/google/api_core/operations_v1/operations_client.py +++ b/google/api_core/operations_v1/operations_client.py @@ -37,9 +37,11 @@ import functools +from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 from google.api_core import page_iterator -from google.api_core.operations_v1 import operations_client_config +from google.api_core import retry as retries +from google.api_core import timeout as timeouts from google.longrunning import operations_pb2 @@ -54,39 +56,44 @@ class OperationsClient(object): the default configuration is used. """ - def __init__(self, channel, client_config=operations_client_config.config): + def __init__(self, channel, client_config=None): # Create the gRPC client stub. self.operations_stub = operations_pb2.OperationsStub(channel) - # Create all wrapped methods using the interface configuration. - # The interface config contains all of the default settings for retry - # and timeout for each RPC method. - interfaces = client_config["interfaces"] - interface_config = interfaces["google.longrunning.Operations"] - method_configs = gapic_v1.config.parse_method_configs(interface_config) + default_retry = retries.Retry( + initial=0.1, # seconds + maximum=60.0, # seconds + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + timeout=600.0, # seconds + ) + default_timeout = timeouts.TimeToDeadlineTimeout(timeout=600.0) self._get_operation = gapic_v1.method.wrap_method( self.operations_stub.GetOperation, - default_retry=method_configs["GetOperation"].retry, - default_timeout=method_configs["GetOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._list_operations = gapic_v1.method.wrap_method( self.operations_stub.ListOperations, - default_retry=method_configs["ListOperations"].retry, - default_timeout=method_configs["ListOperations"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._cancel_operation = gapic_v1.method.wrap_method( self.operations_stub.CancelOperation, - default_retry=method_configs["CancelOperation"].retry, - default_timeout=method_configs["CancelOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) self._delete_operation = gapic_v1.method.wrap_method( self.operations_stub.DeleteOperation, - default_retry=method_configs["DeleteOperation"].retry, - default_timeout=method_configs["DeleteOperation"].timeout, + default_retry=default_retry, + default_timeout=default_timeout, ) # Service calls diff --git a/google/api_core/operations_v1/operations_client_config.py b/google/api_core/operations_v1/operations_client_config.py index 6cf95753..70cfd70a 100644 --- a/google/api_core/operations_v1/operations_client_config.py +++ b/google/api_core/operations_v1/operations_client_config.py @@ -14,6 +14,7 @@ """gapic configuration for the googe.longrunning.operations client.""" +# DEPRECATED: retry and timeout classes are instantiated directly config = { "interfaces": { "google.longrunning.Operations": { diff --git a/google/api_core/operations_v1/transports/rest.py b/google/api_core/operations_v1/transports/rest.py index 27ed7661..bb6cd99c 100644 --- a/google/api_core/operations_v1/transports/rest.py +++ b/google/api_core/operations_v1/transports/rest.py @@ -14,6 +14,7 @@ # limitations under the License. # +import re from typing import Callable, Dict, Optional, Sequence, Tuple, Union from requests import __version__ as requests_version @@ -73,6 +74,7 @@ def __init__( always_use_jwt_access: Optional[bool] = False, url_scheme: str = "https", http_options: Optional[Dict] = None, + path_prefix: str = "v1", ) -> None: """Instantiate the transport. @@ -108,12 +110,24 @@ def __init__( http_options: a dictionary of http_options for transcoding, to override the defaults from operatons.proto. Each method has an entry with the corresponding http rules as value. + path_prefix: path prefix (usually represents API version). Set to + "v1" by default. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object + maybe_url_match = re.match("^(?Phttp(?:s)?://)?(?P.*)$", host) + if maybe_url_match is None: + raise ValueError( + f"Unexpected hostname structure: {host}" + ) # pragma: NO COVER + + url_match_items = maybe_url_match.groupdict() + + host = f"{url_scheme}://{host}" if not url_match_items["scheme"] else host + super().__init__( host=host, credentials=credentials, @@ -127,6 +141,7 @@ def __init__( self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) self._http_options = http_options or {} + self._path_prefix = path_prefix def _list_operations( self, @@ -157,7 +172,10 @@ def _list_operations( """ http_options = [ - {"method": "get", "uri": "/v1/{name=operations}"}, + { + "method": "get", + "uri": "/{}/{{name=**}}/operations".format(self._path_prefix), + }, ] if "google.longrunning.Operations.ListOperations" in self._http_options: http_options = self._http_options[ @@ -188,7 +206,7 @@ def _list_operations( headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( - "https://{host}{uri}".format(host=self._host, uri=uri), + "{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), @@ -234,7 +252,10 @@ def _get_operation( """ http_options = [ - {"method": "get", "uri": "/v1/{name=operations/**}"}, + { + "method": "get", + "uri": "/{}/{{name=**/operations/*}}".format(self._path_prefix), + }, ] if "google.longrunning.Operations.GetOperation" in self._http_options: http_options = self._http_options[ @@ -265,7 +286,7 @@ def _get_operation( headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( - "https://{host}{uri}".format(host=self._host, uri=uri), + "{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), @@ -304,7 +325,10 @@ def _delete_operation( """ http_options = [ - {"method": "delete", "uri": "/v1/{name=operations/**}"}, + { + "method": "delete", + "uri": "/{}/{{name=**/operations/*}}".format(self._path_prefix), + }, ] if "google.longrunning.Operations.DeleteOperation" in self._http_options: http_options = self._http_options[ @@ -335,7 +359,7 @@ def _delete_operation( headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( - "https://{host}{uri}".format(host=self._host, uri=uri), + "{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), @@ -371,7 +395,11 @@ def _cancel_operation( """ http_options = [ - {"method": "post", "uri": "/v1/{name=operations/**}:cancel", "body": "*"}, + { + "method": "post", + "uri": "/{}/{{name=**/operations/*}}:cancel".format(self._path_prefix), + "body": "*", + }, ] if "google.longrunning.Operations.CancelOperation" in self._http_options: http_options = self._http_options[ @@ -411,7 +439,7 @@ def _cancel_operation( headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( - "https://{host}{uri}".format(host=self._host, uri=uri), + "{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), diff --git a/google/api_core/retry.py b/google/api_core/retry.py index ce496937..f9207a12 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -139,15 +139,15 @@ def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULT Yields: float: successive sleep intervals. """ - delay = initial + delay = min(initial, maximum) while True: - # Introduce jitter by yielding a delay that is uniformly distributed - # to average out to the delay time. - yield min(random.uniform(0.0, delay * 2.0), maximum) - delay = delay * multiplier + yield random.uniform(0.0, delay) + delay = min(delay * multiplier, maximum) -def retry_target(target, predicate, sleep_generator, deadline, on_error=None): +def retry_target( + target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs +): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the @@ -161,12 +161,12 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. - deadline (float): How long to keep retrying the target. The last sleep - period is shortened as necessary, so that the last retry runs at - ``deadline`` (and not considerably beyond it). + timeout (float): How long to keep retrying the target. on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. + deadline (float): DEPRECATED: use ``timeout`` instead. For backward + compatibility, if specified it will override ``timeout`` parameter. Returns: Any: the return value of the target function. @@ -176,12 +176,13 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ - if deadline is not None: - deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta( - seconds=deadline - ) + + timeout = kwargs.get("deadline", timeout) + + if timeout is not None: + deadline = datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout) else: - deadline_datetime = None + deadline = None last_exc = None @@ -198,19 +199,17 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if on_error is not None: on_error(exc) - now = datetime_helpers.utcnow() - - if deadline_datetime is not None: - if deadline_datetime <= now: + if deadline is not None: + next_attempt_time = datetime_helpers.utcnow() + datetime.timedelta( + seconds=sleep + ) + if deadline < next_attempt_time: raise exceptions.RetryError( "Deadline of {:.1f}s exceeded while calling target function".format( - deadline + timeout ), last_exc, ) from last_exc - else: - time_to_deadline = (deadline_datetime - now).total_seconds() - sleep = min(time_to_deadline, sleep) _LOGGER.debug( "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep) @@ -223,12 +222,77 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): class Retry(object): """Exponential retry decorator. - This class is a decorator used to add exponential back-off retry behavior - to an RPC call. + This class is a decorator used to add retry or polling behavior to an RPC + call. Although the default behavior is to retry transient API errors, a different predicate can be provided to retry other exceptions. + There two important concepts that retry/polling behavior may operate on, + Deadline and Timeout, which need to be properly defined for the correct + usage of this class and the rest of the library. + + Deadline: a fixed point in time by which a certain operation must + terminate. For example, if a certain operation has a deadline + "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an + error) by that time, regardless of when it was started or whether it + was started at all. + + Timeout: the maximum duration of time after which a certain operation + must terminate (successfully or with an error). The countdown begins right + after an operation was started. For example, if an operation was started at + 09:24:00 with timeout of 75 seconds, it must terminate no later than + 09:25:15. + + Unfortunately, in the past this class (and the api-core library as a whole) has not been + properly distinguishing the concepts of "timeout" and "deadline", and the + ``deadline`` parameter has meant ``timeout``. That is why + ``deadline`` has been deprecated and ``timeout`` should be used instead. If the + ``deadline`` parameter is set, it will override the ``timeout`` parameter. In other words, + ``retry.deadline`` should be treated as just a deprecated alias for + ``retry.timeout``. + + Said another way, it is safe to assume that this class and the rest of this + library operate in terms of timeouts (not deadlines) unless explicitly + noted the usage of deadline semantics. + + It is also important to + understand the three most common applications of the Timeout concept in the + context of this library. + + Usually the generic Timeout term may stand for one of the following actual + timeouts: RPC Timeout, Retry Timeout, or Polling Timeout. + + RPC Timeout: a value supplied by the client to the server so + that the server side knows the maximum amount of time it is expected to + spend handling that specifc RPC. For example, in the case of gRPC transport, + RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2 + request. The `timeout` property of this class normally never represents the + RPC Timeout as it is handled separately by the ``google.api_core.timeout`` + module of this library. + + Retry Timeout: this is the most common meaning of the ``timeout`` property + of this class, and defines how long a certain RPC may be retried in case + the server returns an error. + + Polling Timeout: defines how long the + client side is allowed to call the polling RPC repeatedly to check a status of a + long-running operation. Each polling RPC is + expected to succeed (its errors are supposed to be handled by the retry + logic). The decision as to whether a new polling attempt needs to be made is based + not on the RPC status code but on the status of the returned + status of an operation. In other words: we will poll a long-running operation until the operation is done or the polling timeout expires. Each poll will inform us of the status of the operation. The poll consists of an RPC to the server that may itself be retried as per the poll-specific retry settings in case of errors. The operation-level retry settings do NOT apply to polling-RPC retries. + + With the actual timeout types being defined above, the client libraries + often refer to just Timeout without clarifying which type specifically + that is. In that case the actual timeout type (sometimes also refered to as + Logical Timeout) can be determined from the context. If it is a unary rpc + call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if + provided directly as a standalone value) or Retry Timeout (if provided as + ``retry.timeout`` property of the unary RPC's retry config). For + ``Operation`` or ``PollingFuture`` in general Timeout stands for + Polling Timeout. + Args: predicate (Callable[Exception]): A callable that should return ``True`` if the given exception is retryable. @@ -236,9 +300,9 @@ class Retry(object): must be greater than 0. maximum (float): The maximum amount of time to delay in seconds. multiplier (float): The multiplier applied to the delay. - deadline (float): How long to keep retrying in seconds. The last sleep - period is shortened as necessary, so that the last retry runs at - ``deadline`` (and not considerably beyond it). + timeout (float): How long to keep retrying, in seconds. + deadline (float): DEPRECATED: use `timeout` instead. For backward + compatibility, if specified it will override the ``timeout`` parameter. """ def __init__( @@ -247,14 +311,16 @@ def __init__( initial=_DEFAULT_INITIAL_DELAY, maximum=_DEFAULT_MAXIMUM_DELAY, multiplier=_DEFAULT_DELAY_MULTIPLIER, - deadline=_DEFAULT_DEADLINE, + timeout=_DEFAULT_DEADLINE, on_error=None, + **kwargs ): self._predicate = predicate self._initial = initial self._multiplier = multiplier self._maximum = maximum - self._deadline = deadline + self._timeout = kwargs.get("deadline", timeout) + self._deadline = self._timeout self._on_error = on_error def __call__(self, func, on_error=None): @@ -284,7 +350,7 @@ def retry_wrapped_func(*args, **kwargs): target, self._predicate, sleep_generator, - self._deadline, + self._timeout, on_error=on_error, ) @@ -292,23 +358,45 @@ def retry_wrapped_func(*args, **kwargs): @property def deadline(self): - return self._deadline + """ + DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class + documentation for details. + """ + return self._timeout + + @property + def timeout(self): + return self._timeout def with_deadline(self, deadline): - """Return a copy of this retry with the given deadline. + """Return a copy of this retry with the given timeout. + + DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class + documentation for details. + + Args: + deadline (float): How long to keep retrying in seconds. + + Returns: + Retry: A new retry instance with the given timeout. + """ + return self.with_timeout(timeout=deadline) + + def with_timeout(self, timeout): + """Return a copy of this retry with the given timeout. Args: - deadline (float): How long to keep retrying. + timeout (float): How long to keep retrying, in seconds. Returns: - Retry: A new retry instance with the given deadline. + Retry: A new retry instance with the given timeout. """ return Retry( predicate=self._predicate, initial=self._initial, maximum=self._maximum, multiplier=self._multiplier, - deadline=deadline, + timeout=timeout, on_error=self._on_error, ) @@ -327,7 +415,7 @@ def with_predicate(self, predicate): initial=self._initial, maximum=self._maximum, multiplier=self._multiplier, - deadline=self._deadline, + timeout=self._timeout, on_error=self._on_error, ) @@ -348,19 +436,19 @@ def with_delay(self, initial=None, maximum=None, multiplier=None): initial=initial if initial is not None else self._initial, maximum=maximum if maximum is not None else self._maximum, multiplier=multiplier if multiplier is not None else self._multiplier, - deadline=self._deadline, + timeout=self._timeout, on_error=self._on_error, ) def __str__(self): return ( "".format( + "multiplier={:.1f}, timeout={}, on_error={}>".format( self._predicate, self._initial, self._maximum, self._multiplier, - self._deadline, + self._timeout, # timeout can be None, thus no {:.1f} self._on_error, ) ) diff --git a/google/api_core/retry_async.py b/google/api_core/retry_async.py index 68a25597..81698838 100644 --- a/google/api_core/retry_async.py +++ b/google/api_core/retry_async.py @@ -68,9 +68,12 @@ async def check_if_exists(): _DEFAULT_MAXIMUM_DELAY = 60.0 # seconds _DEFAULT_DELAY_MULTIPLIER = 2.0 _DEFAULT_DEADLINE = 60.0 * 2.0 # seconds +_DEFAULT_TIMEOUT = 60.0 * 2.0 # seconds -async def retry_target(target, predicate, sleep_generator, deadline, on_error=None): +async def retry_target( + target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs +): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the @@ -84,12 +87,12 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. - deadline (float): How long to keep retrying the target. The last sleep - period is shortened as necessary, so that the last retry runs at - ``deadline`` (and not considerably beyond it). + timeout (float): How long to keep retrying the target, in seconds. on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. + deadline (float): DEPRECATED use ``timeout`` instead. For backward + compatibility, if set it will override the ``timeout`` parameter. Returns: Any: the return value of the target function. @@ -99,9 +102,12 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ + + timeout = kwargs.get("deadline", timeout) + deadline_dt = ( - (datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline)) - if deadline + (datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout)) + if timeout else None ) @@ -132,8 +138,8 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No # Chains the raising RetryError with the root cause error, # which helps observability and debugability. raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling target function".format( - deadline + "Timeout of {:.1f}s exceeded while calling target function".format( + timeout ), last_exc, ) from last_exc @@ -165,12 +171,12 @@ class AsyncRetry: must be greater than 0. maximum (float): The maximum amout of time to delay in seconds. multiplier (float): The multiplier applied to the delay. - deadline (float): How long to keep retrying in seconds. The last sleep - period is shortened as necessary, so that the last retry runs at - ``deadline`` (and not considerably beyond it). + timeout (float): How long to keep retrying in seconds. on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. + deadline (float): DEPRECATED use ``timeout`` instead. If set it will + override ``timeout`` parameter. """ def __init__( @@ -179,14 +185,16 @@ def __init__( initial=_DEFAULT_INITIAL_DELAY, maximum=_DEFAULT_MAXIMUM_DELAY, multiplier=_DEFAULT_DELAY_MULTIPLIER, - deadline=_DEFAULT_DEADLINE, + timeout=_DEFAULT_TIMEOUT, on_error=None, + **kwargs ): self._predicate = predicate self._initial = initial self._multiplier = multiplier self._maximum = maximum - self._deadline = deadline + self._timeout = kwargs.get("deadline", timeout) + self._deadline = self._timeout self._on_error = on_error def __call__(self, func, on_error=None): @@ -216,7 +224,7 @@ async def retry_wrapped_func(*args, **kwargs): target, self._predicate, sleep_generator, - self._deadline, + self._timeout, on_error=on_error, ) @@ -228,7 +236,7 @@ def _replace( initial=None, maximum=None, multiplier=None, - deadline=None, + timeout=None, on_error=None, ): return AsyncRetry( @@ -236,12 +244,13 @@ def _replace( initial=initial or self._initial, maximum=maximum or self._maximum, multiplier=multiplier or self._multiplier, - deadline=deadline or self._deadline, + timeout=timeout or self._timeout, on_error=on_error or self._on_error, ) def with_deadline(self, deadline): """Return a copy of this retry with the given deadline. + DEPRECATED: use :meth:`with_timeout` instead. Args: deadline (float): How long to keep retrying. @@ -249,7 +258,18 @@ def with_deadline(self, deadline): Returns: AsyncRetry: A new retry instance with the given deadline. """ - return self._replace(deadline=deadline) + return self._replace(timeout=deadline) + + def with_timeout(self, timeout): + """Return a copy of this retry with the given timeout. + + Args: + timeout (float): How long to keep retrying, in seconds. + + Returns: + AsyncRetry: A new retry instance with the given timeout. + """ + return self._replace(timeout=timeout) def with_predicate(self, predicate): """Return a copy of this retry with the given predicate. @@ -280,12 +300,12 @@ def with_delay(self, initial=None, maximum=None, multiplier=None): def __str__(self): return ( "".format( + "multiplier={:.1f}, timeout={:.1f}, on_error={}>".format( self._predicate, self._initial, self._maximum, self._multiplier, - self._deadline, + self._timeout, self._on_error, ) ) diff --git a/google/api_core/timeout.py b/google/api_core/timeout.py index 73232180..3546d540 100644 --- a/google/api_core/timeout.py +++ b/google/api_core/timeout.py @@ -14,8 +14,9 @@ """Decorators for applying timeout arguments to functions. -These decorators are used to wrap API methods to apply either a constant -or exponential timeout argument. +These decorators are used to wrap API methods to apply either a +Deadline-dependent (recommended), constant (DEPRECATED) or exponential +(DEPRECATED) timeout argument. For example, imagine an API method that can take a while to return results, such as one that might block until a resource is ready: @@ -66,9 +67,69 @@ def is_thing_ready(timeout=None): _DEFAULT_DEADLINE = None +class TimeToDeadlineTimeout(object): + """A decorator that decreases timeout set for an RPC based on how much time + has left till its deadline. The deadline is calculated as + ``now + initial_timeout`` when this decorator is first called for an rpc. + + In other words this decorator implements deadline semantics in terms of a + sequence of decreasing timeouts t0 > t1 > t2 ... tn >= 0. + + Args: + timeout (Optional[float]): the timeout (in seconds) to applied to the + wrapped function. If `None`, the target function is expected to + never timeout. + """ + + def __init__(self, timeout=None, clock=datetime_helpers.utcnow): + self._timeout = timeout + self._clock = clock + + def __call__(self, func): + """Apply the timeout decorator. + + Args: + func (Callable): The function to apply the timeout argument to. + This function must accept a timeout keyword argument. + + Returns: + Callable: The wrapped function. + """ + + first_attempt_timestamp = self._clock().timestamp() + + @functools.wraps(func) + def func_with_timeout(*args, **kwargs): + """Wrapped function that adds timeout.""" + + remaining_timeout = self._timeout + if remaining_timeout is not None: + # All calculations are in seconds + now_timestamp = self._clock().timestamp() + + # To avoid usage of nonlocal but still have round timeout + # numbers for first attempt (in most cases the only attempt made + # for an RPC. + if now_timestamp - first_attempt_timestamp < 0.001: + now_timestamp = first_attempt_timestamp + + time_since_first_attempt = now_timestamp - first_attempt_timestamp + # Avoid setting negative timeout + kwargs["timeout"] = max(0, self._timeout - time_since_first_attempt) + + return func(*args, **kwargs) + + return func_with_timeout + + def __str__(self): + return "".format(self._timeout) + + class ConstantTimeout(object): """A decorator that adds a constant timeout argument. + DEPRECATED: use ``TimeToDeadlineTimeout`` instead. + This is effectively equivalent to ``functools.partial(func, timeout=timeout)``. @@ -140,6 +201,9 @@ def _exponential_timeout_generator(initial, maximum, multiplier, deadline): class ExponentialTimeout(object): """A decorator that adds an exponentially increasing timeout argument. + DEPRECATED: the concept of incrementing timeout exponentially has been + deprecated. Use ``TimeToDeadlineTimeout`` instead. + This is useful if a function is called multiple times. Each time the function is called this decorator will calculate a new timeout parameter based on the the number of times the function has been called. diff --git a/noxfile.py b/noxfile.py index 2d8f1e02..21f0f7b2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -26,7 +26,7 @@ # Black and flake8 clash on the syntax for ignoring flake8's F401 in this file. BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"] -DEFAULT_PYTHON_VERSION = "3.7" +DEFAULT_PYTHON_VERSION = "3.10" CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' @@ -95,7 +95,16 @@ def default(session, install_grpc=True): ) # Install all test dependencies, then install this package in-place. - session.install("dataclasses", "mock", "pytest", "pytest-cov", "pytest-xdist") + session.install( + "dataclasses", + "mock", + # Revert to just "pytest" once + # https://github.com/pytest-dev/pytest/issues/10451 is fixed + "pytest<7.2.0", + "pytest-cov", + "pytest-xdist", + ) + if install_grpc: session.install("-e", ".[grpc]", "-c", constraints_path) else: @@ -192,7 +201,7 @@ def mypy(session): session.run("mypy", "google", "tests") -@nox.session(python="3.8") +@nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. @@ -204,12 +213,12 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.8") +@nox.session(python=DEFAULT_PYTHON_VERSION) def docs(session): """Build the docs for this library.""" session.install("-e", ".[grpc]") - session.install("sphinx==4.0.1", "alabaster", "recommonmark") + session.install("sphinx==4.2.0", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index 11847da7..02d883f6 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -198,41 +198,6 @@ async def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep): method.assert_called_with(timeout=22, metadata=mock.ANY) -@mock.patch("asyncio.sleep") -@mock.patch( - "google.api_core.datetime_helpers.utcnow", - side_effect=_utcnow_monotonic(), - autospec=True, -) -@pytest.mark.asyncio -async def test_wrap_method_with_overriding_retry_deadline(utcnow, unused_sleep): - fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42) - method = mock.Mock( - spec=aio.UnaryUnaryMultiCallable, - side_effect=([exceptions.InternalServerError(None)] * 4) + [fake_call], - ) - - default_retry = retry_async.AsyncRetry() - default_timeout = timeout.ExponentialTimeout(deadline=60) - wrapped_method = gapic_v1.method_async.wrap_method( - method, default_retry, default_timeout - ) - - # Overriding only the retry's deadline should also override the timeout's - # deadline. - result = await wrapped_method(retry=default_retry.with_deadline(30)) - - assert result == 42 - timeout_args = [call[1]["timeout"] for call in method.call_args_list] - assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0] - assert utcnow.call_count == ( - 1 - + 1 # Compute wait_for timeout in retry_async - + 5 # First to set the deadline. - + 5 # One for each min(timeout, maximum, (DEADLINE - NOW).seconds) - ) - - @pytest.mark.asyncio async def test_wrap_method_with_overriding_timeout_as_a_number(): fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42) diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py index 47c3b4b4..34236da7 100644 --- a/tests/asyncio/operations_v1/test_operations_async_client.py +++ b/tests/asyncio/operations_v1/test_operations_async_client.py @@ -17,7 +17,7 @@ try: from grpc import aio -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import grpc_helpers_async diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index 2d0a1bcd..2f8f3460 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -18,11 +18,11 @@ try: import grpc from grpc import aio -except ImportError: +except ImportError: # pragma: NO COVER grpc = aio = None -if grpc is None: +if grpc is None: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 26ad7cef..127ba634 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -18,7 +18,7 @@ try: import grpc # noqa: F401 -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import exceptions diff --git a/tests/asyncio/test_retry_async.py b/tests/asyncio/test_retry_async.py index 873caaf1..14807eb5 100644 --- a/tests/asyncio/test_retry_async.py +++ b/tests/asyncio/test_retry_async.py @@ -116,7 +116,7 @@ async def test_retry_target_deadline_exceeded(utcnow, sleep): await retry_async.retry_target(target, predicate, range(10), deadline=10) assert exc_info.value.cause == exception - assert exc_info.match("Deadline of 10.0s exceeded") + assert exc_info.match("Timeout of 10.0s exceeded") assert exc_info.match("last exception: meep") assert target.call_count == 2 @@ -253,7 +253,7 @@ def if_exception_type(exc): assert re.match( ( r", " - r"initial=1.0, maximum=60.0, multiplier=2.0, deadline=120.0, " + r"initial=1.0, maximum=60.0, multiplier=2.0, timeout=120.0, " r"on_error=None>" ), str(retry_), @@ -276,8 +276,7 @@ async def test___call___and_execute_success(self, sleep): target.assert_called_once_with("meep") sleep.assert_not_called() - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio async def test___call___and_execute_retry(self, sleep, uniform): @@ -302,8 +301,7 @@ async def test___call___and_execute_retry(self, sleep, uniform): sleep.assert_called_once_with(retry_._initial) assert on_error.call_count == 1 - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio async def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform): @@ -376,8 +374,7 @@ async def test___init___without_retry_executed(self, sleep): sleep.assert_not_called() _some_function.assert_not_called() - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio async def test___init___when_retry_is_executed(self, sleep, uniform): diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index 2381d036..f5d9b4f1 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -24,7 +24,7 @@ class PollingFutureImpl(polling.PollingFuture): - def done(self): + def done(self, retry=None): return False def cancel(self): @@ -33,9 +33,6 @@ def cancel(self): def cancelled(self): return False - def running(self): - return True - def test_polling_future_constructor(): future = PollingFutureImpl() @@ -84,20 +81,23 @@ def test_invoke_callback_exception(): class PollingFutureImplWithPoll(PollingFutureImpl): - def __init__(self): + def __init__(self, max_poll_count=1): super(PollingFutureImplWithPoll, self).__init__() self.poll_count = 0 self.event = threading.Event() + self.max_poll_count = max_poll_count - def done(self, retry=polling.DEFAULT_RETRY): + def done(self, retry=None): self.poll_count += 1 + if self.max_poll_count > self.poll_count: + return False self.event.wait() self.set_result(42) return True -def test_result_with_polling(): - future = PollingFutureImplWithPoll() +def test_result_with_one_polling(): + future = PollingFutureImplWithPoll(max_poll_count=1) future.event.set() result = future.result() @@ -109,8 +109,34 @@ def test_result_with_polling(): assert future.poll_count == 1 +def test_result_with_two_pollings(): + future = PollingFutureImplWithPoll(max_poll_count=2) + + future.event.set() + result = future.result() + + assert result == 42 + assert future.poll_count == 2 + # Repeated calls should not cause additional polling + assert future.result() == result + assert future.poll_count == 2 + + +def test_result_with_two_pollings_custom_retry(): + future = PollingFutureImplWithPoll(max_poll_count=2) + + future.event.set() + result = future.result() + + assert result == 42 + assert future.poll_count == 2 + # Repeated calls should not cause additional polling + assert future.result() == result + assert future.poll_count == 2 + + class PollingFutureImplTimeout(PollingFutureImplWithPoll): - def done(self, retry=polling.DEFAULT_RETRY): + def done(self, retry=None): time.sleep(1) return False @@ -132,11 +158,11 @@ def __init__(self, errors): super(PollingFutureImplTransient, self).__init__() self._errors = errors - def done(self, retry=polling.DEFAULT_RETRY): + def done(self, retry=None): + self.poll_count += 1 if self._errors: error, self._errors = self._errors[0], self._errors[1:] raise error("testing") - self.poll_count += 1 self.set_result(42) return True @@ -144,17 +170,17 @@ def done(self, retry=polling.DEFAULT_RETRY): def test_result_transient_error(): future = PollingFutureImplTransient( ( - exceptions.TooManyRequests, - exceptions.InternalServerError, - exceptions.BadGateway, + polling._OperationNotComplete, + polling._OperationNotComplete, + polling._OperationNotComplete, ) ) result = future.result() assert result == 42 - assert future.poll_count == 1 + assert future.poll_count == 4 # Repeated calls should not cause additional polling assert future.result() == result - assert future.poll_count == 1 + assert future.poll_count == 4 def test_callback_background_thread(): @@ -197,23 +223,23 @@ def test_double_callback_background_thread(): class PollingFutureImplWithoutRetry(PollingFutureImpl): - def done(self): + def done(self, retry=None): return True - def result(self): + def result(self, timeout=None, retry=None, polling=None): return super(PollingFutureImplWithoutRetry, self).result() - def _blocking_poll(self, timeout): + def _blocking_poll(self, timeout=None, retry=None, polling=None): return super(PollingFutureImplWithoutRetry, self)._blocking_poll( timeout=timeout ) class PollingFutureImplWith_done_or_raise(PollingFutureImpl): - def done(self): + def done(self, retry=None): return True - def _done_or_raise(self): + def _done_or_raise(self, retry=None): return super(PollingFutureImplWith_done_or_raise, self)._done_or_raise() @@ -223,12 +249,12 @@ def test_polling_future_without_retry(): ) future = PollingFutureImplWithoutRetry() assert future.done() - assert future.running() + assert not future.running() assert future.result() is None with mock.patch.object(future, "done") as done_mock: future._done_or_raise() - done_mock.assert_called_once_with() + done_mock.assert_called_once_with(retry=None) with mock.patch.object(future, "done") as done_mock: future._done_or_raise(retry=custom_retry) @@ -238,5 +264,5 @@ def test_polling_future_without_retry(): def test_polling_future_with__done_or_raise(): future = PollingFutureImplWith_done_or_raise() assert future.done() - assert future.running() + assert not future.running() assert future.result() is None diff --git a/tests/unit/gapic/test_method.py b/tests/unit/gapic/test_method.py index 9778d23a..b1035413 100644 --- a/tests/unit/gapic/test_method.py +++ b/tests/unit/gapic/test_method.py @@ -39,27 +39,6 @@ def _utcnow_monotonic(): curr_value += delta -def test__determine_timeout(): - # Check _determine_timeout always returns a Timeout object. - timeout_type_timeout = timeout.ConstantTimeout(600.0) - returned_timeout = google.api_core.gapic_v1.method._determine_timeout( - 600.0, 600.0, None - ) - assert isinstance(returned_timeout, timeout.ConstantTimeout) - returned_timeout = google.api_core.gapic_v1.method._determine_timeout( - 600.0, timeout_type_timeout, None - ) - assert isinstance(returned_timeout, timeout.ConstantTimeout) - returned_timeout = google.api_core.gapic_v1.method._determine_timeout( - timeout_type_timeout, 600.0, None - ) - assert isinstance(returned_timeout, timeout.ConstantTimeout) - returned_timeout = google.api_core.gapic_v1.method._determine_timeout( - timeout_type_timeout, timeout_type_timeout, None - ) - assert isinstance(returned_timeout, timeout.ConstantTimeout) - - def test_wrap_method_basic(): method = mock.Mock(spec=["__call__"], return_value=42) @@ -199,37 +178,6 @@ def test_wrap_method_with_overriding_retry_and_timeout(unusued_sleep): method.assert_called_with(timeout=22, metadata=mock.ANY) -@mock.patch("time.sleep") -@mock.patch( - "google.api_core.datetime_helpers.utcnow", - side_effect=_utcnow_monotonic(), - autospec=True, -) -def test_wrap_method_with_overriding_retry_deadline(utcnow, unused_sleep): - method = mock.Mock( - spec=["__call__"], - side_effect=([exceptions.InternalServerError(None)] * 4) + [42], - ) - default_retry = retry.Retry() - default_timeout = timeout.ExponentialTimeout(deadline=60) - wrapped_method = google.api_core.gapic_v1.method.wrap_method( - method, default_retry, default_timeout - ) - - # Overriding only the retry's deadline should also override the timeout's - # deadline. - result = wrapped_method(retry=default_retry.with_deadline(30)) - - assert result == 42 - timeout_args = [call[1]["timeout"] for call in method.call_args_list] - assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0] - assert utcnow.call_count == ( - 1 - + 5 # First to set the deadline. - + 5 # One for each min(timeout, maximum, (DEADLINE - NOW).seconds) - ) - - def test_wrap_method_with_overriding_timeout_as_a_number(): method = mock.Mock(spec=["__call__"], return_value=42) default_retry = retry.Retry() diff --git a/tests/unit/operations_v1/test_operations_client.py b/tests/unit/operations_v1/test_operations_client.py index 187f0be3..fb4b14f1 100644 --- a/tests/unit/operations_v1/test_operations_client.py +++ b/tests/unit/operations_v1/test_operations_client.py @@ -16,12 +16,13 @@ try: import grpc # noqa: F401 -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import grpc_helpers from google.api_core import operations_v1 from google.api_core import page_iterator +from google.api_core.operations_v1 import operations_client_config from google.longrunning import operations_pb2 from google.protobuf import empty_pb2 @@ -96,3 +97,7 @@ def test_cancel_operation(): ].metadata assert len(channel.CancelOperation.requests) == 1 assert channel.CancelOperation.requests[0].name == "name" + + +def test_operations_client_config(): + assert operations_client_config.config["interfaces"] diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py index 625539e2..149c463c 100644 --- a/tests/unit/operations_v1/test_operations_rest_client.py +++ b/tests/unit/operations_v1/test_operations_rest_client.py @@ -20,7 +20,7 @@ try: import grpc # noqa: F401 -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from requests import Response # noqa I201 from requests.sessions import Session @@ -121,7 +121,7 @@ def test_operations_client_from_service_account_info(client_class): assert client.transport._credentials == creds assert isinstance(client, client_class) - assert client.transport._host == "longrunning.googleapis.com:443" + assert client.transport._host == "https://longrunning.googleapis.com" @pytest.mark.parametrize( @@ -160,7 +160,7 @@ def test_operations_client_from_service_account_file(client_class): assert client.transport._credentials == creds assert isinstance(client, client_class) - assert client.transport._host == "longrunning.googleapis.com:443" + assert client.transport._host == "https://longrunning.googleapis.com" def test_operations_client_get_transport_class(): @@ -465,10 +465,7 @@ def test_list_operations_rest( actual_args = req.call_args assert actual_args.args[0] == "GET" - assert ( - actual_args.args[1] - == "https://longrunning.googleapis.com:443/v3/operations" - ) + assert actual_args.args[1] == "https://longrunning.googleapis.com/v3/operations" assert actual_args.kwargs["params"] == [ ("filter", "my_filter"), ("pageSize", 10), @@ -574,7 +571,7 @@ def test_get_operation_rest( assert actual_args.args[0] == "GET" assert ( actual_args.args[1] - == "https://longrunning.googleapis.com:443/v3/operations/sample1" + == "https://longrunning.googleapis.com/v3/operations/sample1" ) # Establish that the response is the type that we expect. @@ -591,13 +588,11 @@ def test_get_operation_rest_failure(): response_value.status_code = 400 mock_request = mock.MagicMock() mock_request.method = "GET" - mock_request.url = ( - "https://longrunning.googleapis.com:443/v1/operations/sample1" - ) + mock_request.url = "https://longrunning.googleapis.com/v1/operations/sample1" response_value.request = mock_request req.return_value = response_value with pytest.raises(core_exceptions.GoogleAPIError): - client.get_operation("operations/sample1") + client.get_operation("sammple0/operations/sample1") def test_delete_operation_rest( @@ -619,7 +614,7 @@ def test_delete_operation_rest( assert actual_args.args[0] == "DELETE" assert ( actual_args.args[1] - == "https://longrunning.googleapis.com:443/v3/operations/sample1" + == "https://longrunning.googleapis.com/v3/operations/sample1" ) @@ -631,13 +626,11 @@ def test_delete_operation_rest_failure(): response_value.status_code = 400 mock_request = mock.MagicMock() mock_request.method = "DELETE" - mock_request.url = ( - "https://longrunning.googleapis.com:443/v1/operations/sample1" - ) + mock_request.url = "https://longrunning.googleapis.com/v1/operations/sample1" response_value.request = mock_request req.return_value = response_value with pytest.raises(core_exceptions.GoogleAPIError): - client.delete_operation(name="operations/sample1") + client.delete_operation(name="sample0/operations/sample1") def test_cancel_operation_rest(transport: str = "rest"): @@ -657,7 +650,7 @@ def test_cancel_operation_rest(transport: str = "rest"): assert actual_args.args[0] == "POST" assert ( actual_args.args[1] - == "https://longrunning.googleapis.com:443/v3/operations/sample1:cancel" + == "https://longrunning.googleapis.com/v3/operations/sample1:cancel" ) @@ -670,12 +663,12 @@ def test_cancel_operation_rest_failure(): mock_request = mock.MagicMock() mock_request.method = "POST" mock_request.url = ( - "https://longrunning.googleapis.com:443/v1/operations/sample1:cancel" + "https://longrunning.googleapis.com/v1/operations/sample1:cancel" ) response_value.request = mock_request req.return_value = response_value with pytest.raises(core_exceptions.GoogleAPIError): - client.cancel_operation(name="operations/sample1") + client.cancel_operation(name="sample0/operations/sample1") def test_credentials_transport_error(): @@ -825,7 +818,7 @@ def test_operations_host_no_port(): api_endpoint="longrunning.googleapis.com" ), ) - assert client.transport._host == "longrunning.googleapis.com:443" + assert client.transport._host == "https://longrunning.googleapis.com" def test_operations_host_with_port(): @@ -835,7 +828,7 @@ def test_operations_host_with_port(): api_endpoint="longrunning.googleapis.com:8000" ), ) - assert client.transport._host == "longrunning.googleapis.com:8000" + assert client.transport._host == "https://longrunning.googleapis.com:8000" def test_common_billing_account_path(): diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 7fb16209..f5e2b72b 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -22,7 +22,7 @@ try: import grpc -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import bidi diff --git a/tests/unit/test_client_info.py b/tests/unit/test_client_info.py index f5eebfbe..3361fef6 100644 --- a/tests/unit/test_client_info.py +++ b/tests/unit/test_client_info.py @@ -15,7 +15,7 @@ try: import grpc -except ImportError: +except ImportError: # pragma: NO COVER grpc = None from google.api_core import client_info @@ -26,9 +26,9 @@ def test_constructor_defaults(): assert info.python_version is not None - if grpc is not None: + if grpc is not None: # pragma: NO COVER assert info.grpc_version is not None - else: + else: # pragma: NO COVER assert info.grpc_version is None assert info.api_core_version is not None diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 4169ad44..eb0b12d1 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -22,7 +22,7 @@ try: import grpc from grpc_status import rpc_status -except ImportError: +except ImportError: # pragma: NO COVER grpc = rpc_status = None from google.api_core import exceptions diff --git a/tests/unit/test_general_helpers.py b/tests/unit/test_general_helpers.py new file mode 100644 index 00000000..82d6d4b6 --- /dev/null +++ b/tests/unit/test_general_helpers.py @@ -0,0 +1,13 @@ +# Copyright 2022, Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 8b9fd9f1..9fe938d6 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -17,7 +17,7 @@ try: import grpc -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import exceptions @@ -365,7 +365,7 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c default.assert_called_once_with(scopes=None, default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -400,7 +400,7 @@ def test_create_channel_implicit_with_default_host( mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host ) - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -427,7 +427,7 @@ def test_create_channel_implicit_with_ssl_creds( composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -452,7 +452,7 @@ def test_create_channel_implicit_with_scopes( default.assert_called_once_with(scopes=["one", "two"], default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -477,7 +477,7 @@ def test_create_channel_implicit_with_default_scopes( default.assert_called_once_with(scopes=None, default_scopes=["three", "four"]) - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -509,7 +509,7 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -533,7 +533,7 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -561,7 +561,7 @@ def test_create_channel_explicit_default_scopes( assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -587,7 +587,7 @@ def test_create_channel_explicit_with_quota_project( assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -616,7 +616,7 @@ def test_create_channel_with_credentials_file( assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -648,7 +648,7 @@ def test_create_channel_with_credentials_file_and_scopes( assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -680,7 +680,7 @@ def test_create_channel_with_credentials_file_and_default_scopes( assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: + if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER grpc_secure_channel.assert_called_once_with(target, composite_creds, None) else: grpc_secure_channel.assert_called_once_with(target, composite_creds) @@ -690,7 +690,7 @@ def test_create_channel_with_credentials_file_and_default_scopes( not grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available" ) @mock.patch("grpc_gcp.secure_channel") -def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): +def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): # pragma: NO COVER target = "example.com:443" scopes = ["test_scope"] diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index 22e23bc3..f029866c 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -18,7 +18,7 @@ try: import grpc # noqa: F401 -except ImportError: +except ImportError: # pragma: NO COVER pytest.skip("No GRPC", allow_module_level=True) from google.api_core import exceptions diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 74c5d77c..ec27056d 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -52,7 +52,7 @@ def test_if_transient_error(): # Make uniform return half of its maximum, which will be the calculated # sleep time. -@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) +@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) def test_exponential_sleep_generator_base_2(uniform): gen = retry.exponential_sleep_generator(1, 60, multiplier=2) @@ -172,6 +172,7 @@ def test_constructor_defaults(self): assert retry_._deadline == 120 assert retry_._on_error is None assert retry_.deadline == 120 + assert retry_.timeout == 120 def test_constructor_options(self): _some_function = mock.Mock() @@ -315,7 +316,7 @@ def if_exception_type(exc): assert re.match( ( r", " - r"initial=1.0, maximum=60.0, multiplier=2.0, deadline=120.0, " + r"initial=1.0, maximum=60.0, multiplier=2.0, timeout=120.0, " r"on_error=None>" ), str(retry_), @@ -337,8 +338,7 @@ def test___call___and_execute_success(self, sleep): target.assert_called_once_with("meep") sleep.assert_not_called() - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("time.sleep", autospec=True) def test___call___and_execute_retry(self, sleep, uniform): @@ -360,8 +360,7 @@ def test___call___and_execute_retry(self, sleep, uniform): sleep.assert_called_once_with(retry_._initial) assert on_error.call_count == 1 - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("time.sleep", autospec=True) def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform): @@ -371,7 +370,7 @@ def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform): initial=1.0, maximum=1024.0, multiplier=2.0, - deadline=9.9, + deadline=30.9, ) utcnow = datetime.datetime.utcnow() @@ -406,8 +405,17 @@ def increase_time(sleep_delay): last_wait = sleep.call_args.args[0] total_wait = sum(call_args.args[0] for call_args in sleep.call_args_list) - assert last_wait == 2.9 # and not 8.0, because the last delay was shortened - assert total_wait == 9.9 # the same as the deadline + assert last_wait == 8.0 + # Next attempt would be scheduled in 16 secs, 15 + 16 = 31 > 30.9, thus + # we do not even wait for it to be scheduled (30.9 is configured timeout). + # This changes the previous logic of shortening the last attempt to fit + # in the deadline. The previous logic was removed to make Python retry + # logic consistent with the other languages and to not disrupt the + # randomized retry delays distribution by artificially increasing a + # probability of scheduling two (instead of one) last attempts with very + # short delay between them, while the second retry having very low chance + # of succeeding anyways. + assert total_wait == 15.0 @mock.patch("time.sleep", autospec=True) def test___init___without_retry_executed(self, sleep): @@ -432,8 +440,7 @@ def test___init___without_retry_executed(self, sleep): sleep.assert_not_called() _some_function.assert_not_called() - # Make uniform return half of its maximum, which is the calculated sleep time. - @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0) + @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n) @mock.patch("time.sleep", autospec=True) def test___init___when_retry_is_executed(self, sleep, uniform): _some_function = mock.Mock() diff --git a/tests/unit/test_timeout.py b/tests/unit/test_timeout.py index 30d624e2..a83a2ecb 100644 --- a/tests/unit/test_timeout.py +++ b/tests/unit/test_timeout.py @@ -17,11 +17,11 @@ import mock -from google.api_core import timeout +from google.api_core import timeout as timeouts def test__exponential_timeout_generator_base_2(): - gen = timeout._exponential_timeout_generator(1.0, 60.0, 2.0, deadline=None) + gen = timeouts._exponential_timeout_generator(1.0, 60.0, 2.0, deadline=None) result = list(itertools.islice(gen, 8)) assert result == [1, 2, 4, 8, 16, 32, 60, 60] @@ -34,7 +34,7 @@ def test__exponential_timeout_generator_base_deadline(utcnow): datetime.datetime.min + datetime.timedelta(seconds=n) for n in range(15) ] - gen = timeout._exponential_timeout_generator(1.0, 60.0, 2.0, deadline=30.0) + gen = timeouts._exponential_timeout_generator(1.0, 60.0, 2.0, deadline=30.0) result = list(itertools.islice(gen, 14)) # Should grow until the cumulative time is > 30s, then start decreasing as @@ -42,22 +42,105 @@ def test__exponential_timeout_generator_base_deadline(utcnow): assert result == [1, 2, 4, 8, 16, 24, 23, 22, 21, 20, 19, 18, 17, 16] +class TestTimeToDeadlineTimeout(object): + def test_constructor(self): + timeout_ = timeouts.TimeToDeadlineTimeout() + assert timeout_._timeout is None + + def test_constructor_args(self): + timeout_ = timeouts.TimeToDeadlineTimeout(42.0) + assert timeout_._timeout == 42.0 + + def test___str__(self): + timeout_ = timeouts.TimeToDeadlineTimeout(1) + assert str(timeout_) == "" + + def test_apply(self): + target = mock.Mock(spec=["__call__", "__name__"], __name__="target") + + datetime.datetime.utcnow() + datetime.timedelta(seconds=1) + + now = datetime.datetime.utcnow() + + times = [ + now, + now + datetime.timedelta(seconds=0.0009), + now + datetime.timedelta(seconds=1), + now + datetime.timedelta(seconds=39), + now + datetime.timedelta(seconds=42), + now + datetime.timedelta(seconds=43), + ] + + def _clock(): + return times.pop(0) + + timeout_ = timeouts.TimeToDeadlineTimeout(42.0, _clock) + wrapped = timeout_(target) + + wrapped() + target.assert_called_with(timeout=42.0) + wrapped() + target.assert_called_with(timeout=41.0) + wrapped() + target.assert_called_with(timeout=3.0) + wrapped() + target.assert_called_with(timeout=0.0) + wrapped() + target.assert_called_with(timeout=0.0) + + def test_apply_no_timeout(self): + target = mock.Mock(spec=["__call__", "__name__"], __name__="target") + + datetime.datetime.utcnow() + datetime.timedelta(seconds=1) + + now = datetime.datetime.utcnow() + + times = [ + now, + now + datetime.timedelta(seconds=0.0009), + now + datetime.timedelta(seconds=1), + now + datetime.timedelta(seconds=2), + ] + + def _clock(): + return times.pop(0) + + timeout_ = timeouts.TimeToDeadlineTimeout(clock=_clock) + wrapped = timeout_(target) + + wrapped() + target.assert_called_with() + wrapped() + target.assert_called_with() + + def test_apply_passthrough(self): + target = mock.Mock(spec=["__call__", "__name__"], __name__="target") + timeout_ = timeouts.TimeToDeadlineTimeout(42.0) + wrapped = timeout_(target) + + wrapped(1, 2, meep="moop") + + target.assert_called_once_with(1, 2, meep="moop", timeout=42.0) + + class TestConstantTimeout(object): def test_constructor(self): - timeout_ = timeout.ConstantTimeout() + timeout_ = timeouts.ConstantTimeout() assert timeout_._timeout is None def test_constructor_args(self): - timeout_ = timeout.ConstantTimeout(42.0) + timeout_ = timeouts.ConstantTimeout(42.0) assert timeout_._timeout == 42.0 def test___str__(self): - timeout_ = timeout.ConstantTimeout(1) + timeout_ = timeouts.ConstantTimeout(1) assert str(timeout_) == "" def test_apply(self): target = mock.Mock(spec=["__call__", "__name__"], __name__="target") - timeout_ = timeout.ConstantTimeout(42.0) + timeout_ = timeouts.ConstantTimeout(42.0) wrapped = timeout_(target) wrapped() @@ -66,7 +149,7 @@ def test_apply(self): def test_apply_passthrough(self): target = mock.Mock(spec=["__call__", "__name__"], __name__="target") - timeout_ = timeout.ConstantTimeout(42.0) + timeout_ = timeouts.ConstantTimeout(42.0) wrapped = timeout_(target) wrapped(1, 2, meep="moop") @@ -76,30 +159,30 @@ def test_apply_passthrough(self): class TestExponentialTimeout(object): def test_constructor(self): - timeout_ = timeout.ExponentialTimeout() - assert timeout_._initial == timeout._DEFAULT_INITIAL_TIMEOUT - assert timeout_._maximum == timeout._DEFAULT_MAXIMUM_TIMEOUT - assert timeout_._multiplier == timeout._DEFAULT_TIMEOUT_MULTIPLIER - assert timeout_._deadline == timeout._DEFAULT_DEADLINE + timeout_ = timeouts.ExponentialTimeout() + assert timeout_._initial == timeouts._DEFAULT_INITIAL_TIMEOUT + assert timeout_._maximum == timeouts._DEFAULT_MAXIMUM_TIMEOUT + assert timeout_._multiplier == timeouts._DEFAULT_TIMEOUT_MULTIPLIER + assert timeout_._deadline == timeouts._DEFAULT_DEADLINE def test_constructor_args(self): - timeout_ = timeout.ExponentialTimeout(1, 2, 3, 4) + timeout_ = timeouts.ExponentialTimeout(1, 2, 3, 4) assert timeout_._initial == 1 assert timeout_._maximum == 2 assert timeout_._multiplier == 3 assert timeout_._deadline == 4 def test_with_timeout(self): - original_timeout = timeout.ExponentialTimeout() + original_timeout = timeouts.ExponentialTimeout() timeout_ = original_timeout.with_deadline(42) assert original_timeout is not timeout_ - assert timeout_._initial == timeout._DEFAULT_INITIAL_TIMEOUT - assert timeout_._maximum == timeout._DEFAULT_MAXIMUM_TIMEOUT - assert timeout_._multiplier == timeout._DEFAULT_TIMEOUT_MULTIPLIER + assert timeout_._initial == timeouts._DEFAULT_INITIAL_TIMEOUT + assert timeout_._maximum == timeouts._DEFAULT_MAXIMUM_TIMEOUT + assert timeout_._multiplier == timeouts._DEFAULT_TIMEOUT_MULTIPLIER assert timeout_._deadline == 42 def test___str__(self): - timeout_ = timeout.ExponentialTimeout(1, 2, 3, 4) + timeout_ = timeouts.ExponentialTimeout(1, 2, 3, 4) assert str(timeout_) == ( "" @@ -107,7 +190,7 @@ def test___str__(self): def test_apply(self): target = mock.Mock(spec=["__call__", "__name__"], __name__="target") - timeout_ = timeout.ExponentialTimeout(1, 10, 2) + timeout_ = timeouts.ExponentialTimeout(1, 10, 2) wrapped = timeout_(target) wrapped() @@ -121,7 +204,7 @@ def test_apply(self): def test_apply_passthrough(self): target = mock.Mock(spec=["__call__", "__name__"], __name__="target") - timeout_ = timeout.ExponentialTimeout(42.0, 100, 2) + timeout_ = timeouts.ExponentialTimeout(42.0, 100, 2) wrapped = timeout_(target) wrapped(1, 2, meep="moop") From 7cc329fe1498b0a4285123448e4ea80c6a780d47 Mon Sep 17 00:00:00 2001 From: aribray <45905583+aribray@users.noreply.github.com> Date: Thu, 10 Nov 2022 08:24:52 -0600 Subject: [PATCH 116/126] fix: require google-auth >= 2.14.1 (#463) * bump google-auth version to 2.13.0 * chore: set minimum to 2.14.1 instead Co-authored-by: Anthonios Partheniou --- setup.py | 2 +- testing/constraints-3.7.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1e746c8e..6accc5b7 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", - "google-auth >= 1.25.0, < 3.0dev", + "google-auth >= 2.14.1, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index d58c05b0..ec041297 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -7,7 +7,7 @@ # Then this file should have foo==1.14.0 googleapis-common-protos==1.56.2 protobuf==3.19.5 -google-auth==1.25.0 +google-auth==2.14.1 requests==2.18.0 packaging==14.3 grpcio==1.33.2 From 83ce2d7c0bae3184e0ab1548f00eae96a5da107d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 10 Nov 2022 14:36:04 +0000 Subject: [PATCH 117/126] chore(python): update dependencies in .kokoro/requirements.txt [autoapprove] (#464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): update dependencies in .kokoro/requirements.txt Source-Link: https://github.com/googleapis/synthtool/commit/e3a1277ac35fc88c09db1930533e24292b132ced Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:452901c74a22f9b9a3bd02bce780b8e8805c97270d424684bff809ce5be8c2a2 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * exclude templated lint github action * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix typo Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/requirements.txt | 325 +++++++++++++++++++++----------------- owlbot.py | 1 + 3 files changed, 179 insertions(+), 149 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 3815c983..12edee77 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 + digest: sha256:452901c74a22f9b9a3bd02bce780b8e8805c97270d424684bff809ce5be8c2a2 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index d15994ba..31425f16 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -20,9 +20,9 @@ cachetools==5.2.0 \ --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db # via google-auth -certifi==2022.6.15 \ - --hash=sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d \ - --hash=sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412 +certifi==2022.9.24 \ + --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ + --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ @@ -110,29 +110,33 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==37.0.4 \ - --hash=sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59 \ - --hash=sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596 \ - --hash=sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3 \ - --hash=sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5 \ - --hash=sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab \ - --hash=sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884 \ - --hash=sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82 \ - --hash=sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b \ - --hash=sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441 \ - --hash=sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa \ - --hash=sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d \ - --hash=sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b \ - --hash=sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a \ - --hash=sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6 \ - --hash=sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157 \ - --hash=sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280 \ - --hash=sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282 \ - --hash=sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67 \ - --hash=sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8 \ - --hash=sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046 \ - --hash=sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327 \ - --hash=sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9 +cryptography==38.0.3 \ + --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ + --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ + --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ + --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ + --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ + --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ + --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ + --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ + --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ + --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ + --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ + --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ + --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ + --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ + --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ + --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ + --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ + --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ + --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ + --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ + --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ + --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ + --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ + --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ + --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ + --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 # via # gcp-releasetool # secretstorage @@ -148,23 +152,23 @@ filelock==3.8.0 \ --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 # via virtualenv -gcp-docuploader==0.6.3 \ - --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ - --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b +gcp-docuploader==0.6.4 \ + --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ + --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.8.7 \ - --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ - --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d +gcp-releasetool==1.9.1 \ + --hash=sha256:952f4055d5d986b070ae2a71c4410b250000f9cc5a1e26398fcd55a5bbc5a15f \ + --hash=sha256:d0d3c814a97c1a237517e837d8cfa668ced8df4b882452578ecef4a4e79c583b # via -r requirements.in -google-api-core==2.8.2 \ - --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ - --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 +google-api-core==2.10.2 \ + --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ + --hash=sha256:34f24bd1d5f72a8c4519773d99ca6bf080a6c4e041b4e9f024fe230191dda62e # via # google-cloud-core # google-cloud-storage -google-auth==2.11.0 \ - --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ - --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb +google-auth==2.14.0 \ + --hash=sha256:1ad5b0e6eba5f69645971abb3d2c197537d5914070a8c6d30299dfdb07c5c700 \ + --hash=sha256:cf24817855d874ede2efd071aa22125445f555de1685b739a9782fcf408c2a3d # via # gcp-releasetool # google-api-core @@ -178,72 +182,97 @@ google-cloud-storage==2.5.0 \ --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 # via gcp-docuploader -google-crc32c==1.3.0 \ - --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ - --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ - --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \ - --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \ - --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \ - --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \ - --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \ - --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \ - --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \ - --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \ - --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \ - --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \ - --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \ - --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \ - --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \ - --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \ - --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \ - --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \ - --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \ - --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \ - --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \ - --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \ - --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \ - --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \ - --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \ - --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \ - --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \ - --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \ - --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \ - --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \ - --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \ - --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \ - --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \ - --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \ - --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \ - --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \ - --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \ - --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \ - --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \ - --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \ - --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ - --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ - --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 +google-crc32c==1.5.0 \ + --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ + --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ + --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ + --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ + --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ + --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ + --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ + --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ + --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ + --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ + --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ + --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ + --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ + --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ + --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ + --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ + --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ + --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ + --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ + --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ + --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ + --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ + --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ + --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ + --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ + --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ + --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ + --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ + --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ + --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ + --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ + --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ + --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ + --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ + --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ + --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ + --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ + --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ + --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ + --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ + --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ + --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ + --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ + --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ + --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ + --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ + --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ + --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ + --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ + --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ + --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ + --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ + --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ + --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ + --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ + --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ + --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ + --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ + --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ + --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ + --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ + --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ + --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ + --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ + --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ + --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ + --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ + --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 # via google-resumable-media -google-resumable-media==2.3.3 \ - --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ - --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 +google-resumable-media==2.4.0 \ + --hash=sha256:2aa004c16d295c8f6c33b2b4788ba59d366677c0a25ae7382436cb30f776deaa \ + --hash=sha256:8d5518502f92b9ecc84ac46779bd4f09694ecb3ba38a3e7ca737a86d15cbca1f # via google-cloud-storage googleapis-common-protos==1.56.4 \ --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 # via google-api-core -idna==3.3 \ - --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ - --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 # via requests -importlib-metadata==4.12.0 \ - --hash=sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670 \ - --hash=sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23 +importlib-metadata==5.0.0 \ + --hash=sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab \ + --hash=sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43 # via # -r requirements.in # twine -jaraco-classes==3.2.2 \ - --hash=sha256:6745f113b0b588239ceb49532aa09c3ebb947433ce311ef2f8e3ad64ebb74594 \ - --hash=sha256:e6ef6fd3fcf4579a7a019d87d1e56a883f4e4c35cfe925f86731abc58804e647 +jaraco-classes==3.2.3 \ + --hash=sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158 \ + --hash=sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -255,9 +284,9 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.9.0 \ - --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \ - --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db +keyring==23.9.3 \ + --hash=sha256:69732a15cb1433bdfbc3b980a8a36a04878a6cfd7cb99f497b573f31618001c0 \ + --hash=sha256:69b01dd83c42f590250fe7a1f503fc229b14de83857314b1933a3ddbf595c4a5 # via # gcp-releasetool # twine @@ -303,9 +332,9 @@ markupsafe==2.1.1 \ --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 # via jinja2 -more-itertools==8.14.0 \ - --hash=sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2 \ - --hash=sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750 +more-itertools==9.0.0 \ + --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \ + --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab # via jaraco-classes nox==2022.8.7 \ --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ @@ -325,34 +354,34 @@ platformdirs==2.5.2 \ --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 # via virtualenv -protobuf==3.20.2 \ - --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ - --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ - --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ - --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ - --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ - --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ - --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ - --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ - --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ - --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ - --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ - --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ - --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ - --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ - --hash=sha256:9f876a69ca55aed879b43c295a328970306e8e80a263ec91cf6e9189243c613b \ - --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ - --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ - --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ - --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ - --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ - --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ - --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ - --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 +protobuf==3.20.3 \ + --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ + --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \ + --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \ + --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \ + --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \ + --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \ + --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \ + --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \ + --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \ + --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \ + --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \ + --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \ + --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \ + --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \ + --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \ + --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \ + --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \ + --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \ + --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \ + --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \ + --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \ + --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee # via # gcp-docuploader # gcp-releasetool # google-api-core + # googleapis-common-protos py==1.11.0 \ --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 @@ -377,9 +406,9 @@ pygments==2.13.0 \ # via # readme-renderer # rich -pyjwt==2.4.0 \ - --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ - --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba +pyjwt==2.6.0 \ + --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \ + --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14 # via gcp-releasetool pyparsing==3.0.9 \ --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ @@ -392,9 +421,9 @@ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via gcp-releasetool -readme-renderer==37.0 \ - --hash=sha256:07b7ea234e03e58f77cc222e206e6abb8f4c0435becce5104794ee591f9301c5 \ - --hash=sha256:9fa416704703e509eeb900696751c908ddeb2011319d93700d8f18baff887a69 +readme-renderer==37.3 \ + --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ + --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 # via twine requests==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ @@ -405,17 +434,17 @@ requests==2.28.1 \ # google-cloud-storage # requests-toolbelt # twine -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +requests-toolbelt==0.10.1 \ + --hash=sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7 \ + --hash=sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d # via twine rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==12.5.1 \ - --hash=sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb \ - --hash=sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca +rich==12.6.0 \ + --hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \ + --hash=sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -437,9 +466,9 @@ twine==4.0.1 \ --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 # via -r requirements.in -typing-extensions==4.3.0 \ - --hash=sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02 \ - --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6 +typing-extensions==4.4.0 \ + --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ + --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in urllib3==1.26.12 \ --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ @@ -447,9 +476,9 @@ urllib3==1.26.12 \ # via # requests # twine -virtualenv==20.16.4 \ - --hash=sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782 \ - --hash=sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22 +virtualenv==20.16.6 \ + --hash=sha256:186ca84254abcbde98180fd17092f9628c5fe742273c02724972a1d8a2035108 \ + --hash=sha256:530b850b523c6449406dfba859d6345e48ef19b8439606c5d74d7d3c9e14d76e # via nox webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ @@ -459,13 +488,13 @@ wheel==0.37.1 \ --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 # via -r requirements.in -zipp==3.8.1 \ - --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \ - --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009 +zipp==3.10.0 \ + --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \ + --hash=sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==65.2.0 \ - --hash=sha256:7f4bc85450898a09f76ebf28b72fa25bc7111f6c7d665d514a60bba9c75ef2a9 \ - --hash=sha256:a3ca5857c89f82f5c9410e8508cb32f4872a3bafd4aa7ae122a24ca33bccc750 +setuptools==65.5.0 \ + --hash=sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17 \ + --hash=sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356 # via -r requirements.in diff --git a/owlbot.py b/owlbot.py index ab4f4f0a..8a60b152 100644 --- a/owlbot.py +++ b/owlbot.py @@ -30,6 +30,7 @@ ".coveragerc", # layout "CONTRIBUTING.rst", # no systests ".github/workflows/unittest.yml", # exclude unittest gh action + ".github/workflows/lint.yml", # exclude lint gh action "README.rst", ] templated_files = common.py_library(microgenerator=True, cov_level=100) From ff379e304c353bcab734e1c4706b74b356a1e932 Mon Sep 17 00:00:00 2001 From: Daniel Ziegenberg Date: Thu, 10 Nov 2022 16:09:34 +0100 Subject: [PATCH 118/126] feat: add support for Python 3.11 (#466) * feat: add support for Python 3.11 Signed-off-by: Daniel Ziegenberg * restore check for unit-3.10_wo_grpc * restore check for unit_wo_grpc-3.10 * restore 3.10 test * require grpcio 1.49.1 with python 3.11 * lint Signed-off-by: Daniel Ziegenberg Co-authored-by: Anthonios Partheniou --- .github/sync-repo-settings.yaml | 3 +++ .github/workflows/unittest.yml | 1 + CONTRIBUTING.rst | 6 ++++-- noxfile.py | 8 ++++---- setup.py | 8 +++++++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 6d2c2a0e..cdec3e9b 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -15,11 +15,14 @@ branchProtectionRules: - 'unit_grpc_gcp-3.8' - 'unit_grpc_gcp-3.9' - 'unit_grpc_gcp-3.10' + - 'unit_grpc_gcp-3.11' - 'unit-3.7' - 'unit-3.8' - 'unit-3.9' - 'unit-3.10' + - 'unit-3.11' - 'unit_wo_grpc-3.10' + - 'unit_wo_grpc-3.11' - 'cover' - 'docs' - 'docfx' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index cd1d4d60..0d7789b6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -17,6 +17,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" exclude: - option: "_wo_grpc" python: 3.7 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dddeddb9..13d7a516 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9, 3.10, and 3.11 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -71,7 +71,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.10 -- -k + $ nox -s unit-3.11 -- -k .. note:: @@ -201,11 +201,13 @@ We support: - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ +- `Python 3.11`_ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ +.. _Python 3.11: https://docs.python.org/3.11/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/noxfile.py b/noxfile.py index 21f0f7b2..9bc2d967 100644 --- a/noxfile.py +++ b/noxfile.py @@ -113,7 +113,7 @@ def default(session, install_grpc=True): pytest_args = [ "python", "-m", - "py.test", + "pytest", *( # Helpful for running a single test or testfile. session.posargs @@ -146,13 +146,13 @@ def default(session, install_grpc=True): session.run(*pytest_args) -@nox.session(python=["3.7", "3.8", "3.9", "3.10"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( @@ -166,7 +166,7 @@ def unit_grpc_gcp(session): default(session) -@nox.session(python=["3.8", "3.10"]) +@nox.session(python=["3.8", "3.10", "3.11"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" default(session, install_grpc=False) diff --git a/setup.py b/setup.py index 6accc5b7..6050e5c5 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,12 @@ "requests >= 2.18.0, < 3.0.0dev", ] extras = { - "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], + "grpc": [ + "grpcio >= 1.33.2, < 2.0dev", + "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'", + "grpcio-status >= 1.33.2, < 2.0dev", + "grpcio-status >= 1.49.1, < 2.0dev; python_version>='3.11'", + ], "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } @@ -86,6 +91,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: OS Independent", "Topic :: Internet", ], From 522b98ecc1ebd1c2280d3d7c73a02f6e4fb528d4 Mon Sep 17 00:00:00 2001 From: Victor Chudnovsky Date: Thu, 10 Nov 2022 07:27:57 -0800 Subject: [PATCH 119/126] feat: Allow representing enums with their unqualified symbolic names in headers (#465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Allow non-fully-qualified enums in routing headers * Rename s/fully_qualified_enums/qualified_enums/g for correctness * chore: minor tweaks * chore: Temporary workaround for pytest in noxfile. * Fix import order * bring coverage to 100% * lint * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * remove replacement in owlbot.py causing lint failure Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- google/api_core/gapic_v1/routing_header.py | 20 +++++++++++--- noxfile.py | 1 - owlbot.py | 2 -- tests/unit/gapic/test_routing_header.py | 31 ++++++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index a7bcb5a8..28b13abf 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,21 +20,32 @@ Generally, these headers are specified as gRPC metadata. """ +from enum import Enum from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" -def to_routing_header(params): +def to_routing_header(params, qualified_enums=True): """Returns a routing header string for the given request parameters. Args: params (Mapping[str, Any]): A dictionary containing the request parameters used for routing. + qualified_enums (bool): Whether to represent enum values + as their type-qualified symbol names instead of as their + unqualified symbol names. Returns: str: The routing header string. + """ + if not qualified_enums: + if isinstance(params, dict): + tuples = params.items() + else: + tuples = params + params = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] return urlencode( params, # Per Google API policy (go/api-url-encoding), / is not encoded. @@ -42,16 +53,19 @@ def to_routing_header(params): ) -def to_grpc_metadata(params): +def to_grpc_metadata(params, qualified_enums=True): """Returns the gRPC metadata containing the routing headers for the given request parameters. Args: params (Mapping[str, Any]): A dictionary containing the request parameters used for routing. + qualified_enums (bool): Whether to represent enum values + as their type-qualified symbol names instead of as their + unqualified symbol names. Returns: Tuple(str, str): The gRPC metadata containing the routing header key and value. """ - return (ROUTING_METADATA_KEY, to_routing_header(params)) + return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums)) diff --git a/noxfile.py b/noxfile.py index 9bc2d967..4dcae556 100644 --- a/noxfile.py +++ b/noxfile.py @@ -94,7 +94,6 @@ def default(session, install_grpc=True): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - # Install all test dependencies, then install this package in-place. session.install( "dataclasses", "mock", diff --git a/owlbot.py b/owlbot.py index 8a60b152..5a83032e 100644 --- a/owlbot.py +++ b/owlbot.py @@ -48,8 +48,6 @@ """, ) -s.replace(".github/workflows/lint.yml", "python-version: \"3.10\"", "python-version: \"3.7\"") - python.configure_previous_major_version_branches() s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 30378676..9d31eb39 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from enum import Enum + import pytest try: @@ -35,6 +37,35 @@ def test_to_routing_header_with_slashes(): assert value == "name=me/ep&book.read=1%262" +def test_enum_fully_qualified(): + class Message: + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + params = [("color", Message.Color.RED)] + value = routing_header.to_routing_header(params) + assert value == "color=Color.RED" + value = routing_header.to_routing_header(params, qualified_enums=True) + assert value == "color=Color.RED" + + +def test_enum_nonqualified(): + class Message: + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + params = [("color", Message.Color.RED), ("num", 5)] + value = routing_header.to_routing_header(params, qualified_enums=False) + assert value == "color=RED&num=5" + params = {"color": Message.Color.RED, "num": 5} + value = routing_header.to_routing_header(params, qualified_enums=False) + assert value == "color=RED&num=5" + + def test_to_grpc_metadata(): params = [("name", "meep"), ("book.read", "1")] metadata = routing_header.to_grpc_metadata(params) From cb5e2f2fd54fc85941bc3016c7c80d25cdc83a3a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 10:27:38 -0500 Subject: [PATCH 120/126] chore(main): release 2.11.0 (#468) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf80f36d..403b6359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.11.0](https://github.com/googleapis/python-api-core/compare/v2.10.2...v2.11.0) (2022-11-10) + + +### Features + +* Add support for Python 3.11 ([#466](https://github.com/googleapis/python-api-core/issues/466)) ([ff379e3](https://github.com/googleapis/python-api-core/commit/ff379e304c353bcab734e1c4706b74b356a1e932)) +* Allow representing enums with their unqualified symbolic names in headers ([#465](https://github.com/googleapis/python-api-core/issues/465)) ([522b98e](https://github.com/googleapis/python-api-core/commit/522b98ecc1ebd1c2280d3d7c73a02f6e4fb528d4)) + + +### Bug Fixes + +* Major refactoring of Polling, Retry and Timeout logic ([#462](https://github.com/googleapis/python-api-core/issues/462)) ([434253d](https://github.com/googleapis/python-api-core/commit/434253de16d9efdf984ddb64c409706cda1d5f82)) +* Require google-auth >= 2.14.1 ([#463](https://github.com/googleapis/python-api-core/issues/463)) ([7cc329f](https://github.com/googleapis/python-api-core/commit/7cc329fe1498b0a4285123448e4ea80c6a780d47)) + ## [2.10.2](https://github.com/googleapis/python-api-core/compare/v2.10.1...v2.10.2) (2022-10-08) diff --git a/google/api_core/version.py b/google/api_core/version.py index ad66f162..e6e35743 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.10.2" +__version__ = "2.11.0" From ce77381efefe1016600a0ef692ce2f246cc3f711 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:57:12 -0500 Subject: [PATCH 121/126] chore(python): update release script dependencies (#472) * chore(python): drop flake8-import-order in samples noxfile Source-Link: https://github.com/googleapis/synthtool/commit/6ed3a831cb9ff69ef8a504c353e098ec0192ad93 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:3abfa0f1886adaf0b83f07cb117b24a639ea1cb9cffe56d43280b977033563eb * drop flake8-import-order * lint * use python 3.9 for docs * resolve mypy error * update python version for lint * fix lint * fix lint Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .github/workflows/docs.yml | 4 +- .kokoro/docker/docs/Dockerfile | 12 ++-- .kokoro/requirements.in | 4 +- .kokoro/requirements.txt | 61 ++++++++++--------- .../operations_v1/transports/__init__.py | 3 +- noxfile.py | 8 +-- 7 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 12edee77..bb21147e 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:452901c74a22f9b9a3bd02bce780b8e8805c97270d424684bff809ce5be8c2a2 + digest: sha256:3abfa0f1886adaf0b83f07cb117b24a639ea1cb9cffe56d43280b977033563eb diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7092a139..e97d89e4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 238b87b9..f8137d0a 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -60,16 +60,16 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb -###################### Install python 3.8.11 +###################### Install python 3.9.13 -# Download python 3.8.11 -RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz +# Download python 3.9.13 +RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz # Extract files -RUN tar -xvf Python-3.8.11.tgz +RUN tar -xvf Python-3.9.13.tgz -# Install python 3.8.11 -RUN ./Python-3.8.11/configure --enable-optimizations +# Install python 3.9.13 +RUN ./Python-3.9.13/configure --enable-optimizations RUN make altinstall ###################### Install pip diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index 7718391a..cbd7e77f 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -5,4 +5,6 @@ typing-extensions twine wheel setuptools -nox \ No newline at end of file +nox +charset-normalizer<3 +click<8.1.0 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 31425f16..9c1b9be3 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -93,11 +93,14 @@ cffi==1.15.1 \ charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f - # via requests + # via + # -r requirements.in + # requests click==8.0.4 \ --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb # via + # -r requirements.in # gcp-docuploader # gcp-releasetool colorlog==6.7.0 \ @@ -156,9 +159,9 @@ gcp-docuploader==0.6.4 \ --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.9.1 \ - --hash=sha256:952f4055d5d986b070ae2a71c4410b250000f9cc5a1e26398fcd55a5bbc5a15f \ - --hash=sha256:d0d3c814a97c1a237517e837d8cfa668ced8df4b882452578ecef4a4e79c583b +gcp-releasetool==1.10.0 \ + --hash=sha256:72a38ca91b59c24f7e699e9227c90cbe4dd71b789383cb0164b088abae294c83 \ + --hash=sha256:8c7c99320208383d4bb2b808c6880eb7a81424afe7cdba3c8d84b25f4f0e097d # via -r requirements.in google-api-core==2.10.2 \ --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ @@ -166,9 +169,9 @@ google-api-core==2.10.2 \ # via # google-cloud-core # google-cloud-storage -google-auth==2.14.0 \ - --hash=sha256:1ad5b0e6eba5f69645971abb3d2c197537d5914070a8c6d30299dfdb07c5c700 \ - --hash=sha256:cf24817855d874ede2efd071aa22125445f555de1685b739a9782fcf408c2a3d +google-auth==2.14.1 \ + --hash=sha256:ccaa901f31ad5cbb562615eb8b664b3dd0bf5404a67618e642307f00613eda4d \ + --hash=sha256:f5d8701633bebc12e0deea4df8abd8aff31c28b355360597f7f2ee60f2e4d016 # via # gcp-releasetool # google-api-core @@ -178,9 +181,9 @@ google-cloud-core==2.3.2 \ --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a # via google-cloud-storage -google-cloud-storage==2.5.0 \ - --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ - --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 +google-cloud-storage==2.6.0 \ + --hash=sha256:104ca28ae61243b637f2f01455cc8a05e8f15a2a18ced96cb587241cdd3820f5 \ + --hash=sha256:4ad0415ff61abdd8bb2ae81c1f8f7ec7d91a1011613f2db87c614c550f97bfe9 # via gcp-docuploader google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ @@ -256,9 +259,9 @@ google-resumable-media==2.4.0 \ --hash=sha256:2aa004c16d295c8f6c33b2b4788ba59d366677c0a25ae7382436cb30f776deaa \ --hash=sha256:8d5518502f92b9ecc84ac46779bd4f09694ecb3ba38a3e7ca737a86d15cbca1f # via google-cloud-storage -googleapis-common-protos==1.56.4 \ - --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ - --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 +googleapis-common-protos==1.57.0 \ + --hash=sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46 \ + --hash=sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c # via google-api-core idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ @@ -269,6 +272,7 @@ importlib-metadata==5.0.0 \ --hash=sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43 # via # -r requirements.in + # keyring # twine jaraco-classes==3.2.3 \ --hash=sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158 \ @@ -284,9 +288,9 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.9.3 \ - --hash=sha256:69732a15cb1433bdfbc3b980a8a36a04878a6cfd7cb99f497b573f31618001c0 \ - --hash=sha256:69b01dd83c42f590250fe7a1f503fc229b14de83857314b1933a3ddbf595c4a5 +keyring==23.11.0 \ + --hash=sha256:3dd30011d555f1345dec2c262f0153f2f0ca6bca041fb1dc4588349bb4c0ac1e \ + --hash=sha256:ad192263e2cdd5f12875dedc2da13534359a7e760e77f8d04b50968a821c2361 # via # gcp-releasetool # twine @@ -350,9 +354,9 @@ pkginfo==1.8.3 \ --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c # via twine -platformdirs==2.5.2 \ - --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ - --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 +platformdirs==2.5.4 \ + --hash=sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7 \ + --hash=sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10 # via virtualenv protobuf==3.20.3 \ --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ @@ -381,7 +385,6 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core - # googleapis-common-protos py==1.11.0 \ --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 @@ -476,17 +479,17 @@ urllib3==1.26.12 \ # via # requests # twine -virtualenv==20.16.6 \ - --hash=sha256:186ca84254abcbde98180fd17092f9628c5fe742273c02724972a1d8a2035108 \ - --hash=sha256:530b850b523c6449406dfba859d6345e48ef19b8439606c5d74d7d3c9e14d76e +virtualenv==20.16.7 \ + --hash=sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e \ + --hash=sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29 # via nox webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 # via bleach -wheel==0.37.1 \ - --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ - --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 +wheel==0.38.4 \ + --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ + --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8 # via -r requirements.in zipp==3.10.0 \ --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \ @@ -494,7 +497,7 @@ zipp==3.10.0 \ # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==65.5.0 \ - --hash=sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17 \ - --hash=sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356 +setuptools==65.5.1 \ + --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ + --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f # via -r requirements.in diff --git a/google/api_core/operations_v1/transports/__init__.py b/google/api_core/operations_v1/transports/__init__.py index b443c078..df53e15e 100644 --- a/google/api_core/operations_v1/transports/__init__.py +++ b/google/api_core/operations_v1/transports/__init__.py @@ -14,14 +14,13 @@ # limitations under the License. # from collections import OrderedDict -from typing import Dict, Type from .base import OperationsTransport from .rest import OperationsRestTransport # Compile a registry of transports. -_transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]] +_transport_registry = OrderedDict() _transport_registry["rest"] = OperationsRestTransport __all__ = ( diff --git a/noxfile.py b/noxfile.py index 4dcae556..748485a7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,7 +61,7 @@ def lint(session): Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ - session.install("flake8", "flake8-import-order", BLACK_VERSION) + session.install("flake8", BLACK_VERSION) session.install(".") session.run( "black", @@ -171,7 +171,7 @@ def unit_wo_grpc(session): default(session, install_grpc=False) -@nox.session(python="3.8") +@nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" @@ -212,7 +212,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.9") def docs(session): """Build the docs for this library.""" @@ -234,7 +234,7 @@ def docs(session): ) -@nox.session(python="3.8") +@nox.session(python="3.9") def docfx(session): """Build the docfx yaml files for this library.""" From 758a30dcb9db08685cc54b34979f6fec687215c5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Dec 2022 18:45:55 +0000 Subject: [PATCH 122/126] update python version for docs --- .github/workflows/docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6dd32fec..ee212e97 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel From 8f15fb37320c6ca29a2dfdb81499f0f201a9d04c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Dec 2022 18:47:46 +0000 Subject: [PATCH 123/126] revert testing/constraints-3.8.txt --- testing/constraints-3.8.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index 1b5bb58e..e69de29b 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -1,2 +0,0 @@ -googleapis-common-protos==1.56.3 -protobuf==4.21.6 \ No newline at end of file From aac0a8c0c514f3d30b8e59185208efc0e444de06 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Dec 2022 18:50:19 +0000 Subject: [PATCH 124/126] lint --- google/api_core/grpc_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 6729e440..bb0f8fa1 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -28,6 +28,7 @@ try: import grpc_gcp + warnings.warn( """Support for grpcio-gcp is deprecated. This feature will be removed from `google-api-core` after January 1, 2024. If you need to From c289e4d567f412de2ca94bcbc495b7f390cbcbd6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Dec 2022 18:53:20 +0000 Subject: [PATCH 125/126] update python version for lint --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cabd0e5b..1c35951e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel From 147d09705bd9d7252d0cbb412f969a1bd19e0c34 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Dec 2022 18:54:55 +0000 Subject: [PATCH 126/126] update python version for lint --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 41767bf0..d7cba97c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -171,7 +171,7 @@ def unit_wo_grpc(session): default(session, install_grpc=False) -@nox.session(python="3.8") +@nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): """Verify that setup.py is valid (including RST check)."""