feat: port REST transport to Ads templates (#1003)

Add REST transport to Ads templates
Add REST transport tests to Ads generated unit tests
Boost google-api-core dependency version

Rewrite of REST transport call stubs to assist testing
Add fragment tests for Ads
Multiple bugfixes

.github/sync-repo-settings.yaml

@@ -22,11 +22,19 @@ branchProtectionRules:
  - 'showcase-unit (3.9)'
  - 'showcase-unit (3.9, _alternative_templates)'
  - 'showcase-unit-add-iam-methods'
+ - 'integration'
  - 'style-check'
  - 'unit (3.6)'
  - 'unit (3.7)'
  - 'unit (3.8)'
  - 'unit (3.9)'
+ - 'fragment (3.6)'
+ - 'fragment (3.7)'
+ - 'fragment (3.8)'
+ - 'fragment (3.9)'
+ - 'fragment (3.7, _alternative_templates)'
+ - 'fragment (3.8, _alternative_templates)'
+ - 'fragment (3.9, _alternative_templates)'
  requiredApprovingReviewCount: 1
  requiresCodeOwnerReviews: true
  requiresStrictStatusChecks: true

.github/workflows/tests.yaml

@@ -74,7 +74,7 @@ jobs:
  ./gapic-showcase run &
  cd -
  env:
- SHOWCASE_VERSION: 0.16.0
+ SHOWCASE_VERSION: 0.18.0
  - name: Install nox.
  run: python -m pip install nox
  - name: Install protoc 3.12.1.
@@ -144,7 +144,7 @@ jobs:
  cd ..
  nox -s ${{ matrix.target }}
  env:
- SHOWCASE_VERSION: 0.16.0
+ SHOWCASE_VERSION: 0.18.0
  # TODO(yon-mg): add compute unit tests
  showcase-unit:
  strategy:
@@ -183,7 +183,7 @@ jobs:
  - name: Run unit tests.
  run: nox -s showcase_unit${{ matrix.variant }}-${{ matrix.python }}
  env:
- SHOWCASE_VERSION: 0.16.0
+ SHOWCASE_VERSION: 0.18.0
  showcase-unit-add-iam-methods:
  runs-on: ubuntu-latest
  steps:
@@ -214,7 +214,7 @@ jobs:
  - name: Run unit tests.
  run: nox -s showcase_unit_add_iam_methods
  env:
- SHOWCASE_VERSION: 0.16.0
+ SHOWCASE_VERSION: 0.18.0
  showcase-mypy:
  runs-on: ubuntu-latest
  strategy:
@@ -248,7 +248,7 @@ jobs:
  - name: Typecheck the generated output.
  run: nox -s showcase_mypy${{ matrix.variant }}
  env:
- SHOWCASE_VERSION: 0.16.0
+ SHOWCASE_VERSION: 0.18.0
  snippetgen:
  runs-on: ubuntu-latest
  steps:
@@ -299,6 +299,10 @@ jobs:
  strategy:
  matrix:
  python: [3.6, 3.7, 3.8, 3.9]
+ variant: ['', _alternative_templates]
+ exclude:
+ - python: 3.6
+ variant: _alternative_templates
  runs-on: ubuntu-latest
  steps:
  - name: Cancel Previous Runs
@@ -319,7 +323,7 @@ jobs:
  run: |
  python -m pip install nox
  - name: Run fragment tests.
- run: nox -s fragment-${{ matrix.python }}
+ run: nox -s fragment${{ matrix.variant }}-${{ matrix.python }}
  integration:
  runs-on: ubuntu-latest
  container: gcr.io/gapic-images/googleapis-bazel:20210105

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/pagers.py.j2

@@ -7,7 +7,7 @@
 {# This lives within the loop in order to ensure that this template
  is empty if there are no paged methods.
  -#}
-from typing import Any, Callable, Iterable, Sequence, Tuple
+from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator
 
 {% filter sort_lines %}
 {% for method in service.methods.values() | selectattr('paged_result_field') %}
@@ -70,9 +70,18 @@ class {{ method.name }}Pager:
  self._response = self._method(self._request, metadata=self._metadata)
  yield self._response
 
- def __iter__(self) -> {{ method.paged_result_field.ident | replace('Sequence', 'Iterable') }}:
+ {% if method.paged_result_field.map %}
+ def __iter__(self) -> Iterator[Tuple[str, {{ method.paged_result_field.type.fields.get('value').ident }}]]:
+ for page in self.pages:
+ yield from page.{{ method.paged_result_field.name}}.items()
+
+ def get(self, key: str) -> Optional[{{ method.paged_result_field.type.fields.get('value').ident }}]:
+ return self._response.{{ method.paged_result_field.name }}.get(key)
+ {% else %}
+ def __iter__(self) -> {{ method.paged_result_field.ident | replace('Sequence', 'Iterator') }}:
  for page in self.pages:
  yield from page.{{ method.paged_result_field.name }}
+ {% endif %}
 
  def __repr__(self) -> str:
  return '{0}<{1!r}>'.format(self.__class__.__name__, self._response)

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/__init__.py.j2

@@ -6,16 +6,29 @@ from collections import OrderedDict
 from typing import Dict, Type
 
 from .base import {{ service.name }}Transport
+{% if 'grpc' in opts.transport %}
 from .grpc import {{ service.name }}GrpcTransport
-
+{% endif %}
+{% if 'rest' in opts.transport %}
+from .rest import {{ service.name }}RestTransport
+{% endif %}
 
 # Compile a registry of transports.
 _transport_registry = OrderedDict() # type: Dict[str, Type[{{ service.name }}Transport]]
+{% if 'grpc' in opts.transport %}
 _transport_registry['grpc'] = {{ service.name }}GrpcTransport
-
+{% endif %}
+{% if 'rest' in opts.transport %}
+_transport_registry['rest'] = {{ service.name }}RestTransport
+{% endif %}
 
 __all__ = (
  '{{ service.name }}Transport',
+ {% if 'grpc' in opts.transport %}
  '{{ service.name }}GrpcTransport',
+ {% endif %}
+ {% if 'rest' in opts.transport %}
+ '{{ service.name }}RestTransport',
+ {% endif %}
 )
 {% endblock %}

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/base.py.j2

@@ -3,22 +3,29 @@
 {% block content %}
 
 import abc
-import typing
+from typing import Awaitable, Callable, Dict, Optional, Sequence, Union
 import pkg_resources
 
 import google.auth # type: ignore
-from google.api_core import gapic_v1
-from google.api_core import retry as retries
+import google.api_core # type: ignore
+from google.api_core import exceptions as core_exceptions # type: ignore
+from google.api_core import gapic_v1 # type: ignore
+from google.api_core import retry as retries # type: ignore
 {% if service.has_lro %}
 from google.api_core import operations_v1
 {% endif %}
 from google.auth import credentials as ga_credentials # type: ignore
+from google.oauth2 import service_account # type: ignore
 
 {% filter sort_lines %}
 {% for method in service.methods.values() %}
 {{ method.input.ident.python_import }}
 {{ method.output.ident.python_import }}
 {% endfor %}
+{% if opts.add_iam_methods %}
+from google.iam.v1 import iam_policy_pb2 # type: ignore
+from google.iam.v1 import policy_pb2 # type: ignore
+{% endif %}
 {% endfilter %}
 
 try:
@@ -31,7 +38,7 @@ except pkg_resources.DistributionNotFound:
  DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo()
 
 
-class {{ service.name }}Transport(metaclass=abc.ABCMeta):
+class {{ service.name }}Transport(abc.ABC):
  """Abstract transport class for {{ service.name }}."""
 
  AUTH_SCOPES = (
@@ -40,11 +47,18 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
  {% endfor %}
  )
 
+ DEFAULT_HOST: str = {% if service.host %}'{{ service.host }}'{% else %}{{ '' }}{% endif %}
+
  def __init__(
  self, *,
- host: str{% if service.host %} = '{{ service.host }}'{% endif %},
+ host: str = DEFAULT_HOST,
  credentials: ga_credentials.Credentials = None,
+ credentials_file: Optional[str] = None,
+ scopes: Optional[Sequence[str]] = None,
+ quota_project_id: Optional[str] = None,
  client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
+ always_use_jwt_access: Optional[bool] = False,
+ **kwargs,
  ) -> None:
  """Instantiate the transport.
 
@@ -56,30 +70,54 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
  credentials identify the application to the service; if none
  are specified, the client will attempt to ascertain the
  credentials from the environment.
+ credentials_file (Optional[str]): A file with credentials that can
+ be loaded with :func:`google.auth.load_credentials_from_file`.
+ This argument is mutually exclusive with credentials.
+ scopes (Optional[Sequence[str]]): A list of scopes.
+ quota_project_id (Optional[str]): An optional project to use for billing
+ and quota.
  client_info (google.api_core.gapic_v1.client_info.ClientInfo):
  The client info used to send a user-agent string along with
  API requests. If ``None``, then default info will be used.
  Generally, you only need to set this if you're developing
  your own client library.
+ always_use_jwt_access (Optional[bool]): Whether self signed JWT should
+ be used for service account credentials.
  """
  # Save the hostname. Default to port 443 (HTTPS) if none is specified.
  if ':' not in host:
  host += ':443'
  self._host = host
 
+ scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES}
+
+ # Save the scopes.
+ self._scopes = scopes
+
  # If no credentials are provided, then determine the appropriate
  # defaults.
- if credentials is None:
- credentials, _ = google.auth.default(scopes=self.AUTH_SCOPES)
+ if credentials and credentials_file:
+ raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive")
+
+ if credentials_file is not None:
+ credentials, _ = google.auth.load_credentials_from_file(
+ credentials_file,
+ **scopes_kwargs,
+ quota_project_id=quota_project_id
+ )
+ elif credentials is None:
+ credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id)
+
+ # If the credentials are service account credentials, then always try to use self signed JWT.
+ if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"):
+ credentials = credentials.with_always_use_jwt_access(True)
 
  # Save the credentials.
  self._credentials = credentials
 
- # Lifted into its own function so it can be stubbed out during tests.
- self._prep_wrapped_messages(client_info)
 
  def _prep_wrapped_messages(self, client_info):
- # Precomputed wrapped methods
+ # Precompute the wrapped methods.
  self._wrapped_methods = {
  {% for method in service.methods.values() %}
  self.{{ method.name|snake_case }}: gapic_v1.method.wrap_method(
@@ -106,7 +144,7 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
  def close(self):
  """Closes resources associated with the transport.
 
-  .. warning::
+ .. warning::
  Only call this method if the transport is NOT shared
  with other clients - this may cause errors in other clients!
  """
@@ -117,18 +155,53 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
  @property
  def operations_client(self):
  """Return the client designed to process long-running operations."""
- raise NotImplementedError
+ raise NotImplementedError()
  {% endif %}
  {% for method in service.methods.values() %}
 
  @property
- def {{ method.name|snake_case }}(self) -> typing.Callable[
+ def {{ method.name|snake_case }}(self) -> Callable[
  [{{ method.input.ident }}],
- {{ method.output.ident }}]:
- raise NotImplementedError
+ Union[
+ {{ method.output.ident }},
+ Awaitable[{{ method.output.ident }}]
+ ]]:
+ raise NotImplementedError()
  {% endfor %}
 
 
+ {% if opts.add_iam_methods %}
+ @property
+ def set_iam_policy(
+ self,
+ ) -> Callable[
+ [iam_policy_pb2.SetIamPolicyRequest],
+ Union[policy_pb2.Policy, Awaitable[policy_pb2.Policy]],
+ ]:
+ raise NotImplementedError()
+
+ @property
+ def get_iam_policy(
+ self,
+ ) -> Callable[
+ [iam_policy_pb2.GetIamPolicyRequest],
+ Union[policy_pb2.Policy, Awaitable[policy_pb2.Policy]],
+ ]:
+ raise NotImplementedError()
+
+ @property
+ def test_iam_permissions(
+ self,
+ ) -> Callable[
+ [iam_policy_pb2.TestIamPermissionsRequest],
+ Union[
+ iam_policy_pb2.TestIamPermissionsResponse,
+ Awaitable[iam_policy_pb2.TestIamPermissionsResponse],
+ ],
+ ]:
+ raise NotImplementedError()
+ {% endif %}
+
 __all__ = (
  '{{ service.name }}Transport',
 )