chore: add async_rest extra for async rest dependencies (#2195)

googleapis · Sep 30, 2024 · 58191fd · 58191fd
1 parent 3f3cd2d
commit 58191fd
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 326 deletions.
diff --git a/.../templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2 b/.../templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2
@@ -7,30 +7,23 @@
 
 
 import google.auth
-{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2187): Add an `async_rest` extra for GAPICs and update the error message to include that the extra needs to be installed. #}
 try:
     import aiohttp # type: ignore
     from google.auth.aio.transport.sessions import AsyncAuthorizedSession # type: ignore
+    from google.api_core import rest_streaming_async # type: ignore
 {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #}
+{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2200): Add coverage for ImportError. #}
 except ImportError as e:  # pragma: NO COVER
-    {# TODO(https://github.com/googleapis/google-auth-library-python/pull/1577): Update the version of google-auth once the linked PR is merged. #}
-    raise ImportError("async rest transport requires google-auth >= 2.35.0 with aiohttp extra. Install google-auth with the aiohttp extra using `pip install google-auth[aiohttp]==2.35.0`.") from e
+    raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install {{ api.naming.warehouse_package_name }}[async_rest]`") from e
 
 from google.auth.aio import credentials as ga_credentials_async  # type: ignore
 
 from google.api_core import exceptions as core_exceptions
 from google.api_core import gapic_v1
 from google.api_core import retry_async as retries
 from google.api_core import rest_helpers
-{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2137): raise an import error if an older version of google.api.core is installed. #}
 from google.api_core import rest_streaming_async  # type: ignore
 
-try:
-    from google.api_core import rest_streaming_async # type: ignore
-    HAS_ASYNC_REST_SUPPORT_IN_CORE = True
-{# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #}
-except ImportError as e:  # pragma: NO COVER
-    raise ImportError("async rest transport requires google-api-core >= 2.20.0. Install google-api-core using `pip install google-api-core==2.35.0`.") from e
 
 from google.protobuf import json_format
 {% if service.has_lro %}

diff --git a/gapic/templates/noxfile.py.j2 b/gapic/templates/noxfile.py.j2
@@ -74,6 +74,7 @@ def unit(session, protobuf_implementation):
             "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
         },
     )
+{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2201) Add a `unit_rest_async` nox session to run tests with [async_rest] extra installed. #}
 
 @nox.session(python=ALL_PYTHON[-1])
 @nox.parametrize(

diff --git a/gapic/templates/setup.py.j2 b/gapic/templates/setup.py.j2
@@ -1,3 +1,5 @@
+{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove the following variable (and the condition later in this file) for async rest transport once support for it is GA. #}
+{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %}
 {% extends '_base.py.j2' %}
 {% from '_pypi_packages.j2' import pypi_packages %}
 {% block content %}
@@ -46,6 +48,14 @@ dependencies = [
     {% endif %}
     {% endfor %}
 ]
+extras = {
+{% if rest_async_io_enabled %}
+    "async_rest": [
+        "google-api-core[grpc] >= 2.20.0, < 3.0.0dev",
+        "google-auth[aiohttp] >= 2.35.0, <3.0.0dev"
+    ],
+{% endif %}
+}
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/{{ api.naming.warehouse_package_name }}"
 
 package_root = os.path.abspath(os.path.dirname(__file__))
@@ -88,6 +98,7 @@ setuptools.setup(
     packages=packages,
     python_requires=">=3.7",
     install_requires=dependencies,
+    extras_require=extras,
     include_package_data=True,
     zip_safe=False,
 )

diff --git a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2
@@ -33,10 +33,12 @@ from proto.marshal.rules import wrappers
 {% if rest_async_io_enabled %}
 try:
     import aiohttp  # type: ignore
-    HAS_AIOHTTP_INSTALLED = True
+    from google.auth.aio.transport.sessions import AsyncAuthorizedSession
+    from google.api_core import rest_streaming_async
+    HAS_ASYNC_REST_EXTRA = True
 {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #}
 except ImportError: # pragma: NO COVER
-    HAS_AIOHTTP_INSTALLED = False
+    HAS_ASYNC_REST_EXTRA = False
 {% endif %}{# if rest_async_io_enabled #}
 from requests import Response
 from requests import Request, PreparedRequest
@@ -46,24 +48,10 @@ from google.protobuf import json_format
 
 try:
     from google.auth.aio import credentials as ga_credentials_async
-    {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove this condition when async rest is GA. #}
-    {% if rest_async_io_enabled %}
-    from google.auth.aio.transport.sessions import AsyncAuthorizedSession
-    {% endif %}
     HAS_GOOGLE_AUTH_AIO = True
 {# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #}
 except ImportError: # pragma: NO COVER
     HAS_GOOGLE_AUTH_AIO = False
-{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove this condition when async rest is GA. #}
-{% if rest_async_io_enabled %}
-
-try:
-    from google.api_core import rest_streaming_async
-    HAS_ASYNC_REST_SUPPORT_IN_CORE = True
-{# NOTE: `pragma: NO COVER` is needed since the coverage for presubmits isn't combined. #}
-except ImportError: # pragma: NO COVER
-    HAS_ASYNC_REST_SUPPORT_IN_CORE = False
-{% endif %}
 
 {# Import the service itself as well as every proto module that it imports. #}
 {% filter sort_lines %}

diff --git a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2
@@ -1761,12 +1761,8 @@ def test_{{ method_name }}_rest_no_http_options():
 {% set transport_name = get_transport_name(transport, is_async) %}
 def test_transport_kind_{{ transport_name }}():
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     transport = {{ get_client(service, is_async) }}.get_transport_class("{{ transport_name }}")(
         credentials={{get_credentials(is_async)}}
@@ -1787,12 +1783,8 @@ def test_transport_kind_{{ transport_name }}():
 {{async_decorator}}
 {{async_prefix}}def test_transport_close_{{transport_name}}():
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -1808,12 +1800,8 @@ def test_transport_kind_{{ transport_name }}():
 {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121): Remove / Update this test macro when async rest is GA. #}
 {% macro async_rest_unsupported_params_test(service) %}
 def test_unsupported_parameter_rest_asyncio():
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     options = client_options.ClientOptions(quota_project_id="octopus")
     {# TODO(https://github.com/googleapis/gapic-generator-python/issues/2137): Remove `type: ignore` once we add a version check for google-api-core. #}
     with pytest.raises(core_exceptions.AsyncRestUnsupportedParameterError, match="google.api_core.client_options.ClientOptions.quota_project_id") as exc:  # type: ignore
@@ -1903,12 +1891,8 @@ def test_unsupported_parameter_rest_asyncio():
 {{async_decorator}}
 {{async_prefix}}def test_{{ method_name }}_{{transport_name}}_error():
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
 
     client = {{ get_client(service, is_async) }}(
@@ -1934,12 +1918,8 @@ def test_unsupported_parameter_rest_asyncio():
 {% set transport_name = get_transport_name(transport, is_async) %}
 def test_initialize_client_w_{{transport_name}}():
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -1966,12 +1946,8 @@ def test_initialize_client_w_{{transport_name}}():
     raise NotImplementedError("gRPC is currently not supported for this test case.")
 {% else %}{# 'rest' in transport #}
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -2016,13 +1992,8 @@ def test_initialize_client_w_{{transport_name}}():
     raise NotImplementedError("gRPC is currently not supported for this test case.")
 {% else %}{# 'rest' in transport #}
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
-
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -2086,13 +2057,8 @@ def test_initialize_client_w_{{transport_name}}():
     raise NotImplementedError("gRPC is currently not supported for this test case.")
 {% else %}{# 'rest' in transport #}
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
-
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -2314,13 +2280,8 @@ def test_initialize_client_w_{{transport_name}}():
     raise NotImplementedError("gRPC is currently not supported for this test case.")
 {% else %}{# 'rest' in transport #}
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
-
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     client = {{ get_client(service, is_async) }}(
         credentials={{get_credentials(is_async)}},
@@ -2422,13 +2383,8 @@ def test_initialize_client_w_{{transport_name}}():
     raise NotImplementedError("gRPC is currently not supported for this test case.")
 {% else %}{# 'rest' in transport #}
     {% if transport_name == 'rest_asyncio' %}
-    if not HAS_GOOGLE_AUTH_AIO:
-        pytest.skip("google-auth >= 2.35.0 is required for async rest transport.")
-    elif not HAS_AIOHTTP_INSTALLED:
-        pytest.skip("aiohttp is required for async rest transport.")
-    elif not HAS_ASYNC_REST_SUPPORT_IN_CORE:
-        pytest.skip("google-api-core >= 2.20.0 is required for async rest transport.")
-
+    if not HAS_ASYNC_REST_EXTRA:
+        pytest.skip("the library must be installed with the `async_rest` extra to test this feature.")
     {% endif %}
     transport = transports.{{async_method_prefix}}{{ service.name }}RestTransport(
         credentials={{get_credentials(is_async)}},

diff --git a/noxfile.py b/noxfile.py
@@ -304,7 +304,16 @@ def showcase_library(
             f"{tmp_dir}/testing/constraints-{session.python}.txt"
             )
             # Install the library with a constraints file.
-            session.install("-e", tmp_dir, "-r", constraints_path)
+            if session.python == "3.7":
+                session.install("-e", tmp_dir, "-r", constraints_path)
+                if rest_async_io_enabled:
+                    # NOTE: We re-install `google-api-core` and `google-auth` to override the respective
+                    # versions for each specified in constraints-3.7.txt. This is needed because async REST
+                    # is not supported with the minimum version of `google-api-core` and `google-auth`.
+                    session.install('--no-cache-dir', '--force-reinstall', "google-api-core==2.20.0")
+                    session.install('--no-cache-dir', '--force-reinstall', "google-auth[aiohttp]==2.35.0")
+            else:
+                session.install("-e", tmp_dir + ("[async_rest]" if rest_async_io_enabled else ""), "-r", constraints_path)
         else:
             # The ads templates do not have constraints files.
             # See https://github.com/googleapis/gapic-generator-python/issues/1788
@@ -363,8 +372,6 @@ def showcase_w_rest_async(
             ignore_path = test_directory / ignore_file
             pytest_command.extend(["--ignore", str(ignore_path)])
 
-        # Note: google-auth is re-installed here with aiohttp option to override the version installed in constraints.
-        session.install('--no-cache-dir', '--force-reinstall', "google-auth[aiohttp]")
         session.run(
             *pytest_command,
             env=env,
@@ -485,8 +492,6 @@ def showcase_unit_w_rest_async(
     """Run the generated unit tests with async rest transport against the Showcase library."""
     with showcase_library(session, templates=templates, other_opts=other_opts, rest_async_io_enabled=True) as lib:
         session.chdir(lib)
-        # Note: google-auth is re-installed here with aiohttp option to override the version installed in constraints.
-        session.install('--no-cache-dir', '--force-reinstall', "google-auth[aiohttp]")
         run_showcase_unit_tests(session, rest_async_io_enabled=True)
 
 

diff --git a/tests/integration/goldens/asset/setup.py b/tests/integration/goldens/asset/setup.py
@@ -49,6 +49,8 @@
     "google-cloud-os-config >= 1.0.0, <2.0.0dev",
     "grpc-google-iam-v1 >= 0.12.4, <1.0.0dev",
 ]
+extras = {
+}
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-asset"
 
 package_root = os.path.abspath(os.path.dirname(__file__))
@@ -91,6 +93,7 @@
     packages=packages,
     python_requires=">=3.7",
     install_requires=dependencies,
+    extras_require=extras,
     include_package_data=True,
     zip_safe=False,
 )
diff --git a/tests/integration/goldens/credentials/setup.py b/tests/integration/goldens/credentials/setup.py
@@ -46,6 +46,8 @@
     "proto-plus >= 1.22.3, <2.0.0dev",
     "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
 ]
+extras = {
+}
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-iam-credentials"
 
 package_root = os.path.abspath(os.path.dirname(__file__))
@@ -88,6 +90,7 @@
     packages=packages,
     python_requires=">=3.7",
     install_requires=dependencies,
+    extras_require=extras,
     include_package_data=True,
     zip_safe=False,
 )
diff --git a/tests/integration/goldens/eventarc/setup.py b/tests/integration/goldens/eventarc/setup.py
@@ -47,6 +47,8 @@
     "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     "grpc-google-iam-v1 >= 0.12.4, <1.0.0dev",
 ]
+extras = {
+}
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-eventarc"
 
 package_root = os.path.abspath(os.path.dirname(__file__))
@@ -89,6 +91,7 @@
     packages=packages,
     python_requires=">=3.7",
     install_requires=dependencies,
+    extras_require=extras,
     include_package_data=True,
     zip_safe=False,
 )
diff --git a/tests/integration/goldens/logging/setup.py b/tests/integration/goldens/logging/setup.py
@@ -46,6 +46,8 @@
     "proto-plus >= 1.22.3, <2.0.0dev",
     "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
 ]
+extras = {
+}
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-logging"
 
 package_root = os.path.abspath(os.path.dirname(__file__))
@@ -88,6 +90,7 @@
     packages=packages,
     python_requires=">=3.7",
     install_requires=dependencies,
+    extras_require=extras,
     include_package_data=True,
     zip_safe=False,
 )
diff --git a/...ation/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py b/...ation/goldens/redis/google/cloud/redis_v1/services/cloud_redis/transports/rest_asyncio.py
@@ -18,8 +18,9 @@
 try:
     import aiohttp # type: ignore
     from google.auth.aio.transport.sessions import AsyncAuthorizedSession # type: ignore
+    from google.api_core import rest_streaming_async # type: ignore
 except ImportError as e:  # pragma: NO COVER
-    raise ImportError("async rest transport requires google-auth >= 2.35.0 with aiohttp extra. Install google-auth with the aiohttp extra using `pip install google-auth[aiohttp]==2.35.0`.") from e
+    raise ImportError("`rest_asyncio` transport requires the library to be installed with the `async_rest` extra. Install the library with the `async_rest` extra using `pip install google-cloud-redis[async_rest]`") from e
 
 from google.auth.aio import credentials as ga_credentials_async  # type: ignore
 
@@ -29,11 +30,6 @@
 from google.api_core import rest_helpers
 from google.api_core import rest_streaming_async  # type: ignore
 
-try:
-    from google.api_core import rest_streaming_async # type: ignore
-    HAS_ASYNC_REST_SUPPORT_IN_CORE = True
-except ImportError as e:  # pragma: NO COVER
-    raise ImportError("async rest transport requires google-api-core >= 2.20.0. Install google-api-core using `pip install google-api-core==2.35.0`.") from e
 
 from google.protobuf import json_format
 from google.api_core import operations_v1