diff --git a/CHANGES.md b/CHANGES.md index 94accd921ef8..ceb3a5540c6e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -54,6 +54,19 @@ ## I/Os ## New Features / Improvements +* Python SDK will now use Python 3 type annotations as pipeline type hints. +([#10717](https://github.com/apache/beam/pull/10717)) + + If you suspect that this feature is causing your pipeline to fail, calling + `apache_beam.typehints.disable_type_annotations()` before pipeline creation + will disable is completely, and decorating specific functions (such as + `process()`) with `@apache_beam.typehints.no_annotations` will disable it + for that function. + + More details will be in + [Ensuring Python Type Safety](https://beam.apache.org/documentation/sdks/python-type-safety/) + and an upcoming + [blog post](https://beam.apache.org/blog/python/typing/2020/03/06/python-typing.html). ## Breaking Changes diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index 437dabf92577..dad0a31f6403 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -120,6 +120,7 @@ def foo((a, b)): funcsigs = None __all__ = [ + 'disable_type_annotations', 'no_annotations', 'with_input_types', 'with_output_types', @@ -138,8 +139,7 @@ def foo((a, b)): _ANY_VAR_POSITIONAL = typehints.Tuple[typehints.Any, ...] _ANY_VAR_KEYWORD = typehints.Dict[typehints.Any, typehints.Any] -# TODO(BEAM-8280): Remove this when from_callable is ready to be enabled. -_enable_from_callable = False +_disable_from_callable = False try: _original_getfullargspec = inspect.getfullargspec @@ -231,6 +231,16 @@ def no_annotations(fn): return fn +def disable_type_annotations(): + """Prevent Beam from using type hint annotations to determine input and output + types of transforms. + + This setting applies globally. + """ + global _disable_from_callable + _disable_from_callable = True + + class IOTypeHints(NamedTuple( 'IOTypeHints', [('input_types', Optional[Tuple[Tuple[Any, ...], Dict[str, Any]]]), @@ -298,7 +308,7 @@ def from_callable(cls, fn): Returns: A new IOTypeHints or None if no annotations found. """ - if not _enable_from_callable or getattr(fn, '_beam_no_annotations', False): + if _disable_from_callable or getattr(fn, '_beam_no_annotations', False): return None signature = get_signature(fn) if (all(param.annotation == param.empty diff --git a/sdks/python/apache_beam/typehints/decorators_test.py b/sdks/python/apache_beam/typehints/decorators_test.py index f8d5923053cb..766cd6aa9ea8 100644 --- a/sdks/python/apache_beam/typehints/decorators_test.py +++ b/sdks/python/apache_beam/typehints/decorators_test.py @@ -33,8 +33,6 @@ from apache_beam.typehints import decorators from apache_beam.typehints import typehints -decorators._enable_from_callable = True - class IOTypeHintsTest(unittest.TestCase): def test_get_signature(self): @@ -221,5 +219,15 @@ def __init__(self): Subclass().with_input_types(str)._type_hints, Subclass._type_hints) +class DecoratorsTest(unittest.TestCase): + def tearDown(self): + decorators._disable_from_callable = False + + def test_disable_type_annotations(self): + self.assertFalse(decorators._disable_from_callable) + decorators.disable_type_annotations() + self.assertTrue(decorators._disable_from_callable) + + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/typehints/decorators_test_py3.py b/sdks/python/apache_beam/typehints/decorators_test_py3.py index fb31bc7ba84e..024a0d205b53 100644 --- a/sdks/python/apache_beam/typehints/decorators_test_py3.py +++ b/sdks/python/apache_beam/typehints/decorators_test_py3.py @@ -37,7 +37,6 @@ from apache_beam.typehints import TypeVariable from apache_beam.typehints import decorators -decorators._enable_from_callable = True T = TypeVariable('T') # Name is 'T' so it converts to a beam type with the same name. # mypy requires that the name of the variable match, so we must ignore this. @@ -194,5 +193,23 @@ def fn(a: int) -> int: _ = ['a', 'b', 'c'] | Map(fn) +class DecoratorsTest(unittest.TestCase): + def test_no_annotations(self): + def fn(a: int) -> int: + return a + + _ = [1, 2, 3] | Map(fn) # Doesn't raise - correct types. + + with self.assertRaisesRegex(TypeCheckError, + r'requires .*int.* but got .*str'): + _ = ['a', 'b', 'c'] | Map(fn) + + @decorators.no_annotations + def fn2(a: int) -> int: + return a + + _ = ['a', 'b', 'c'] | Map(fn2) # Doesn't raise - no input type hints. + + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 6cfa2f96b5f1..96349feea069 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -36,15 +36,12 @@ from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to from apache_beam.typehints import WithTypeHints -from apache_beam.typehints import decorators from apache_beam.typehints.decorators import get_signature # These test often construct a pipeline as value | PTransform to test side # effects (e.g. errors). # pylint: disable=expression-not-assigned -decorators._enable_from_callable = True - class MainInputTest(unittest.TestCase): def test_bad_main_input(self): diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test_py3.py b/sdks/python/apache_beam/typehints/typed_pipeline_test_py3.py index 46fdf10e15f5..2f511ee98f9c 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test_py3.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test_py3.py @@ -26,9 +26,6 @@ import apache_beam as beam from apache_beam import typehints -from apache_beam.typehints import decorators - -decorators._enable_from_callable = True class MainInputTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/typehints/typehints_test_py3.py b/sdks/python/apache_beam/typehints/typehints_test_py3.py index b02e079333d6..a7c23f04d219 100644 --- a/sdks/python/apache_beam/typehints/typehints_test_py3.py +++ b/sdks/python/apache_beam/typehints/typehints_test_py3.py @@ -28,9 +28,6 @@ from apache_beam.transforms.core import DoFn from apache_beam.typehints import KV from apache_beam.typehints import Iterable -from apache_beam.typehints import decorators - -decorators._enable_from_callable = True class TestParDoAnnotations(unittest.TestCase):