From 1784974b2f11f156aa8bec1b406d7068fd61dabf Mon Sep 17 00:00:00 2001 From: Andy Ye Date: Mon, 29 Aug 2022 16:12:22 -0400 Subject: [PATCH 1/4] Change jump offset multiplier --- .../apache_beam/typehints/trivial_inference.py | 18 +++++++++++++----- .../typehints/trivial_inference_test.py | 8 ++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index 4de89ffd83ba..a9fd4a8e74ca 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -376,6 +376,14 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): inst_size = 2 opt_arg_size = 0 + # Python 3.10: bpo-27129 changes to instruction offsets, not byte offsets. The + # offsets were halved (16 bits fro instructions vs 8 bits for bytes), so we + # have to double the value of arg. + if (sys.version_info.major, sys.version_info.minor) == (3, 10): + jump_multiplier = 2 + else: + jump_multiplier = 1 + last_pc = -1 while pc < end: # pylint: disable=too-many-nested-blocks start = pc @@ -495,23 +503,23 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): elif opname == 'YIELD_VALUE': yields.add(state.stack[-1]) elif opname == 'JUMP_FORWARD': - jmp = pc + arg + jmp = pc + arg * jump_multiplier jmp_state = state state = None elif opname == 'JUMP_ABSOLUTE': - jmp = arg + jmp = arg * jump_multiplier jmp_state = state state = None elif opname in ('POP_JUMP_IF_TRUE', 'POP_JUMP_IF_FALSE'): state.stack.pop() - jmp = arg + jmp = arg * jump_multiplier jmp_state = state.copy() elif opname in ('JUMP_IF_TRUE_OR_POP', 'JUMP_IF_FALSE_OR_POP'): - jmp = arg + jmp = arg * jump_multiplier jmp_state = state.copy() state.stack.pop() elif opname == 'FOR_ITER': - jmp = pc + arg + jmp = pc + arg * jump_multiplier jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 4cb3e1b04ee9..7e69407dba5c 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -36,6 +36,14 @@ def assertReturnType(self, expected, f, inputs=(), depth=5): expected, trivial_inference.infer_return_type(f, inputs, debug=True, depth=depth)) + # The meaning of Jump Offsets in Python 3.10 was changed. + # https://github.com/python/cpython/issues/71316 + # Reported as a bug in Beam https://github.com/apache/beam/issues/21671 + def testJumpOffsets(self): + fn = lambda x: False + wrapper = lambda x, *args, **kwargs: [x] if fn(x, *args, **kwargs) else [] + self.assertReturnType(typehints.List[int], wrapper, [int]) + def testBuildListUnpack(self): # Lambda uses BUILD_LIST_UNPACK opcode in Python 3. self.assertReturnType( From db3edad44115b1337a0b6f1c81d227719cc3649a Mon Sep 17 00:00:00 2001 From: Andy Ye Date: Mon, 29 Aug 2022 16:13:07 -0400 Subject: [PATCH 2/4] Add GEN_START --- sdks/python/apache_beam/typehints/opcodes.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index 560f360caae5..f09f84701e4d 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -421,6 +421,12 @@ def delete_fast(state, arg): state.vars[arg] = Any # really an error +# bpo-43683 Adds GEN_START in Python 3.10, but removed in Python 3.11 +# https://github.com/python/cpython/pull/25138 +def gen_start(state, arg): + assert len(state.stack) == 0 + + def load_closure(state, arg): state.stack.append(state.get_closure(arg)) From 809a98dae5fe54cf2cfde1e4f4c4c66618492fa3 Mon Sep 17 00:00:00 2001 From: Andy Ye Date: Thu, 1 Sep 2022 09:51:40 -0400 Subject: [PATCH 3/4] Update comment --- sdks/python/apache_beam/typehints/trivial_inference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index a9fd4a8e74ca..a3a5b70b5e64 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -376,9 +376,9 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): inst_size = 2 opt_arg_size = 0 - # Python 3.10: bpo-27129 changes to instruction offsets, not byte offsets. The - # offsets were halved (16 bits fro instructions vs 8 bits for bytes), so we - # have to double the value of arg. + # Python 3.10: bpo-27129 changes jump offsets to use instruction offsets, + # not byte offsets. The offsets were halved (16 bits fro instructions vs 8 + # bits for bytes), so we have to double the value of arg. if (sys.version_info.major, sys.version_info.minor) == (3, 10): jump_multiplier = 2 else: From 830aaa322842dde61242b9e4f2e8c556f4853b2d Mon Sep 17 00:00:00 2001 From: Andy Ye Date: Fri, 2 Sep 2022 17:04:41 -0400 Subject: [PATCH 4/4] Update comment --- sdks/python/apache_beam/typehints/trivial_inference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index a3a5b70b5e64..c9fb37bcba44 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -376,10 +376,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): inst_size = 2 opt_arg_size = 0 - # Python 3.10: bpo-27129 changes jump offsets to use instruction offsets, - # not byte offsets. The offsets were halved (16 bits fro instructions vs 8 + # Python 3.10+: bpo-27129 changes jump offsets to use instruction offsets, + # not byte offsets. The offsets were halved (16 bits for instructions vs 8 # bits for bytes), so we have to double the value of arg. - if (sys.version_info.major, sys.version_info.minor) == (3, 10): + if (sys.version_info.major, sys.version_info.minor) >= (3, 10): jump_multiplier = 2 else: jump_multiplier = 1