From 1c4a4cad97cfc4da21dbf759808fe3978e49e17e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 28 Apr 2023 23:01:00 +0100 Subject: [PATCH 1/4] gh-87092: Expose assembler to unit tests --- Include/internal/pycore_compile.h | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/support/bytecode_helper.py | 34 +++-- Lib/test/test_compiler_assemble.py | 72 +++++++++++ Modules/_testinternalcapi.c | 65 +++++++++- Modules/clinic/_testinternalcapi.c.h | 64 ++++++++- Python/compile.c | 121 +++++++++++++----- Python/flowgraph.c | 14 ++ 11 files changed, 332 insertions(+), 48 deletions(-) create mode 100644 Lib/test/test_compiler_assemble.py diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a032f652dddaf..4bd4ef57238f98 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -103,6 +103,10 @@ PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, PyObject *consts); +PyAPI_FUNC(PyCodeObject*) +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4fa15d74b3ad64..fa14b97a1495bd 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1027,6 +1027,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memlimit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(message)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e19d8ff1b50468..b3a196dc5feb00 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -515,6 +515,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(memlimit) STRUCT_FOR_ID(message) STRUCT_FOR_ID(metaclass) + STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 42c4874d9466bf..3b06cf65993c50 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1021,6 +1021,7 @@ extern "C" { INIT_ID(memlimit), \ INIT_ID(message), \ INIT_ID(metaclass), \ + INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(mod), \ INIT_ID(mode), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 6d9cd24d9f3a13..9064df4dc4e61f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1398,6 +1398,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(metaclass); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(metadata); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 1d9b889c920986..0308d85084bb58 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,7 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg +from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object _UNSPECIFIED = object() @@ -108,6 +108,18 @@ def normalize_insts(self, insts): res.append((opcode, arg, *loc)) return res + def complete_insts_info(self, insts): + # fill in omitted fields in location, and oparg 0 for ops with no arg. + res = [] + for item in insts: + assert isinstance(item, tuple) + inst = list(item) + opcode = dis.opmap[inst[0]] + oparg = inst[1] + loc = inst[2:] + [-1] * (6 - len(inst)) + res.append((opcode, oparg, *loc)) + return res + class CodegenTestCase(CompilationStepTestCase): @@ -118,20 +130,16 @@ def generate_code(self, ast): class CfgOptimizationTestCase(CompilationStepTestCase): - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] - for item in insts: - assert isinstance(item, tuple) - inst = list(reversed(item)) - opcode = dis.opmap[inst.pop()] - oparg = inst.pop() - loc = inst + [-1] * (4 - len(inst)) - res.append((opcode, oparg, *loc)) - return res - def get_optimized(self, insts, consts): insts = self.normalize_insts(insts) insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts + +class AssemblerTestCase(CompilationStepTestCase): + + def get_code_object(self, filename, insts, metadata): + co = assemble_code_object(filename, insts, metadata) + return co + + diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py new file mode 100644 index 00000000000000..1f73c3b993ad41 --- /dev/null +++ b/Lib/test/test_compiler_assemble.py @@ -0,0 +1,72 @@ + +import ast +import types + +from test.support.bytecode_helper import AssemblerTestCase, CfgOptimizationTestCase + +from _testinternalcapi import compiler_codegen, optimize_cfg + +# Tests for the code-object creation stage of the compiler. + +class IsolatedAssembleTests(AssemblerTestCase): + + def complete_metadata(self, metadata, filename="myfile.py"): + if metadata is None: + metadata = {} + for key in ['name', 'qualname']: + metadata.setdefault(key, key) + for key in ['consts']: + metadata.setdefault(key, []) + for key in ['names', 'varnames', 'cellvars', 'freevars']: + metadata.setdefault(key, {}) + for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: + metadata.setdefault(key, 0) + metadata.setdefault('firstlineno', 1) + metadata.setdefault('filename', filename) + return metadata + + def assemble_test(self, insts, metadata, expected): + co = self.get_code_object(metadata['filename'], insts, metadata) + self.assertIsInstance(co, types.CodeType) + + expected_metadata = {} + for key, value in metadata.items(): + if isinstance(value, list): + expected_metadata[key] = tuple(value) + elif isinstance(value, dict): + expected_metadata[key] = tuple(value.keys()) + else: + expected_metadata[key] = value + + for key, value in expected_metadata.items(): + self.assertEqual(getattr(co, "co_" + key), value) + + f = types.FunctionType(co, {}) + for args, res in expected.items(): + self.assertEqual(f(*args), res) + + def test_simple_expr(self): + metadata = { + 'filename' : 'avg.py', + 'name' : 'avg', + 'qualname' : 'stats.avg', + 'consts' : [2], + 'argcount' : 2, + 'varnames' : {'x' : 0, 'y' : 1}, + } + metadata = self.complete_metadata(metadata) + + # code for "return (x+y)/2" + insts = [ + ('RESUME', 0), + ('LOAD_FAST', 0, 1), # 'x' + ('LOAD_FAST', 1, 1), # 'y' + ('BINARY_OP', 0, 1), # '+' + ('LOAD_CONST', 0, 1), # 2 + ('BINARY_OP', 11, 1), # '/' + ('RETURN_VALUE', 1), + ] + insts = self.complete_insts_info(insts) + expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} + self.assemble_test(insts, metadata, expected) + diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 632fac2de0c419..1e38f1aa63499b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,7 +14,7 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -625,6 +625,68 @@ _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions, return _PyCompile_OptimizeCfg(instructions, consts); } +static int +get_nonnegative_int_from_dict(PyObject *dict, const char *key) { + PyObject *obj = PyDict_GetItemString(dict, key); + if (obj == NULL) { + return -1; + } + return PyLong_AsLong(obj); +} + +/*[clinic input] + +_testinternalcapi.assemble_code_object -> object + + filename: object + instructions: object + metadata: object + +Create a code object for the given instructions. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata) +/*[clinic end generated code: output=38003dc16a930f48 input=e713ad77f08fb3a8]*/ + +{ + assert(PyDict_Check(metadata)); + _PyCompile_CodeUnitMetadata umd; + + umd.u_name = PyDict_GetItemString(metadata, "name"); + umd.u_qualname = PyDict_GetItemString(metadata, "qualname"); + + assert(PyUnicode_Check(umd.u_name)); + assert(PyUnicode_Check(umd.u_qualname)); + + umd.u_consts = PyDict_GetItemString(metadata, "consts"); + umd.u_names = PyDict_GetItemString(metadata, "names"); + umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); + umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); + umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); + + assert(PyList_Check(umd.u_consts)); + assert(PyDict_Check(umd.u_names)); + assert(PyDict_Check(umd.u_varnames)); + assert(PyDict_Check(umd.u_cellvars)); + assert(PyDict_Check(umd.u_freevars)); + + umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); + umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); + umd.u_kwonlyargcount = get_nonnegative_int_from_dict(metadata, "kwonlyargcount"); + umd.u_firstlineno = get_nonnegative_int_from_dict(metadata, "firstlineno"); + + assert(umd.u_argcount >= 0); + assert(umd.u_posonlyargcount >= 0); + assert(umd.u_kwonlyargcount >= 0); + assert(umd.u_firstlineno >= 0); + + return (PyObject*)_PyCompile_Assemble(&umd, filename, instructions); +} + static PyObject * get_interp_settings(PyObject *self, PyObject *args) @@ -705,6 +767,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF + _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, {"clear_extension", clear_extension, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index e8d5681b194916..89573222572594 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -128,4 +128,66 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=efe95836482fd542 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testinternalcapi_assemble_code_object__doc__, +"assemble_code_object($module, /, filename, instructions, metadata)\n" +"--\n" +"\n" +"Create a code object for the given instructions."); + +#define _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF \ + {"assemble_code_object", _PyCFunction_CAST(_testinternalcapi_assemble_code_object), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_assemble_code_object__doc__}, + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata); + +static PyObject * +_testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filename), &_Py_ID(instructions), &_Py_ID(metadata), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filename", "instructions", "metadata", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "assemble_code_object", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *filename; + PyObject *instructions; + PyObject *metadata; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + filename = args[0]; + instructions = args[1]; + metadata = args[2]; + return_value = _testinternalcapi_assemble_code_object_impl(module, filename, instructions, metadata); + +exit: + return return_value; +} +/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e8789def867308..cbe5403aafbc87 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -308,7 +308,6 @@ instr_sequence_fini(instr_sequence *seq) { seq->s_instrs = NULL; } - static int instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { memset(g, 0, sizeof(cfg_builder)); @@ -6754,11 +6753,11 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) static int * -build_cellfixedoffsets(struct compiler_unit *u) +build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; int *fixed = PyMem_New(int, noffsets); @@ -6772,8 +6771,8 @@ build_cellfixedoffsets(struct compiler_unit *u) PyObject *varname, *cellindex; Py_ssize_t pos = 0; - while (PyDict_Next(u->u_metadata.u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(u->u_metadata.u_varnames, varname); + while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); @@ -6787,17 +6786,17 @@ build_cellfixedoffsets(struct compiler_unit *u) } static int -insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, +insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixed, int nfreevars, int code_flags) { - assert(u->u_metadata.u_firstlineno > 0); + assert(umd->u_firstlineno > 0); /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(u->u_metadata.u_firstlineno, u->u_metadata.u_firstlineno, -1, -1), + .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen)); @@ -6811,12 +6810,12 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); + const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); if (ncellvars) { - // u->u_metadata.u_cellvars has the cells out of order so we sort them + // umd->u_cellvars has the cells out of order so we sort them // before adding the MAKE_CELL instructions. Note that we // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); + const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); if (sorted == NULL) { PyErr_NoMemory(); @@ -6860,11 +6859,11 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } static int -fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) +fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; // First deal with duplicates (arg cells). @@ -6906,30 +6905,30 @@ fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) static int -prepare_localsplus(struct compiler_unit* u, cfg_builder *g, int code_flags) +prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) { - assert(PyDict_GET_SIZE(u->u_metadata.u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); assert(INT_MAX - nlocals - ncellvars > 0); assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(u); + int* cellfixedoffsets = build_cellfixedoffsets(umd); if (cellfixedoffsets == NULL) { return ERROR; } // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(u, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { PyMem_Free(cellfixedoffsets); return ERROR; } - int numdropped = fix_cell_offsets(u, g->g_entryblock, cellfixedoffsets); + int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); PyMem_Free(cellfixedoffsets); // At this point we're done with it. cellfixedoffsets = NULL; if (numdropped < 0) { @@ -6980,7 +6979,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, } /** Assembly **/ - int nlocalsplus = prepare_localsplus(u, &g, code_flags); + int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -7157,11 +7156,6 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) goto error; } } - if (seq->s_used && !IS_TERMINATOR_OPCODE(seq->s_instrs[seq->s_used-1].i_opcode)) { - if (instr_sequence_addop(seq, RETURN_VALUE, 0, NO_LOCATION) < 0) { - goto error; - } - } PyMem_Free(is_target); return SUCCESS; error: @@ -7328,6 +7322,67 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) return res; } +int _PyCfg_JumpLabelsToTargets(basicblock *entryblock); + +PyCodeObject * +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions) +{ + PyCodeObject *co = NULL; + instr_sequence optimized_instrs; + memset(&optimized_instrs, 0, sizeof(instr_sequence)); + + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + cfg_builder g; + if (instructions_to_cfg(instructions, &g) < 0) { + goto error; + } + + if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) { + goto error; + } + + int code_flags = 0; + int nlocalsplus = prepare_localsplus(umd, &g, code_flags); + if (nlocalsplus < 0) { + goto error; + } + + int maxdepth = _PyCfg_Stackdepth(g.g_entryblock, code_flags); + if (maxdepth < 0) { + goto error; + } + + _PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock); + + /* Order of basic blocks must have been determined by now */ + + if (_PyCfg_ResolveJumps(&g) < 0) { + goto error; + } + + /* Can't modify the bytecode after computing jump offsets. */ + + if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { + goto error; + } + + PyObject *consts = umd->u_consts; + co = _PyAssemble_MakeCodeObject(umd, const_cache, + consts, maxdepth, &optimized_instrs, + nlocalsplus, code_flags, filename); + +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Fini(&g); + instr_sequence_fini(&optimized_instrs); + return co; +} + /* Retained for API compatibility. * Optimization is now done in _PyCfg_OptimizeCodeUnit */ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 6f83a910cab392..f79afb4c66cd8b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -223,6 +223,15 @@ dump_basicblock(const basicblock *b) } } } + +void +_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +{ + for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { + dump_basicblock(b); + } +} + #endif @@ -592,6 +601,11 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } +int +_PyCfg_JumpLabelsToTargets(basicblock *entryblock) +{ + return translate_jump_labels_to_targets(entryblock); +} static int mark_except_handlers(basicblock *entryblock) { From b334d44f7168bf11b7162fc418b12f53c74315e4 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 1 May 2023 21:40:34 +0100 Subject: [PATCH 2/4] move complete_metadata, complete_insts_info to assemble_test --- Lib/test/test_compiler_assemble.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 1f73c3b993ad41..5ed6f4d46ab53e 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -26,6 +26,9 @@ def complete_metadata(self, metadata, filename="myfile.py"): return metadata def assemble_test(self, insts, metadata, expected): + metadata = self.complete_metadata(metadata) + insts = self.complete_insts_info(insts) + co = self.get_code_object(metadata['filename'], insts, metadata) self.assertIsInstance(co, types.CodeType) @@ -54,7 +57,6 @@ def test_simple_expr(self): 'argcount' : 2, 'varnames' : {'x' : 0, 'y' : 1}, } - metadata = self.complete_metadata(metadata) # code for "return (x+y)/2" insts = [ @@ -66,7 +68,6 @@ def test_simple_expr(self): ('BINARY_OP', 11, 1), # '/' ('RETURN_VALUE', 1), ] - insts = self.complete_insts_info(insts) expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) From 58ce6d97cb34edcb2a109f8a96fde5f67978445c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 1 May 2023 21:41:43 +0100 Subject: [PATCH 3/4] whitespace --- Lib/test/support/bytecode_helper.py | 2 -- Lib/test/test_compiler_assemble.py | 1 - 2 files changed, 3 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 0308d85084bb58..357ec44dbc218d 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -141,5 +141,3 @@ class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): co = assemble_code_object(filename, insts, metadata) return co - - diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 5ed6f4d46ab53e..a8e679501d65ff 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -70,4 +70,3 @@ def test_simple_expr(self): ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) - From 1c00e20fe2552b3cbc3bd7b03b69072cccdf4e76 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 1 May 2023 21:43:31 +0100 Subject: [PATCH 4/4] remove unused imports --- Lib/test/test_compiler_assemble.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index a8e679501d65ff..96c1691e24a9c8 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -2,9 +2,8 @@ import ast import types -from test.support.bytecode_helper import AssemblerTestCase, CfgOptimizationTestCase +from test.support.bytecode_helper import AssemblerTestCase -from _testinternalcapi import compiler_codegen, optimize_cfg # Tests for the code-object creation stage of the compiler.