Play around with variable-length instructions (DRAFT!)

Lib/importlib/_bootstrap_external.py

@@ -413,6 +413,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative)
 #     Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative)
 #     Python 3.11a7 3494 (New location info table)
+
 #     Python 3.12a1 3500 (Remove PRECALL opcode)
 #     Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth)
 #     Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST)
@@ -423,13 +424,14 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.12a1 3507 (Set lineno of module's RESUME to 0)
 #     Python 3.12a1 3508 (Add CLEANUP_THROW)
 #     Python 3.12a1 3509 (Conditional jumps only jump forward)
-#     Python 3.12a1 3510 (FOR_ITER leaves iterator on the stack)
-#     Python 3.12a1 3511 (Add STOPITERATION_ERROR instruction)
-#     Python 3.12a1 3512 (Remove all unused consts from code objects)
-#     Python 3.12a1 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR)
-#     Python 3.12a1 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE)
-#     Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg)
-#     Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction)
+#     Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack)
+#     Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction)
+#     Python 3.12a2 3512 (Remove all unused consts from code objects)
+#     Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR)
+#     Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE)
+#     Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg)
+#     Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction)
+#     Python 3.12a5 3518 (Add MAKE_FUNCTION_FROM_CODE instruction)
 
 #     Python 3.13 will start with 3550
 
@@ -442,7 +444,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3518).to_bytes(2, 'little') + b'\r\n'
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 

Lib/opcode.py

@@ -219,6 +219,8 @@ def pseudo_op(name, op, real_ops):
 def_op('KW_NAMES', 172)
 hasconst.append(172)
 def_op('CALL_INTRINSIC_1', 173)
+def_op('MAKE_FUNCTION_FROM_CODE', 174)
+def_op('EXTENDED_ARG_3', 175)
 
 hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT])
 

Objects/codeobject.c

@@ -1508,6 +1508,10 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len)
         int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
         int caches = _PyOpcode_Caches[opcode];
         instructions[i].opcode = opcode;
+        // TODO: Generalize this
+        if (opcode == MAKE_FUNCTION_FROM_CODE) {
+            i++;
+        }
         while (caches--) {
             instructions[++i].opcode = CACHE;
             instructions[i].oparg = 0;

Objects/frameobject.c

@@ -112,6 +112,10 @@ get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i)
 {
     _Py_CODEUNIT word;
     unsigned int oparg = _Py_OPARG(codestr[i]);
+    if (i >= 1 && _Py_OPCODE(word = codestr[i-1]) == EXTENDED_ARG_3) {
+        // TODO: Support EXTENDED_ARG_3
+        Py_FatalError("EXTENDED_ARG_3 not supported in get_arg()");
+    }
     if (i >= 1 && _Py_OPCODE(word = codestr[i-1]) == EXTENDED_ARG) {
         oparg |= _Py_OPARG(word) << 8;
         if (i >= 2 && _Py_OPCODE(word = codestr[i-2]) == EXTENDED_ARG) {

Python/bytecodes.c

@@ -68,6 +68,8 @@ dummy_func(
     _PyInterpreterFrame *frame,
     unsigned char opcode,
     unsigned int oparg,
+    unsigned int oparg2,
+    unsigned int oparg3,
     _Py_atomic_int * const eval_breaker,
     _PyCFrame cframe,
     PyObject *names,
@@ -3122,6 +3124,7 @@ dummy_func(
 
         // error: MAKE_FUNCTION has irregular stack effect
         inst(MAKE_FUNCTION) {
+            Py_FatalError("MAKE_FUNCTION should not be used");
             PyObject *codeobj = POP();
             PyFunctionObject *func = (PyFunctionObject *)
                 PyFunction_New(codeobj, GLOBALS());
@@ -3152,6 +3155,40 @@ dummy_func(
             PUSH((PyObject *)func);
         }
 
+        // oparg: flags
+        // oparg2: unused
+        // oparg3: index of code object in consts
+        inst(MAKE_FUNCTION_FROM_CODE) {
+            PyObject *codeobj = GETITEM(consts, oparg3);
+            assert(PyCode_Check(codeobj));
+            PyFunctionObject *func = (PyFunctionObject *)
+                PyFunction_New(codeobj, GLOBALS());
+
+            if (func == NULL) {
+                goto error;
+            }
+
+            if (oparg & 0x08) {
+                assert(PyTuple_CheckExact(TOP()));
+                func->func_closure = POP();
+            }
+            if (oparg & 0x04) {
+                assert(PyTuple_CheckExact(TOP()));
+                func->func_annotations = POP();
+            }
+            if (oparg & 0x02) {
+                assert(PyDict_CheckExact(TOP()));
+                func->func_kwdefaults = POP();
+            }
+            if (oparg & 0x01) {
+                assert(PyTuple_CheckExact(TOP()));
+                func->func_defaults = POP();
+            }
+
+            func->func_version = ((PyCodeObject *)codeobj)->co_version;
+            PUSH((PyObject *)func);
+        }
+
         // stack effect: ( -- )
         inst(RETURN_GENERATOR) {
             assert(PyFunction_Check(frame->f_funcobj));
@@ -3298,6 +3335,28 @@ dummy_func(
             DISPATCH_GOTO();
         }
 
+        // Uses oparg as the upper part of the following two-word instruction
+        inst(EXTENDED_ARG_3) {
+            assert(oparg);
+            oparg3 = oparg;
+            assert(cframe.use_tracing == 0);
+            PRE_DISPATCH_GOTO();
+            // Decode the next two-word instruction
+            opcode = _Py_OPCODE(*next_instr);
+            oparg = _Py_OPARG(*next_instr);
+            next_instr++;
+            // oparg2 = _Py_OPCODE(*next_instr);  // Nothing uses it yet
+            oparg3 = oparg3 << 8 | _Py_OPARG(*next_instr);
+            frame->prev_instr = next_instr++;
+            // Jump into the middle of that instruction
+            switch (opcode) {
+                INSERT_EXTENDED_CASES();
+                default:
+                    Py_FatalError("Unexpected opcode in EXTENDED_ARG_3");
+                    while (1) { abort(); }
+            }
+        }
+
         // stack effect: ( -- )
         inst(CACHE) {
             Py_UNREACHABLE();

Python/ceval.c

@@ -155,7 +155,7 @@ lltrace_resume_frame(_PyInterpreterFrame *frame)
         fobj == NULL ||
         !PyFunction_Check(fobj)
     ) {
-        printf("\nResuming frame.");
+        printf("\nResuming frame.\n");
         return;
     }
     PyFunctionObject *f = (PyFunctionObject *)fobj;
@@ -726,6 +726,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     // for the big switch below (in combination with the EXTRA_CASES macro).
     uint8_t opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
+    // int oparg2;        /* Second opcode argument (from second word, if any) */
+    int oparg3;        /* Third opcode argument (from second word, if any) */
     _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
 #ifdef LLTRACE
     int lltrace = 0;
@@ -923,6 +925,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         }
         NEXTOPARG();
         PRE_DISPATCH_GOTO();
+        // TODO: Support EXTENDED_ARG_3.
+        assert(opcode != EXTENDED_ARG_3);
         // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms:
         while (opcode == EXTENDED_ARG) {
             // CPython hasn't ever traced the instruction after an EXTENDED_ARG.