bpo-45565: Specialize LOAD_ATTR_CLASS

Include/internal/pycore_object.h

@@ -192,6 +192,9 @@ void _PyObject_ClearInstanceAttributes(PyObject *self);
 void _PyObject_FreeInstanceAttributes(PyObject *self);
 int _PyObject_IsInstanceDictEmpty(PyObject *);
 
+PyObject *_PyType_FindNameInMRO(PyTypeObject *type, PyObject *name, int *error,
+    Py_ssize_t *mro_index);
+
 #ifdef __cplusplus
 }
 #endif

Lib/opcode.py

@@ -244,6 +244,7 @@ def jabs_op(name, op):
     "LOAD_ATTR_WITH_HINT",
     "LOAD_ATTR_SLOT",
     "LOAD_ATTR_MODULE",
+    "LOAD_ATTR_CLASS",
     "LOAD_GLOBAL_ADAPTIVE",
     "LOAD_GLOBAL_MODULE",
     "LOAD_GLOBAL_BUILTIN",

Misc/NEWS.d/next/Core and Builtins/2021-10-22-15-20-43.bpo-45565.rQvNzI.rst

@@ -0,0 +1,2 @@
+Improve ``LOAD_METHOD_CLASS`` specialization by adding more specialized
+forms.

Objects/typeobject.c

@@ -3776,8 +3776,9 @@ _PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def)
 /* Internal API to look for a name through the MRO, bypassing the method cache.
    This returns a borrowed reference, and might set an exception.
    'error' is set to: -1: error with exception; 1: error without exception; 0: ok */
-static PyObject *
-find_name_in_mro(PyTypeObject *type, PyObject *name, int *error)
+PyObject *
+_PyType_FindNameInMRO(PyTypeObject *type, PyObject *name, int *error,
+    Py_ssize_t *mro_index)
 {
     Py_ssize_t i, n;
     PyObject *mro, *res, *base, *dict;
@@ -3830,6 +3831,9 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error)
         }
     }
     *error = 0;
+    if (mro_index != NULL) {
+        *mro_index = i;
+    }
 done:
     Py_DECREF(mro);
     return res;
@@ -3858,7 +3862,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name)
     /* We may end up clearing live exceptions below, so make sure it's ours. */
     assert(!PyErr_Occurred());
 
-    res = find_name_in_mro(type, name, &error);
+    res = _PyType_FindNameInMRO(type, name, &error, NULL);
     /* Only put NULL results into cache if there was no error. */
     if (error) {
         /* It's not ideal to clear the error condition,
@@ -8330,7 +8334,7 @@ update_one_slot(PyTypeObject *type, slotdef *p)
     assert(!PyErr_Occurred());
     do {
         /* Use faster uncached lookup as we won't get any cache hits during type setup. */
-        descr = find_name_in_mro(type, p->name_strobj, &error);
+        descr = _PyType_FindNameInMRO(type, p->name_strobj, &error, NULL);
         if (descr == NULL) {
             if (error == -1) {
                 /* It is unlikely but not impossible that there has been an exception

Python/ceval.c

@@ -3750,6 +3750,40 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             DISPATCH();
         }
 
+        TARGET(LOAD_ATTR_CLASS) {
+            assert(cframe.use_tracing == 0);
+            PyObject *owner = TOP();
+            PyObject *res;
+            SpecializedCacheEntry *caches = GET_CACHE();
+            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
+            _PyAttrCache *cache1 = &caches[-1].attr;
+            _PyObjectCache *cache2 = &caches[-2].obj;
+            assert(cache1->tp_version != 0);
+            DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
+            PyTypeObject *tp = (PyTypeObject *)owner;
+
+            DEOPT_IF(tp->tp_mro != cache2->obj, LOAD_ATTR);
+            PyTypeObject *real_owner = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro,
+                cache0->index);
+            assert(PyType_Check(real_owner));
+            PyDictObject *dict = (PyDictObject *)real_owner->tp_dict;
+            assert(dict != NULL);
+            assert(PyDict_CheckExact((PyObject *)dict));
+            DEOPT_IF(dict->ma_keys->dk_version != cache1->tp_version, LOAD_ATTR);
+            PyObject *name = GETITEM(names, cache0->original_oparg);
+            uint32_t hint = cache1->dk_version_or_hint;
+            DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
+            PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
+            DEOPT_IF(ep->me_key != name, LOAD_ATTR);
+            res = ep->me_value;
+            DEOPT_IF(res == NULL, LOAD_ATTR);
+            STAT_INC(LOAD_ATTR, hit);
+            Py_INCREF(res);
+            SET_TOP(res);
+            Py_DECREF(owner);
+            DISPATCH();
+        }
+
         TARGET(STORE_ATTR_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             SpecializedCacheEntry *cache = GET_CACHE();

Python/specialize.c

@@ -9,6 +9,7 @@
 
 #include <stdlib.h> // rand()
 
+
 /* For guidance on adding or extending families of instructions see
  * ./adaptive.md
  */
@@ -243,7 +244,7 @@ static uint8_t adaptive_opcodes[256] = {
 
 /* The number of cache entries required for a "family" of instructions. */
 static uint8_t cache_requirements[256] = {
-    [LOAD_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
+    [LOAD_ATTR] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache*/
     [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
     [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
     [BINARY_ADD] = 0,
@@ -688,13 +689,87 @@ specialize_dict_access(
     }
 }
 
+#if COLLECT_SPECIALIZATION_STATS_DETAILED
+static int load_method_fail_kind(DesciptorClassification kind);
+#endif
+
+static int
+specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
+    _PyAdaptiveEntry *cache0, _PyAttrCache *cache1, _PyObjectCache *cache2)
+{
+    PyTypeObject *tp = (PyTypeObject *)owner;
+    Py_ssize_t mro_index;
+    int error;
+    if (tp->tp_dict == NULL) {
+        if (PyType_Ready(tp) < 0) {
+            return 1;
+        }
+    }
+    PyObject *descr = NULL;
+    DesciptorClassification kind = analyze_descriptor(tp, name, &descr, 0);
+    switch (kind) {
+        case METHOD:
+        case NON_DESCRIPTOR:
+            break;
+        default:
+            SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
+            return 1;
+    }
+    if (!PyType_HasFeature(tp, Py_TPFLAGS_VALID_VERSION_TAG)) {
+        return 1;
+    }
+    PyObject *res = _PyType_FindNameInMRO(tp, name, &error, &mro_index);
+    if (res == NULL || error) {
+        if (error == -1) {
+            PyErr_Clear();
+        }
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
+        return 1;
+    }
+    if (mro_index != (uint16_t)mro_index) {
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
+        return 1;
+    }
+    assert(tp->tp_mro != NULL);
+    assert(PyTuple_CheckExact(tp->tp_mro));
+    assert(mro_index >= 0);
+    assert(mro_index < PyTuple_GET_SIZE(tp->tp_mro));
+
+    PyObject *real_owner = PyTuple_GET_ITEM(tp->tp_mro, mro_index);
+    assert(PyType_Check(real_owner));
+    PyDictObject *real_owner_dict = (PyDictObject *)((PyTypeObject *)real_owner)->tp_dict;
+    PyObject *value = NULL;
+    Py_ssize_t hint = _PyDict_GetItemHint(real_owner_dict, name, -1, &value);
+    assert(hint != DKIX_ERROR);
+    assert(hint != DKIX_EMPTY);
+    if (hint != (uint32_t)hint) {
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
+        return -1;
+    }
+    uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
+        real_owner_dict->ma_keys);
+    if (keys_version == 0) {
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+        return -1;
+    }
+    cache0->index = (uint16_t)mro_index;
+    cache1->dk_version_or_hint = (uint32_t)hint;
+    cache1->tp_version = keys_version;
+    cache2->obj = tp->tp_mro;
+    *instr = _Py_MAKECODEUNIT(LOAD_ATTR_CLASS, _Py_OPARG(*instr));
+    return 0;
+}
+
 int
 _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache)
 {
     _PyAdaptiveEntry *cache0 = &cache->adaptive;
     _PyAttrCache *cache1 = &cache[-1].attr;
+    _PyObjectCache *cache2 = &cache[-2].obj;
+
+    int err;
     if (PyModule_CheckExact(owner)) {
-        int err = specialize_module_load_attr(owner, instr, name, cache0, cache1,
+        err = specialize_module_load_attr(owner, instr, name, cache0, cache1,
             LOAD_ATTR, LOAD_ATTR_MODULE);
         if (err) {
             goto fail;
@@ -707,6 +782,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
             return -1;
         }
     }
+    if (PyType_Check(owner)) {
+        err = specialize_class_load_attr(owner, instr, name, cache0, cache1, cache2);
+        if (err) {
+            goto fail;
+        }
+        goto success;
+    }
     PyObject *descr;
     DesciptorClassification kind = analyze_descriptor(type, name, &descr, 0);
     switch(kind) {
@@ -764,7 +846,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
         case ABSENT:
             break;
     }
-    int err = specialize_dict_access(
+    err = specialize_dict_access(
         owner, instr, type, kind, name, cache0, cache1,
         LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
     );