gh-108216: Add pycore_dict_struct.h internal header file

Include/internal/pycore_dict.h

@@ -1,4 +1,3 @@
-
 #ifndef Py_INTERNAL_DICT_H
 #define Py_INTERNAL_DICT_H
 #ifdef __cplusplus
@@ -9,8 +8,12 @@ extern "C" {
 # error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_dict_state.h" // DICT_MAX_WATCHERS
+#include "pycore_dict_struct.h" // export PyDictKeysObject and DKIX_EMPTY
+#include "pycore_interp.h" // PyInterpreter.dict_state
 #include "pycore_object.h" // PyDictOrValues
 
+
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
 
@@ -39,18 +42,6 @@ extern void _PyDict_Fini(PyInterpreterState *interp);
 
 /* other API */
 
-typedef struct {
- /* Cached hash code of me_key. */
- Py_hash_t me_hash;
- PyObject *me_key;
- PyObject *me_value; /* This field is only meaningful for combined tables */
-} PyDictKeyEntry;
-
-typedef struct {
- PyObject *me_key; /* The key must be Unicode and have hash. */
- PyObject *me_value; /* This field is only meaningful for combined tables */
-} PyDictUnicodeEntry;
-
 extern PyDictKeysObject *_PyDict_NewKeysForClass(void);
 extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
 
@@ -78,95 +69,10 @@ extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject
 
 extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObject *);
 
-#define DKIX_EMPTY (-1)
-#define DKIX_DUMMY (-2) /* Used internally */
-#define DKIX_ERROR (-3)
-#define DKIX_KEY_CHANGED (-4) /* Used internally */
-
-typedef enum {
- DICT_KEYS_GENERAL = 0,
- DICT_KEYS_UNICODE = 1,
- DICT_KEYS_SPLIT = 2
-} DictKeysKind;
-
-/* See dictobject.c for actual layout of DictKeysObject */
-struct _dictkeysobject {
- Py_ssize_t dk_refcnt;
-
- /* Size of the hash table (dk_indices). It must be a power of 2. */
- uint8_t dk_log2_size;
-
- /* Size of the hash table (dk_indices) by bytes. */
- uint8_t dk_log2_index_bytes;
-
- /* Kind of keys */
- uint8_t dk_kind;
-
- /* Version number -- Reset to 0 by any modification to keys */
- uint32_t dk_version;
-
- /* Number of usable entries in dk_entries. */
- Py_ssize_t dk_usable;
-
- /* Number of used entries in dk_entries. */
- Py_ssize_t dk_nentries;
-
- /* Actual hash table of dk_size entries. It holds indices in dk_entries,
- or DKIX_EMPTY(-1) or DKIX_DUMMY(-2).
-
- Indices must be: 0 <= indice < USABLE_FRACTION(dk_size).
-
- The size in bytes of an indice depends on dk_size:
-
- - 1 byte if dk_size <= 0xff (char*)
- - 2 bytes if dk_size <= 0xffff (int16_t*)
- - 4 bytes if dk_size <= 0xffffffff (int32_t*)
- - 8 bytes otherwise (int64_t*)
-
- Dynamically sized, SIZEOF_VOID_P is minimum. */
- char dk_indices[]; /* char is required to avoid strict aliasing. */
-
- /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows:
- see the DK_ENTRIES() macro */
-};
-
 /* This must be no more than 250, for the prefix size to fit in one byte. */
 #define SHARED_KEYS_MAX_SIZE 30
 #define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6
 
-/* Layout of dict values:
- *
- * The PyObject *values are preceded by an array of bytes holding
- * the insertion order and size.
- * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
- */
-struct _dictvalues {
- PyObject *values[1];
-};
-
-#define DK_LOG_SIZE(dk) _Py_RVALUE((dk)->dk_log2_size)
-#if SIZEOF_VOID_P > 4
-#define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk))
-#else
-#define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk))
-#endif
-
-static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
- int8_t *indices = (int8_t*)(dk->dk_indices);
- size_t index = (size_t)1 << dk->dk_log2_index_bytes;
- return (&indices[index]);
-}
-static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
- assert(dk->dk_kind == DICT_KEYS_GENERAL);
- return (PyDictKeyEntry*)_DK_ENTRIES(dk);
-}
-static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
- assert(dk->dk_kind != DICT_KEYS_GENERAL);
- return (PyDictUnicodeEntry*)_DK_ENTRIES(dk);
-}
-
-#define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL)
-
 #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
 #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
 
@@ -218,4 +124,4 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_INTERNAL_DICT_H */
+#endif // !Py_INTERNAL_DICT_H

Include/internal/pycore_dict_struct.h

@@ -0,0 +1,117 @@
+// Internal header which can be used by debuggers and profilers
+// to inspect a Python dictionary by reading memory, without executing
+// Python functions. Only define static inline functions to inspect a
+// dictionary. Sub-set of pycore_dict.h which should be usable in C++
+// (gh-108216).
+
+#ifndef Py_INTERNAL_DICT_STRUCT_H
+#define Py_INTERNAL_DICT_STRUCT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+typedef struct {
+ /* Cached hash code of me_key. */
+ Py_hash_t me_hash;
+ PyObject *me_key;
+ PyObject *me_value; /* This field is only meaningful for combined tables */
+} PyDictKeyEntry;
+
+typedef struct {
+ PyObject *me_key; /* The key must be Unicode and have hash. */
+ PyObject *me_value; /* This field is only meaningful for combined tables */
+} PyDictUnicodeEntry;
+
+#define DKIX_EMPTY (-1)
+#define DKIX_DUMMY (-2) /* Used internally */
+#define DKIX_ERROR (-3)
+#define DKIX_KEY_CHANGED (-4) /* Used internally */
+
+typedef enum {
+ DICT_KEYS_GENERAL = 0,
+ DICT_KEYS_UNICODE = 1,
+ DICT_KEYS_SPLIT = 2
+} DictKeysKind;
+
+/* See dictobject.c for actual layout of DictKeysObject */
+struct _dictkeysobject {
+ Py_ssize_t dk_refcnt;
+
+ /* Size of the hash table (dk_indices). It must be a power of 2. */
+ uint8_t dk_log2_size;
+
+ /* Size of the hash table (dk_indices) by bytes. */
+ uint8_t dk_log2_index_bytes;
+
+ /* Kind of keys */
+ uint8_t dk_kind;
+
+ /* Version number -- Reset to 0 by any modification to keys */
+ uint32_t dk_version;
+
+ /* Number of usable entries in dk_entries. */
+ Py_ssize_t dk_usable;
+
+ /* Number of used entries in dk_entries. */
+ Py_ssize_t dk_nentries;
+
+ /* Actual hash table of dk_size entries. It holds indices in dk_entries,
+ or DKIX_EMPTY(-1) or DKIX_DUMMY(-2).
+
+ Indices must be: 0 <= indice < USABLE_FRACTION(dk_size).
+
+ The size in bytes of an indice depends on dk_size:
+
+ - 1 byte if dk_size <= 0xff (char*)
+ - 2 bytes if dk_size <= 0xffff (int16_t*)
+ - 4 bytes if dk_size <= 0xffffffff (int32_t*)
+ - 8 bytes otherwise (int64_t*)
+
+ Dynamically sized, SIZEOF_VOID_P is minimum. */
+ char dk_indices[]; /* char is required to avoid strict aliasing. */
+
+ /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows:
+ see the DK_ENTRIES() macro */
+};
+
+/* Layout of dict values:
+ *
+ * The PyObject *values are preceded by an array of bytes holding
+ * the insertion order and size.
+ * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
+ */
+struct _dictvalues {
+ PyObject *values[1];
+};
+
+#define DK_LOG_SIZE(dk) _Py_RVALUE((dk)->dk_log2_size)
+#if SIZEOF_VOID_P > 4
+#define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk))
+#else
+#define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk))
+#endif
+
+static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
+ int8_t *indices = (int8_t*)(dk->dk_indices);
+ size_t index = (size_t)1 << dk->dk_log2_index_bytes;
+ return (&indices[index]);
+}
+static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
+ assert(dk->dk_kind == DICT_KEYS_GENERAL);
+ return (PyDictKeyEntry*)_DK_ENTRIES(dk);
+}
+static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
+ assert(dk->dk_kind != DICT_KEYS_GENERAL);
+ return (PyDictUnicodeEntry*)_DK_ENTRIES(dk);
+}
+
+#define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL)
+
+#ifdef __cplusplus
+}
+#endif
+#endif // !Py_INTERNAL_DICT_STRUCT_H

Makefile.pre.in

@@ -1759,9 +1759,10 @@ PYTHON_HEADERS= \
  $(srcdir)/Include/internal/pycore_complexobject.h \
  $(srcdir)/Include/internal/pycore_condvar.h \
  $(srcdir)/Include/internal/pycore_context.h \
+ $(srcdir)/Include/internal/pycore_descrobject.h \
  $(srcdir)/Include/internal/pycore_dict.h \
  $(srcdir)/Include/internal/pycore_dict_state.h \
- $(srcdir)/Include/internal/pycore_descrobject.h \
+ $(srcdir)/Include/internal/pycore_dict_struct.h \
  $(srcdir)/Include/internal/pycore_dtoa.h \
  $(srcdir)/Include/internal/pycore_exceptions.h \
  $(srcdir)/Include/internal/pycore_faulthandler.h \

Misc/NEWS.d/next/C API/2023-08-21-20-41-47.gh-issue-108216.lzQ6Uq.rst

@@ -0,0 +1,3 @@
+Add a new internal pycore_dict_struct.h header file. It can be used by
+debuggers and profilers to inspect a Python dictionary without having to
+call Python functions. It should be usable in C++. Patch by Victor Stinner.

Objects/odictobject.c

@@ -466,8 +466,8 @@ Potential Optimizations
 
 #include "Python.h"
 #include "pycore_call.h" // _PyObject_CallNoArgs()
-#include "pycore_object.h" // _PyObject_GC_UNTRACK()
 #include "pycore_dict.h" // _Py_dict_lookup()
+#include "pycore_object.h" // _PyObject_GC_UNTRACK()
 #include <stddef.h> // offsetof()
 
 #include "clinic/odictobject.c.h"

PCbuild/pythoncore.vcxproj

@@ -218,6 +218,7 @@
  <ClInclude Include="..\Include\internal\pycore_descrobject.h" />
  <ClInclude Include="..\Include\internal\pycore_dict.h" />
  <ClInclude Include="..\Include\internal\pycore_dict_state.h" />
+ <ClInclude Include="..\Include\internal\pycore_dict_struct.h" />
  <ClInclude Include="..\Include\internal\pycore_dtoa.h" />
  <ClInclude Include="..\Include\internal\pycore_exceptions.h" />
  <ClInclude Include="..\Include\internal\pycore_faulthandler.h" />
@@ -248,7 +249,7 @@
  <ClInclude Include="..\Include\internal\pycore_object_state.h" />
  <ClInclude Include="..\Include\internal\pycore_obmalloc.h" />
  <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" />
- <ClInclude Include="..\Include\internal\pycore_optimizer.h" /> 
+ <ClInclude Include="..\Include\internal\pycore_optimizer.h" />
  <ClInclude Include="..\Include\internal\pycore_pathconfig.h" />
  <ClInclude Include="..\Include\internal\pycore_pyarena.h" />
  <ClInclude Include="..\Include\internal\pycore_pyerrors.h" />
@@ -280,7 +281,7 @@
  <ClInclude Include="..\Include\internal\pycore_unionobject.h" />
  <ClInclude Include="..\Include\internal\pycore_unicodeobject.h" />
  <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
- <ClInclude Include="..\Include\internal\pycore_uops.h" /> 
+ <ClInclude Include="..\Include\internal\pycore_uops.h" />
  <ClInclude Include="..\Include\internal\pycore_warnings.h" />
  <ClInclude Include="..\Include\internal\pycore_weakref.h" />
  <ClInclude Include="..\Include\interpreteridobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -561,6 +561,9 @@
  <ClInclude Include="..\Include\internal\pycore_dict_state.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
+ <ClInclude Include="..\Include\internal\pycore_dict_struct.h">
+ <Filter>Include\internal</Filter>
+ </ClInclude>
  <ClInclude Include="..\Include\internal\pycore_dtoa.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
@@ -650,7 +653,7 @@
  </ClInclude>
  <ClInclude Include="..\Include\internal\pycore_optimizer.h">
  <Filter>Include\internal</Filter>
- </ClInclude> 
+ </ClInclude>
  <ClInclude Include="..\Include\internal\pycore_pathconfig.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
@@ -737,7 +740,7 @@
  </ClInclude>
  <ClInclude Include="..\Include\internal\pycore_uops.h">
  <Filter>Include\internal</Filter>
- </ClInclude> 
+ </ClInclude>
  <ClInclude Include="$(zlibDir)\crc32.h">
  <Filter>Modules\zlib</Filter>
  </ClInclude>