Skip to content

Commit 7fa511b

Browse files
corona10corona10
authored
gh-111968: Use per-thread freelists for generator in free-threading (gh-114189)
1 parent 2d3f6b5 commit 7fa511b

File tree

9 files changed

+48
-74
lines changed

9 files changed

+48
-74
lines changed

Diff for: ‎Include/internal/pycore_freelist.h

+17
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ extern "C" {
1919
# define PyList_MAXFREELIST 80
2020
# define PyFloat_MAXFREELIST 100
2121
# define PyContext_MAXFREELIST 255
22+
# define _PyAsyncGen_MAXFREELIST 80
2223
#else
2324
# define PyTuple_NFREELISTS 0
2425
# define PyTuple_MAXFREELIST 0
2526
# define PyList_MAXFREELIST 0
2627
# define PyFloat_MAXFREELIST 0
2728
# define PyContext_MAXFREELIST 0
29+
# define _PyAsyncGen_MAXFREELIST 0
2830
#endif
2931

3032
struct _Py_list_state {
@@ -77,12 +79,27 @@ struct _Py_context_state {
7779
#endif
7880
};
7981

82+
struct _Py_async_gen_state {
83+
#ifdef WITH_FREELISTS
84+
/* Freelists boost performance 6-10%; they also reduce memory
85+
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
86+
are short-living objects that are instantiated for every
87+
__anext__() call. */
88+
struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST];
89+
int value_numfree;
90+
91+
struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST];
92+
int asend_numfree;
93+
#endif
94+
};
95+
8096
typedef struct _Py_freelist_state {
8197
struct _Py_float_state float_state;
8298
struct _Py_tuple_state tuple_state;
8399
struct _Py_list_state list_state;
84100
struct _Py_slice_state slice_state;
85101
struct _Py_context_state context_state;
102+
struct _Py_async_gen_state async_gen_state;
86103
} _PyFreeListState;
87104

88105
#ifdef __cplusplus

Diff for: ‎Include/internal/pycore_gc.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization)
251251
extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
252252
extern void _PySlice_ClearCache(_PyFreeListState *state);
253253
extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
254-
extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
254+
extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization);
255255
extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization);
256256
extern void _Py_ScheduleGC(PyInterpreterState *interp);
257257
extern void _Py_RunGC(PyThreadState *tstate);

Diff for: ‎Include/internal/pycore_genobject.h

+3-28
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_freelist.h"
12+
1113
extern PyObject *_PyGen_yf(PyGenObject *);
1214
extern void _PyGen_Finalize(PyObject *self);
1315

@@ -26,34 +28,7 @@ extern PyTypeObject _PyAsyncGenAThrow_Type;
2628

2729
/* runtime lifecycle */
2830

29-
extern void _PyAsyncGen_Fini(PyInterpreterState *);
30-
31-
32-
/* other API */
33-
34-
#ifndef WITH_FREELISTS
35-
// without freelists
36-
# define _PyAsyncGen_MAXFREELIST 0
37-
#endif
38-
39-
#ifndef _PyAsyncGen_MAXFREELIST
40-
# define _PyAsyncGen_MAXFREELIST 80
41-
#endif
42-
43-
struct _Py_async_gen_state {
44-
#if _PyAsyncGen_MAXFREELIST > 0
45-
/* Freelists boost performance 6-10%; they also reduce memory
46-
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
47-
are short-living objects that are instantiated for every
48-
__anext__() call. */
49-
struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST];
50-
int value_numfree;
51-
52-
struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST];
53-
int asend_numfree;
54-
#endif
55-
};
56-
31+
extern void _PyAsyncGen_Fini(_PyFreeListState *);
5732

5833
#ifdef __cplusplus
5934
}

Diff for: ‎Include/internal/pycore_interp.h

-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,6 @@ struct _is {
190190

191191
struct _Py_tuple_state tuple;
192192
struct _Py_dict_state dict_state;
193-
struct _Py_async_gen_state async_gen;
194193
struct _Py_exc_state exc_state;
195194

196195
struct ast_state ast;

Diff for: ‎Objects/genobject.c

+24-40
Original file line numberDiff line numberDiff line change
@@ -1628,12 +1628,12 @@ PyTypeObject PyAsyncGen_Type = {
16281628
};
16291629

16301630

1631-
#if _PyAsyncGen_MAXFREELIST > 0
1631+
#ifdef WITH_FREELISTS
16321632
static struct _Py_async_gen_state *
16331633
get_async_gen_state(void)
16341634
{
1635-
PyInterpreterState *interp = _PyInterpreterState_GET();
1636-
return &interp->async_gen;
1635+
_PyFreeListState *state = _PyFreeListState_GET();
1636+
return &state->async_gen_state;
16371637
}
16381638
#endif
16391639

@@ -1656,36 +1656,36 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
16561656

16571657

16581658
void
1659-
_PyAsyncGen_ClearFreeLists(PyInterpreterState *interp)
1659+
_PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization)
16601660
{
1661-
#if _PyAsyncGen_MAXFREELIST > 0
1662-
struct _Py_async_gen_state *state = &interp->async_gen;
1661+
#ifdef WITH_FREELISTS
1662+
struct _Py_async_gen_state *state = &freelist_state->async_gen_state;
16631663

1664-
while (state->value_numfree) {
1664+
while (state->value_numfree > 0) {
16651665
_PyAsyncGenWrappedValue *o;
16661666
o = state->value_freelist[--state->value_numfree];
16671667
assert(_PyAsyncGenWrappedValue_CheckExact(o));
16681668
PyObject_GC_Del(o);
16691669
}
16701670

1671-
while (state->asend_numfree) {
1671+
while (state->asend_numfree > 0) {
16721672
PyAsyncGenASend *o;
16731673
o = state->asend_freelist[--state->asend_numfree];
16741674
assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
16751675
PyObject_GC_Del(o);
16761676
}
1677+
1678+
if (is_finalization) {
1679+
state->value_numfree = -1;
1680+
state->asend_numfree = -1;
1681+
}
16771682
#endif
16781683
}
16791684

16801685
void
1681-
_PyAsyncGen_Fini(PyInterpreterState *interp)
1686+
_PyAsyncGen_Fini(_PyFreeListState *state)
16821687
{
1683-
_PyAsyncGen_ClearFreeLists(interp);
1684-
#if defined(Py_DEBUG) && _PyAsyncGen_MAXFREELIST > 0
1685-
struct _Py_async_gen_state *state = &interp->async_gen;
1686-
state->value_numfree = -1;
1687-
state->asend_numfree = -1;
1688-
#endif
1688+
_PyAsyncGen_ClearFreeLists(state, 1);
16891689
}
16901690

16911691

@@ -1732,13 +1732,9 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
17321732
_PyObject_GC_UNTRACK((PyObject *)o);
17331733
Py_CLEAR(o->ags_gen);
17341734
Py_CLEAR(o->ags_sendval);
1735-
#if _PyAsyncGen_MAXFREELIST > 0
1735+
#ifdef WITH_FREELISTS
17361736
struct _Py_async_gen_state *state = get_async_gen_state();
1737-
#ifdef Py_DEBUG
1738-
// async_gen_asend_dealloc() must not be called after _PyAsyncGen_Fini()
1739-
assert(state->asend_numfree != -1);
1740-
#endif
1741-
if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) {
1737+
if (state->asend_numfree >= 0 && state->asend_numfree < _PyAsyncGen_MAXFREELIST) {
17421738
assert(PyAsyncGenASend_CheckExact(o));
17431739
_PyGC_CLEAR_FINALIZED((PyObject *)o);
17441740
state->asend_freelist[state->asend_numfree++] = o;
@@ -1906,13 +1902,9 @@ static PyObject *
19061902
async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
19071903
{
19081904
PyAsyncGenASend *o;
1909-
#if _PyAsyncGen_MAXFREELIST > 0
1905+
#ifdef WITH_FREELISTS
19101906
struct _Py_async_gen_state *state = get_async_gen_state();
1911-
#ifdef Py_DEBUG
1912-
// async_gen_asend_new() must not be called after _PyAsyncGen_Fini()
1913-
assert(state->asend_numfree != -1);
1914-
#endif
1915-
if (state->asend_numfree) {
1907+
if (state->asend_numfree > 0) {
19161908
state->asend_numfree--;
19171909
o = state->asend_freelist[state->asend_numfree];
19181910
_Py_NewReference((PyObject *)o);
@@ -1945,13 +1937,9 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
19451937
{
19461938
_PyObject_GC_UNTRACK((PyObject *)o);
19471939
Py_CLEAR(o->agw_val);
1948-
#if _PyAsyncGen_MAXFREELIST > 0
1940+
#ifdef WITH_FREELISTS
19491941
struct _Py_async_gen_state *state = get_async_gen_state();
1950-
#ifdef Py_DEBUG
1951-
// async_gen_wrapped_val_dealloc() must not be called after _PyAsyncGen_Fini()
1952-
assert(state->value_numfree != -1);
1953-
#endif
1954-
if (state->value_numfree < _PyAsyncGen_MAXFREELIST) {
1942+
if (state->value_numfree >= 0 && state->value_numfree < _PyAsyncGen_MAXFREELIST) {
19551943
assert(_PyAsyncGenWrappedValue_CheckExact(o));
19561944
state->value_freelist[state->value_numfree++] = o;
19571945
OBJECT_STAT_INC(to_freelist);
@@ -2022,13 +2010,9 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
20222010
_PyAsyncGenWrappedValue *o;
20232011
assert(val);
20242012

2025-
#if _PyAsyncGen_MAXFREELIST > 0
2026-
struct _Py_async_gen_state *state = &tstate->interp->async_gen;
2027-
#ifdef Py_DEBUG
2028-
// _PyAsyncGenValueWrapperNew() must not be called after _PyAsyncGen_Fini()
2029-
assert(state->value_numfree != -1);
2030-
#endif
2031-
if (state->value_numfree) {
2013+
#ifdef WITH_FREELISTS
2014+
struct _Py_async_gen_state *state = get_async_gen_state();
2015+
if (state->value_numfree > 0) {
20322016
state->value_numfree--;
20332017
o = state->value_freelist[state->value_numfree];
20342018
OBJECT_STAT_INC(from_freelist);

Diff for: ‎Python/gc_free_threading.c

-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ void
1515
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
1616
{
1717
_PyDict_ClearFreeList(interp);
18-
_PyAsyncGen_ClearFreeLists(interp);
1918

2019
HEAD_LOCK(&_PyRuntime);
2120
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;

Diff for: ‎Python/gc_gil.c

-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ void
1212
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
1313
{
1414
_PyDict_ClearFreeList(interp);
15-
_PyAsyncGen_ClearFreeLists(interp);
1615

1716
_Py_ClearFreeLists(&interp->freelist_state, 0);
1817
}

Diff for: ‎Python/pylifecycle.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -1735,7 +1735,6 @@ finalize_interp_types(PyInterpreterState *interp)
17351735
_PySys_FiniTypes(interp);
17361736
_PyXI_FiniTypes(interp);
17371737
_PyExc_Fini(interp);
1738-
_PyAsyncGen_Fini(interp);
17391738
_PyFloat_FiniType(interp);
17401739
_PyLong_FiniTypes(interp);
17411740
_PyThread_FiniType(interp);
@@ -1759,6 +1758,7 @@ finalize_interp_types(PyInterpreterState *interp)
17591758
_PyFloat_Fini(state);
17601759
_PySlice_Fini(state);
17611760
_PyContext_Fini(state);
1761+
_PyAsyncGen_Fini(state);
17621762

17631763
#ifdef Py_DEBUG
17641764
_PyStaticObjects_CheckRefcnt(interp);

Diff for: ‎Python/pystate.c

+2-1
Original file line numberDiff line numberDiff line change
@@ -1462,6 +1462,7 @@ _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization)
14621462
_PyTuple_ClearFreeList(state, is_finalization);
14631463
_PyList_ClearFreeList(state, is_finalization);
14641464
_PyContext_ClearFreeList(state, is_finalization);
1465+
_PyAsyncGen_ClearFreeLists(state, is_finalization);
14651466
}
14661467

14671468
void
@@ -1549,7 +1550,7 @@ PyThreadState_Clear(PyThreadState *tstate)
15491550
#ifdef Py_GIL_DISABLED
15501551
// Each thread should clear own freelists in free-threading builds.
15511552
_PyFreeListState *freelist_state = &((_PyThreadStateImpl*)tstate)->freelist_state;
1552-
_Py_ClearFreeLists(freelist_state, 0);
1553+
_Py_ClearFreeLists(freelist_state, 1);
15531554
_PySlice_ClearCache(freelist_state);
15541555
#endif
15551556

0 commit comments

Comments
 (0)