diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 55a139bb9158db..5c5613237c3c77 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -105,6 +105,9 @@ _Py_atomic_add_int32(int32_t *obj, int32_t value); static inline int64_t _Py_atomic_add_int64(int64_t *obj, int64_t value); +static inline long +_Py_atomic_add_long(long *obj, long value); + static inline intptr_t _Py_atomic_add_intptr(intptr_t *obj, intptr_t value); @@ -155,6 +158,9 @@ _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desir static inline int _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired); +static inline int +_Py_atomic_compare_exchange_long(long *obj, long *expected, long desired); + static inline int _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired); @@ -348,6 +354,9 @@ _Py_atomic_load_uint32_relaxed(const uint32_t *obj); static inline uint64_t _Py_atomic_load_uint64_relaxed(const uint64_t *obj); +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + static inline uintptr_t _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index c0f3747be45758..99e9ef1b88be4f 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -30,6 +30,10 @@ static inline int64_t _Py_atomic_add_int64(int64_t *obj, int64_t value) { return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } +static inline long +_Py_atomic_add_long(long *obj, long value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + static inline intptr_t _Py_atomic_add_intptr(intptr_t *obj, intptr_t value) { return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } @@ -90,6 +94,11 @@ _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desir { return __atomic_compare_exchange_n(obj, expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } +static inline int +_Py_atomic_compare_exchange_long(long *obj, long *expected, long desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + static inline int _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) { return __atomic_compare_exchange_n(obj, expected, desired, 0, @@ -342,6 +351,10 @@ static inline uint64_t _Py_atomic_load_uint64_relaxed(const uint64_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline uintptr_t _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index f32995c1f578ac..66bc939235807e 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -59,6 +59,23 @@ _Py_atomic_add_int64(int64_t *obj, int64_t value) #endif } +static inline long +_Py_atomic_add_long(long *obj, long value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(long); + return (long)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +#else + long old_value = _Py_atomic_load_long_relaxed(obj); + for (;;) { + long new_value = old_value + value; + if (_Py_atomic_compare_exchange_long(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + static inline uint8_t _Py_atomic_add_uint8(uint8_t *obj, uint8_t value) @@ -187,6 +204,21 @@ _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value return 0; } +static inline int +_Py_atomic_compare_exchange_long(long *obj, long *expected, long value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + long initial = (long)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + static inline int _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) { @@ -688,6 +720,12 @@ _Py_atomic_load_uint64_relaxed(const uint64_t *obj) return *(volatile uint64_t *)obj; } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + static inline uintptr_t _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 0cdce4e6dd39f0..031493eb563d9b 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -55,6 +55,13 @@ _Py_atomic_add_int64(int64_t *obj, int64_t value) return atomic_fetch_add((_Atomic(int64_t)*)obj, value); } +static inline long +_Py_atomic_add_long(long *obj, long value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(long)*)obj, value); +} + static inline intptr_t _Py_atomic_add_intptr(intptr_t *obj, intptr_t value) { @@ -154,6 +161,14 @@ _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desir expected, desired); } +static inline int +_Py_atomic_compare_exchange_long(long *obj, long *expected, long desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(long)*)obj, + expected, desired); +} + static inline int _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) { @@ -587,6 +602,15 @@ _Py_atomic_load_uint64_relaxed(const uint64_t *obj) memory_order_relaxed); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + + static inline uintptr_t _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) { diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 7da6162744ffd6..9adeaf92bf6cbb 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -816,6 +816,21 @@ PyTypeObject PyRange_Type = { static PyObject * rangeiter_next(_PyRangeIterObject *r) { +#ifdef Py_GIL_DISABLED + // r->step is readonly attribute, so we can assume it is not changed. + long step = r->step; + do { + long len = _Py_atomic_load_long_relaxed(&r->len); + if (len <= 0) { + return NULL; + } + long result = _Py_atomic_load_long_relaxed(&r->start); + if (_Py_atomic_compare_exchange_long(&r->start, &result, result + step)) { + _Py_atomic_add_long(&r->len, -1); + return PyLong_FromLong(result); + } + } while (1); +#else if (r->len > 0) { long result = r->start; r->start = result + r->step; @@ -823,6 +838,7 @@ rangeiter_next(_PyRangeIterObject *r) return PyLong_FromLong(result); } return NULL; +#endif } static PyObject *