Skip to content

Commit 05dda6b

Browse files
kjbraceykjbracey
committed
Add non-volatile atomic loads and stores
Volatile makes no real difference when we're using assembler, or locked functions, but leaving it off could be more efficient for the basic loads and stores. So add non-volatile overloads in C++ for them.
1 parent 7a1ac23 commit 05dda6b

File tree

2 files changed

+113
-13
lines changed

2 files changed

+113
-13
lines changed

‎platform/internal/mbed_atomic_impl.h‎

Lines changed: 97 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -606,30 +606,30 @@ MBED_FORCEINLINE bool core_util_atomic_flag_test_and_set_explicit(volatile core_
606606

607607
/* Lock-free loads and stores don't need assembler - just aligned accesses */
608608
/* Silly ordering of `T volatile` is because T can be `void *` */
609-
#define DO_MBED_LOCKFREE_LOADSTORE(T, fn_suffix) \
610-
MBED_FORCEINLINE T core_util_atomic_load_##fn_suffix(T const volatile *valuePtr) \
609+
#define DO_MBED_LOCKFREE_LOADSTORE(T, V, fn_suffix) \
610+
MBED_FORCEINLINE T core_util_atomic_load_##fn_suffix(T const V *valuePtr) \
611611
{ \
612612
T value = *valuePtr; \
613613
MBED_BARRIER(); \
614614
return value; \
615615
} \
616616
\
617-
MBED_FORCEINLINE T core_util_atomic_load_explicit_##fn_suffix(T const volatile *valuePtr, mbed_memory_order order) \
617+
MBED_FORCEINLINE T core_util_atomic_load_explicit_##fn_suffix(T const V *valuePtr, mbed_memory_order order) \
618618
{ \
619619
MBED_CHECK_LOAD_ORDER(order); \
620620
T value = *valuePtr; \
621621
MBED_ACQUIRE_BARRIER(order); \
622622
return value; \
623623
} \
624624
\
625-
MBED_FORCEINLINE void core_util_atomic_store_##fn_suffix(T volatile *valuePtr, T value) \
625+
MBED_FORCEINLINE void core_util_atomic_store_##fn_suffix(T V *valuePtr, T value) \
626626
{ \
627627
MBED_BARRIER(); \
628628
*valuePtr = value; \
629629
MBED_BARRIER(); \
630630
} \
631631
\
632-
MBED_FORCEINLINE void core_util_atomic_store_explicit_##fn_suffix(T volatile *valuePtr, T value, mbed_memory_order order) \
632+
MBED_FORCEINLINE void core_util_atomic_store_explicit_##fn_suffix(T V *valuePtr, T value, mbed_memory_order order) \
633633
{ \
634634
MBED_CHECK_STORE_ORDER(order); \
635635
MBED_RELEASE_BARRIER(order); \
@@ -651,15 +651,51 @@ MBED_FORCEINLINE void core_util_atomic_flag_clear_explicit(volatile core_util_at
651651
flagPtr->_flag = false;
652652
MBED_SEQ_CST_BARRIER(order);
653653
}
654-
DO_MBED_LOCKFREE_LOADSTORE(uint8_t, u8)
655-
DO_MBED_LOCKFREE_LOADSTORE(uint16_t, u16)
656-
DO_MBED_LOCKFREE_LOADSTORE(uint32_t, u32)
657-
DO_MBED_LOCKFREE_LOADSTORE(int8_t, s8)
658-
DO_MBED_LOCKFREE_LOADSTORE(int16_t, s16)
659-
DO_MBED_LOCKFREE_LOADSTORE(int32_t, s32)
660-
DO_MBED_LOCKFREE_LOADSTORE(bool, bool)
661-
DO_MBED_LOCKFREE_LOADSTORE(void *, ptr)
662654

655+
#ifdef __cplusplus
656+
// Temporarily turn off extern "C", so we can provide non-volatile load/store
657+
// overloads for efficiency. All these functions are static inline, so this has
658+
// no linkage effect exactly, it just permits the overloads.
659+
} // extern "C"
660+
661+
// For efficiency it's worth having non-volatile overloads
662+
MBED_FORCEINLINE void core_util_atomic_flag_clear(core_util_atomic_flag *flagPtr)
663+
{
664+
MBED_BARRIER();
665+
flagPtr->_flag = false;
666+
MBED_BARRIER();
667+
}
668+
669+
MBED_FORCEINLINE void core_util_atomic_flag_clear_explicit(core_util_atomic_flag *flagPtr, mbed_memory_order order)
670+
{
671+
MBED_RELEASE_BARRIER(order);
672+
flagPtr->_flag = false;
673+
MBED_SEQ_CST_BARRIER(order);
674+
}
675+
676+
DO_MBED_LOCKFREE_LOADSTORE(uint8_t,, u8)
677+
DO_MBED_LOCKFREE_LOADSTORE(uint16_t,, u16)
678+
DO_MBED_LOCKFREE_LOADSTORE(uint32_t,, u32)
679+
DO_MBED_LOCKFREE_LOADSTORE(int8_t,, s8)
680+
DO_MBED_LOCKFREE_LOADSTORE(int16_t,, s16)
681+
DO_MBED_LOCKFREE_LOADSTORE(int32_t,, s32)
682+
DO_MBED_LOCKFREE_LOADSTORE(bool,, bool)
683+
DO_MBED_LOCKFREE_LOADSTORE(void *,, ptr)
684+
685+
#endif
686+
687+
DO_MBED_LOCKFREE_LOADSTORE(uint8_t, volatile, u8)
688+
DO_MBED_LOCKFREE_LOADSTORE(uint16_t, volatile, u16)
689+
DO_MBED_LOCKFREE_LOADSTORE(uint32_t, volatile, u32)
690+
DO_MBED_LOCKFREE_LOADSTORE(int8_t, volatile, s8)
691+
DO_MBED_LOCKFREE_LOADSTORE(int16_t, volatile, s16)
692+
DO_MBED_LOCKFREE_LOADSTORE(int32_t, volatile, s32)
693+
DO_MBED_LOCKFREE_LOADSTORE(bool, volatile, bool)
694+
DO_MBED_LOCKFREE_LOADSTORE(void *, volatile, ptr)
695+
696+
#ifdef __cplusplus
697+
extern "C" {
698+
#endif
663699

664700
/********************* GENERIC VARIANTS - SIGNED, BOOL, POINTERS ****************/
665701

@@ -906,7 +942,19 @@ inline T core_util_atomic_load(const volatile T *valuePtr)
906942
} \
907943
\
908944
template<> \
945+
inline T core_util_atomic_load(const T *valuePtr) \
946+
{ \
947+
return core_util_atomic_load_##fn_suffix(valuePtr); \
948+
} \
949+
\
950+
template<> \
909951
inline T core_util_atomic_load_explicit(const volatile T *valuePtr, mbed_memory_order order) \
952+
{ \
953+
return core_util_atomic_load_explicit_##fn_suffix(valuePtr, order); \
954+
} \
955+
\
956+
template<> \
957+
inline T core_util_atomic_load_explicit(const T *valuePtr, mbed_memory_order order) \
910958
{ \
911959
return core_util_atomic_load_explicit_##fn_suffix(valuePtr, order); \
912960
}
@@ -917,12 +965,24 @@ inline T *core_util_atomic_load(T *const volatile *valuePtr)
917965
return (T *) core_util_atomic_load_ptr((void *const volatile *) valuePtr);
918966
}
919967

968+
template<typename T>
969+
inline T *core_util_atomic_load(T *const *valuePtr)
970+
{
971+
return (T *) core_util_atomic_load_ptr((void *const *) valuePtr);
972+
}
973+
920974
template<typename T>
921975
inline T *core_util_atomic_load_explicit(T *const volatile *valuePtr, mbed_memory_order order)
922976
{
923977
return (T *) core_util_atomic_load_explicit_ptr((void *const volatile *) valuePtr, order);
924978
}
925979

980+
template<typename T>
981+
inline T *core_util_atomic_load_explicit(T *const *valuePtr, mbed_memory_order order)
982+
{
983+
return (T *) core_util_atomic_load_explicit_ptr((void *const *) valuePtr, order);
984+
}
985+
926986
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint8_t, u8)
927987
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint16_t, u16)
928988
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint32_t, u32)
@@ -941,7 +1001,19 @@ inline void core_util_atomic_store(volatile T *valuePtr, T val)
9411001
} \
9421002
\
9431003
template<> \
1004+
inline void core_util_atomic_store(T *valuePtr, T val) \
1005+
{ \
1006+
core_util_atomic_store_##fn_suffix(valuePtr, val); \
1007+
} \
1008+
\
1009+
template<> \
9441010
inline void core_util_atomic_store_explicit(volatile T *valuePtr, T val, mbed_memory_order order) \
1011+
{ \
1012+
core_util_atomic_store_explicit_##fn_suffix(valuePtr, val, order); \
1013+
} \
1014+
\
1015+
template<> \
1016+
inline void core_util_atomic_store_explicit(T *valuePtr, T val, mbed_memory_order order) \
9451017
{ \
9461018
core_util_atomic_store_explicit_##fn_suffix(valuePtr, val, order); \
9471019
}
@@ -952,12 +1024,24 @@ inline void core_util_atomic_store(T *volatile *valuePtr, T *val)
9521024
core_util_atomic_store_ptr((void *volatile *) valuePtr, val);
9531025
}
9541026

1027+
template<typename T>
1028+
inline void core_util_atomic_store(T **valuePtr, T *val)
1029+
{
1030+
core_util_atomic_store_ptr((void **) valuePtr, val);
1031+
}
1032+
9551033
template<typename T>
9561034
inline void core_util_atomic_store_explicit(T *volatile *valuePtr, T *val, mbed_memory_order order)
9571035
{
9581036
core_util_atomic_store_ptr((void *volatile *) valuePtr, val, order);
9591037
}
9601038

1039+
template<typename T>
1040+
inline void core_util_atomic_store_explicit(T **valuePtr, T *val, mbed_memory_order order)
1041+
{
1042+
core_util_atomic_store_ptr((void **) valuePtr, val, order);
1043+
}
1044+
9611045
DO_MBED_ATOMIC_STORE_TEMPLATE(uint8_t, u8)
9621046
DO_MBED_ATOMIC_STORE_TEMPLATE(uint16_t, u16)
9631047
DO_MBED_ATOMIC_STORE_TEMPLATE(uint32_t, u32)

‎platform/mbed_atomic.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -888,8 +888,12 @@ MBED_FORCEINLINE uint64_t core_util_atomic_fetch_xor_explicit_u64(volatile uint6
888888

889889
/** \copydoc core_util_atomic_load_u8 */
890890
template<typename T> T core_util_atomic_load(const volatile T *valuePtr);
891+
/** \copydoc core_util_atomic_load_u8 */
892+
template<typename T> T core_util_atomic_load(const T *valuePtr);
891893
/** \copydoc core_util_atomic_store_u8 */
892894
template<typename T> void core_util_atomic_store(volatile T *valuePtr, T desiredValue);
895+
/** \copydoc core_util_atomic_store_u8 */
896+
template<typename T> void core_util_atomic_store(T *valuePtr, T desiredValue);
893897
/** \copydoc core_util_atomic_exchange_u8 */
894898
template<typename T> T core_util_atomic_exchange(volatile T *ptr, T desiredValue);
895899
/** \copydoc core_util_atomic_cas_u8 */
@@ -909,8 +913,12 @@ template<typename T> T core_util_atomic_fetch_xor(volatile T *valuePtr, T arg);
909913

910914
/** \copydoc core_util_atomic_load_explicit_u8 */
911915
template<typename T> T core_util_atomic_load_explicit(const volatile T *valuePtr, mbed_memory_order order);
916+
/** \copydoc core_util_atomic_load_explicit_u8 */
917+
template<typename T> T core_util_atomic_load_explicit(const T *valuePtr, mbed_memory_order order);
912918
/** \copydoc core_util_atomic_store_explicit_u8 */
913919
template<typename T> void core_util_atomic_store_explicit(volatile T *valuePtr, T desiredValue, mbed_memory_order order);
920+
/** \copydoc core_util_atomic_store_explicit_u8 */
921+
template<typename T> void core_util_atomic_store_explicit(T *valuePtr, T desiredValue, mbed_memory_order order);
914922
/** \copydoc core_util_atomic_exchange_explicit_u8 */
915923
template<typename T> T core_util_atomic_exchange_explicit(volatile T *ptr, T desiredValue, mbed_memory_order order);
916924
/** \copydoc core_util_atomic_cas_explicit_u8 */
@@ -930,8 +938,12 @@ template<typename T> T core_util_atomic_fetch_xor_explicit(volatile T *valuePtr,
930938

931939
/** \copydoc core_util_atomic_load_ptr */
932940
template<typename T> inline T *core_util_atomic_load(T *const volatile *valuePtr);
941+
/** \copydoc core_util_atomic_load_ptr */
942+
template<typename T> inline T *core_util_atomic_load(T *const *valuePtr);
933943
/** \copydoc core_util_atomic_store_ptr */
934944
template<typename T> inline void core_util_atomic_store(T *volatile *valuePtr, T *desiredValue);
945+
/** \copydoc core_util_atomic_store_ptr */
946+
template<typename T> inline void core_util_atomic_store(T **valuePtr, T *desiredValue);
935947
/** \copydoc core_util_atomic_exchange_ptr */
936948
template<typename T> inline T *core_util_atomic_exchange(T *volatile *valuePtr, T *desiredValue);
937949
/** \copydoc core_util_atomic_cas_ptr */
@@ -945,8 +957,12 @@ template<typename T> inline T *core_util_atomic_fetch_sub(T *volatile *valuePtr,
945957

946958
/** \copydoc core_util_atomic_load_explicit_ptr */
947959
template<typename T> inline T *core_util_atomic_load_explicit(T *const volatile *valuePtr, mbed_memory_order order);
960+
/** \copydoc core_util_atomic_load_explicit_ptr */
961+
template<typename T> inline T *core_util_atomic_load_explicit(T *const *valuePtr, mbed_memory_order order);
948962
/** \copydoc core_util_atomic_store_explicit_ptr */
949963
template<typename T> inline void core_util_atomic_store_explicit(T *volatile *valuePtr, T *desiredValue, mbed_memory_order order);
964+
/** \copydoc core_util_atomic_store_explicit_ptr */
965+
template<typename T> inline void core_util_atomic_store_explicit(T **valuePtr, T *desiredValue, mbed_memory_order order);
950966
/** \copydoc core_util_atomic_exchange_explicit_ptr */
951967
template<typename T> inline T *core_util_atomic_exchange_explicit(T *volatile *valuePtr, T *desiredValue, mbed_memory_order order);
952968
/** \copydoc core_util_atomic_cas_explicit_ptr */

0 commit comments

Comments
 (0)