From d7ccfd290f88acdc9af19c6e03defc55aa1563dc Mon Sep 17 00:00:00 2001 From: Bart Janssens Date: Sun, 31 Oct 2021 18:30:02 +0100 Subject: [PATCH] Backport PR #42152 to 1.7 (#42828) Co-authored-by: Jameson Nash --- src/Makefile | 26 ++- src/array.c | 22 +- src/ast.c | 2 +- src/atomics.h | 163 --------------- src/builtins.c | 2 +- src/cgmemmgr.cpp | 4 +- src/codegen.cpp | 33 +-- src/datatype.c | 160 +++++++-------- src/dlload.c | 2 +- src/dump.c | 21 +- src/gc-stacks.c | 2 +- src/gc.c | 129 +++++++----- src/gc.h | 2 +- src/gf.c | 24 +-- src/iddict.c | 24 +-- src/interpreter.c | 4 +- src/intrinsics.cpp | 3 +- src/jitlayers.cpp | 25 +-- src/jl_uv.c | 2 +- src/jlapi.c | 2 +- src/jltypes.c | 12 +- src/julia.h | 100 +++++----- src/julia_atomics.h | 311 +++++++++++++++++++++++++++++ src/julia_internal.h | 32 +-- src/{locks.h => julia_locks.h} | 4 +- src/julia_threads.h | 34 ++-- src/llvm-version.h | 2 + src/method.c | 17 +- src/module.c | 24 ++- src/partr.c | 8 +- src/rtutils.c | 4 +- src/runtime_ccall.cpp | 2 +- src/runtime_intrinsics.c | 12 +- src/safepoint.c | 6 +- src/signals-unix.c | 17 +- src/smallintset.c | 23 ++- src/staticdata.c | 8 +- src/subtype.c | 4 +- src/support/analyzer_annotations.h | 2 +- src/symbol.c | 14 +- src/task.c | 14 +- src/threading.c | 10 +- src/typemap.c | 63 +++--- test/clangsa/GCPushPop.cpp | 2 +- test/clangsa/MissingRoots.c | 2 +- 45 files changed, 801 insertions(+), 578 deletions(-) delete mode 100644 src/atomics.h create mode 100644 src/julia_atomics.h rename src/{locks.h => julia_locks.h} (98%) diff --git a/src/Makefile b/src/Makefile index 412967003ed5a..479d1652f6808 100644 --- a/src/Makefile +++ b/src/Makefile @@ -87,8 +87,8 @@ SRCS += $(RUNTIME_SRCS) # headers are used for dependency tracking, while public headers will be part of the dist UV_HEADERS := -HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h locks.h atomics.h julia_internal.h options.h timing.h) -PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h locks.h atomics.h julia_gcext.h jloptions.h) +HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h julia_internal.h options.h timing.h) $(addprefix $(BUILDDIR)/, $(DTRACE_HEADERS)) +PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h julia_gcext.h jloptions.h) ifeq ($(USE_SYSTEM_LIBUV),0) UV_HEADERS += uv.h UV_HEADERS += uv/*.h @@ -358,15 +358,23 @@ endif clangsa: $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) -clang-sa-%: $(SRCDIR)/%.c $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) | analyzegc-deps-check - @$(call PRINT_ANALYZE, $(build_depsbindir)/clang --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text -Xclang -load -Xclang $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) $(CLANGSA_FLAGS) $(JCPPFLAGS) $(JCFLAGS) $(DEBUGFLAGS) -Xclang -analyzer-checker=core$(COMMA)julia.GCChecker --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c $<) -clang-sa-%: $(SRCDIR)/%.cpp $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) | analyzegc-deps-check - @$(call PRINT_ANALYZE, $(build_depsbindir)/clang --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text -Xclang -load -Xclang $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) $(CLANGSA_FLAGS) $(CLANGSA_CXXFLAGS) $(LLVM_CXXFLAGS) $(JCPPFLAGS) $(JCXXFLAGS) $(DEBUGFLAGS) -Xclang -analyzer-checker=core$(COMMA)julia.GCChecker --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c++ $<) +clang-sagc-%: $(SRCDIR)/%.c $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) .FORCE | analyzegc-deps-check + @$(call PRINT_ANALYZE, $(build_depsbindir)/clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text -Xclang -load -Xclang $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) $(CLANGSA_FLAGS) $(JCPPFLAGS) $(JCFLAGS) $(DEBUGFLAGS) -Xclang -analyzer-checker=core$(COMMA)julia.GCChecker --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c $<) +clang-sagc-%: $(SRCDIR)/%.cpp $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) .FORCE | analyzegc-deps-check + @$(call PRINT_ANALYZE, $(build_depsbindir)/clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text -Xclang -load -Xclang $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) $(CLANGSA_FLAGS) $(CLANGSA_CXXFLAGS) $(LLVM_CXXFLAGS) $(JCPPFLAGS) $(JCXXFLAGS) $(DEBUGFLAGS) -Xclang -analyzer-checker=core$(COMMA)julia.GCChecker --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c++ $<) -# Add C files as a target of `analyzegc` -analyzegc: $(addprefix clang-sa-,$(RUNTIME_SRCS)) +clang-sa-%: $(SRCDIR)/%.c .FORCE | analyzegc-deps-check + @$(call PRINT_ANALYZE, $(build_depsbindir)/clang --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text $(CLANGSA_FLAGS) $(JCPPFLAGS) $(JCFLAGS) $(DEBUGFLAGS) --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c $<) +clang-sa-%: $(SRCDIR)/%.cpp .FORCE | analyzegc-deps-check + @$(call PRINT_ANALYZE, $(build_depsbindir)/clang --analyze -Xanalyzer -analyzer-werror -Xanalyzer -analyzer-output=text $(CLANGSA_FLAGS) $(CLANGSA_CXXFLAGS) $(LLVM_CXXFLAGS) $(JCPPFLAGS) $(JCXXFLAGS) $(DEBUGFLAGS) --analyzer-no-default-checks -fcolor-diagnostics -Werror -x c++ $<) + + +# Add C files as a target of `analyzesrc` and `analyzegc` +analyzesrc: $(addprefix clang-sa-,$(SRCS)) +analyzegc: analyzesrc $(addprefix clang-sagc-,$(RUNTIME_SRCS)) clean-analyzegc: rm -f $(build_shlibdir)/libGCCheckerPlugin.$(SHLIB_EXT) -.PHONY: default all debug release clean cleanall clean-* libccalltest libllvmcalltest julia_flisp.boot.inc.phony analyzegc clang-sa-* +.FORCE: +.PHONY: default all debug release clean cleanall clean-* libccalltest libllvmcalltest julia_flisp.boot.inc.phony analyzegc analyzesrc .FORCE diff --git a/src/array.c b/src/array.c index bc86eb50b95c5..43aeb3f805492 100644 --- a/src/array.c +++ b/src/array.c @@ -26,7 +26,7 @@ static inline void arrayassign_safe(int hasptr, jl_value_t *parent, char *dst, c assert(nb >= jl_datatype_size(jl_typeof(src))); // nb might move some undefined bits, but we should be okay with that if (hasptr) { size_t nptr = nb / sizeof(void*); - memmove_refs((void**)dst, (void**)src, nptr); + memmove_refs((void**)dst, (void* const*)src, nptr); jl_gc_multi_wb(parent, src); } else { @@ -581,7 +581,7 @@ JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t { assert(i < jl_array_len(a)); assert(a->flags.ptrarray); - jl_value_t *elt = jl_atomic_load_relaxed(((jl_value_t**)a->data) + i); + jl_value_t *elt = jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)a->data) + i); if (elt == NULL) jl_throw(jl_undefref_exception); return elt; @@ -610,7 +610,7 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i) JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i) { if (a->flags.ptrarray) { - return jl_atomic_load_relaxed(((jl_value_t**)jl_array_data(a)) + i) != NULL; + return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)jl_array_data(a)) + i) != NULL; } else if (a->flags.hasptr) { jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a)); @@ -649,7 +649,7 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs arrayassign_safe(hasptr, jl_array_owner(a), &((char*)a->data)[i * a->elsize], rhs, a->elsize); } else { - jl_atomic_store_relaxed(((jl_value_t**)a->data) + i, rhs); + jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)a->data) + i, rhs); jl_gc_wb(jl_array_owner(a), rhs); } } @@ -659,7 +659,7 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i) if (i >= jl_array_len(a)) jl_bounds_error_int((jl_value_t*)a, i + 1); if (a->flags.ptrarray) - jl_atomic_store_relaxed(((jl_value_t**)a->data) + i, NULL); + jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)a->data) + i, NULL); else if (a->flags.hasptr) { size_t elsize = a->elsize; jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); @@ -1236,9 +1236,11 @@ static NOINLINE ssize_t jl_array_ptr_copy_forward(jl_value_t *owner, void **src_p, void **dest_p, ssize_t n) JL_NOTSAFEPOINT { + _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; + _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; for (ssize_t i = 0; i < n; i++) { - void *val = jl_atomic_load_relaxed(src_p + i); - jl_atomic_store_relaxed(dest_p + i, val); + void *val = jl_atomic_load_relaxed(src_pa + i); + jl_atomic_store_relaxed(dest_pa + i, val); // `val` is young or old-unmarked if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { jl_gc_queue_root(owner); @@ -1252,9 +1254,11 @@ static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, void **src_p, void **dest_p, ssize_t n) JL_NOTSAFEPOINT { + _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; + _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; for (ssize_t i = 0; i < n; i++) { - void *val = jl_atomic_load_relaxed(src_p + n - i - 1); - jl_atomic_store_relaxed(dest_p + n - i - 1, val); + void *val = jl_atomic_load_relaxed(src_pa + n - i - 1); + jl_atomic_store_relaxed(dest_pa + n - i - 1, val); // `val` is young or old-unmarked if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { jl_gc_queue_root(owner); diff --git a/src/ast.c b/src/ast.c index de2492db08c94..5312bf74756ba 100644 --- a/src/ast.c +++ b/src/ast.c @@ -125,7 +125,7 @@ typedef struct _jl_ast_context_t { static jl_ast_context_t jl_ast_main_ctx; -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ jl_ast_context_t *jl_ast_ctx(fl_context_t *fl) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; #else #define jl_ast_ctx(fl_ctx) container_of(fl_ctx, jl_ast_context_t, fl) diff --git a/src/atomics.h b/src/atomics.h deleted file mode 100644 index 4a33368745aa1..0000000000000 --- a/src/atomics.h +++ /dev/null @@ -1,163 +0,0 @@ -// This file is a part of Julia. License is MIT: https://julialang.org/license - -#ifndef JL_ATOMICS_H -#define JL_ATOMICS_H - -// Low-level atomic operations - -#if defined(__i386__) && defined(__GNUC__) && !defined(__SSE2__) -# error Julia can only be built for architectures above Pentium 4. Pass -march=pentium4, or set MARCH=pentium4 and ensure that -march is not passed separately with an older architecture. -#endif -#ifdef _COMPILER_MICROSOFT_ -# include -# include -#endif -#if defined(_CPU_X86_64_) || defined(_CPU_X86_) -# include -#endif -#ifndef _OS_WINDOWS_ -# include -#endif -#include - -enum jl_memory_order { - jl_memory_order_unspecified = -2, - jl_memory_order_invalid = -1, - jl_memory_order_notatomic = 0, - jl_memory_order_unordered, - jl_memory_order_monotonic, - jl_memory_order_consume, - jl_memory_order_acquire, - jl_memory_order_release, - jl_memory_order_acq_rel, - jl_memory_order_seq_cst -}; - -/** - * Thread synchronization primitives: - * - * These roughly follows the c11/c++11 memory model and the act as memory - * barriers at both the compiler level and the hardware level. - * The only exception is the GC safepoint and GC state transitions for which - * we use only a compiler (signal) barrier and use the signal handler to do the - * synchronization in order to lower the mutator overhead as much as possible. - * - * We use the compiler intrinsics to implement a similar API to the c11/c++11 - * one instead of using it directly because, we need interoperability between - * code written in different languages. The current c++ standard (c++14) does - * not allow using c11 atomic functions or types and there's currently no - * guarantee that the two types are compatible (although most of them probably - * are). We also need to access these atomic variables from the LLVM JIT code - * which is very hard unless the layout of the object is fully specified. - */ -#define jl_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST) -#define jl_fence_release() __atomic_thread_fence(__ATOMIC_RELEASE) -#define jl_signal_fence() __atomic_signal_fence(__ATOMIC_SEQ_CST) - - -# define jl_atomic_fetch_add_relaxed(obj, arg) \ - __atomic_fetch_add(obj, arg, __ATOMIC_RELAXED) -# define jl_atomic_fetch_add(obj, arg) \ - __atomic_fetch_add(obj, arg, __ATOMIC_SEQ_CST) -# define jl_atomic_add_fetch(obj, arg) \ - __atomic_add_fetch(obj, arg, __ATOMIC_SEQ_CST) -# define jl_atomic_fetch_and_relaxed(obj, arg) \ - __atomic_fetch_and(obj, arg, __ATOMIC_RELAXED) -# define jl_atomic_fetch_and(obj, arg) \ - __atomic_fetch_and(obj, arg, __ATOMIC_SEQ_CST) -# define jl_atomic_fetch_or_relaxed(obj, arg) \ - __atomic_fetch_or(obj, arg, __ATOMIC_RELAXED) -# define jl_atomic_fetch_or(obj, arg) \ - __atomic_fetch_or(obj, arg, __ATOMIC_SEQ_CST) -# define jl_atomic_cmpswap(obj, expected, desired) \ - __atomic_compare_exchange_n(obj, expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) -# define jl_atomic_cmpswap_relaxed(obj, expected, desired) \ - __atomic_compare_exchange_n(obj, expected, desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED) -// TODO: Maybe add jl_atomic_cmpswap_weak for spin lock -# define jl_atomic_exchange(obj, desired) \ - __atomic_exchange_n(obj, desired, __ATOMIC_SEQ_CST) -# define jl_atomic_exchange_relaxed(obj, desired) \ - __atomic_exchange_n(obj, desired, __ATOMIC_RELAXED) -# define jl_atomic_store(obj, val) \ - __atomic_store_n(obj, val, __ATOMIC_SEQ_CST) -# define jl_atomic_store_relaxed(obj, val) \ - __atomic_store_n(obj, val, __ATOMIC_RELAXED) - -# if defined(__clang__) || defined(__ICC) || defined(__INTEL_COMPILER) || \ - !(defined(_CPU_X86_) || defined(_CPU_X86_64_)) -// ICC and Clang doesn't have this bug... -# define jl_atomic_store_release(obj, val) \ - __atomic_store_n(obj, val, __ATOMIC_RELEASE) -# else -// Workaround a GCC bug when using store with release order by using the -// stronger version instead. -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67458 -// fixed in https://gcc.gnu.org/git/?p=gcc.git&a=commit;h=d8c40eff56f69877b33c697ded756d50fde90c27 -# define jl_atomic_store_release(obj, val) do { \ - jl_signal_fence(); \ - __atomic_store_n(obj, val, __ATOMIC_RELEASE); \ - } while (0) -# endif -# define jl_atomic_load(obj) \ - __atomic_load_n(obj, __ATOMIC_SEQ_CST) -# define jl_atomic_load_acquire(obj) \ - __atomic_load_n(obj, __ATOMIC_ACQUIRE) -#ifdef JL_TSAN_ENABLED -// For the sake of tsan, call these loads consume ordering since they will act -// as such on the processors we support while normally, the compiler would -// upgrade this to acquire ordering, which is strong (and slower) than we want. -# define jl_atomic_load_relaxed(obj) \ - __atomic_load_n(obj, __ATOMIC_CONSUME) -#else -# define jl_atomic_load_relaxed(obj) \ - __atomic_load_n(obj, __ATOMIC_RELAXED) -#endif - -#ifdef __clang_analyzer__ -// for the purposes of the analyzer, we can turn these into non-atomic expressions with similar properties -// (for the sake of the analyzer, we don't care if it is an exact match for behavior) - -#undef jl_atomic_exchange -#undef jl_atomic_exchange_relaxed -#define jl_atomic_exchange(obj, desired) \ - (__extension__({ \ - __typeof__((obj)) p__analyzer__ = (obj); \ - __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ - *p__analyzer__ = (desired); \ - temp__analyzer__; \ - })) -#define jl_atomic_exchange_relaxed jl_atomic_exchange - -#undef jl_atomic_cmpswap -#undef jl_atomic_cmpswap_relaxed -#define jl_atomic_cmpswap(obj, expected, desired) \ - (__extension__({ \ - __typeof__((obj)) p__analyzer__ = (obj); \ - __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ - __typeof__((expected)) x__analyzer__ = (expected); \ - if (temp__analyzer__ == *x__analyzer__) \ - *p__analyzer__ = (desired); \ - else \ - *x__analyzer__ = temp__analyzer__; \ - temp__analyzer__ == *x__analyzer__; \ - })) -#define jl_atomic_cmpswap_relaxed jl_atomic_cmpswap - -#undef jl_atomic_store -#undef jl_atomic_store_release -#undef jl_atomic_store_relaxed -#define jl_atomic_store(obj, val) (*(obj) = (val)) -#define jl_atomic_store_release jl_atomic_store -#define jl_atomic_store_relaxed jl_atomic_store - -#undef jl_atomic_load -#undef jl_atomic_load_acquire -#undef jl_atomic_load_relaxed -#define jl_atomic_load(obj) (*(obj)) -#define jl_atomic_load_acquire jl_atomic_load -#define jl_atomic_load_relaxed jl_atomic_load - -#endif - - -#endif // JL_ATOMICS_H diff --git a/src/builtins.c b/src/builtins.c index b186aadbed5d2..01723bf81733e 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -706,7 +706,7 @@ static jl_value_t *do_apply( jl_value_t **args, uint32_t nargs, jl_value_t *iter } if (arg_heap) { // optimization: keep only the first root, free the others -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ ((void**)roots)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(1); #endif } diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index 23d8b7437b823..344f044737f9d 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -205,7 +205,7 @@ static intptr_t get_anon_hdl(void) return -1; } -static size_t map_offset = 0; +static _Atomic(size_t) map_offset{0}; // Multiple of 128MB. // Hopefully no one will set a ulimit for this to be a problem... static constexpr size_t map_size_inc_default = 128 * 1024 * 1024; @@ -239,7 +239,7 @@ static intptr_t init_shared_map() anon_hdl = get_anon_hdl(); if (anon_hdl == -1) return -1; - map_offset = 0; + jl_atomic_store_relaxed(&map_offset, 0); map_size = get_map_size_inc(); int ret = ftruncate(anon_hdl, map_size); if (ret != 0) { diff --git a/src/codegen.cpp b/src/codegen.cpp index a5eda27b95424..616da07c13939 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3655,12 +3655,14 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const else { jl_value_t *ci = ctx.params->lookup(mi, ctx.world, ctx.world); // TODO: need to use the right pair world here jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; - if (ci != jl_nothing && codeinst->invoke != jl_fptr_sparam) { // check if we know we definitely can't handle this specptr - if (codeinst->invoke == jl_fptr_const_return) { + if (ci != jl_nothing) { + auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); + // check if we know how to handle this specptr + if (invoke == jl_fptr_const_return) { result = mark_julia_const(codeinst->rettype_const); handled = true; } - else { + else if (invoke != jl_fptr_sparam) { bool specsig, needsparams; std::tie(specsig, needsparams) = uses_specsig(mi, codeinst->rettype, ctx.params->prefer_specsig); std::string name; @@ -3669,9 +3671,11 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const if (ctx.use_cache) { // optimization: emit the correct name immediately, if we know it // TODO: use `emitted` map here too to try to consolidate names? - if (codeinst->specptr.fptr) { - if (specsig ? codeinst->isspecsig : codeinst->invoke == jl_fptr_args) { - protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); + auto fptr = jl_atomic_load_relaxed(&codeinst->specptr.fptr); + if (fptr) { + if (specsig ? codeinst->isspecsig : invoke == jl_fptr_args) { + protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, codeinst); need_to_emit = false; } } @@ -5040,8 +5044,9 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod ctx.builder.SetInsertPoint(b0); Function *theFunc; Value *theFarg; - if (params.cache && codeinst->invoke != NULL) { - StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->invoke, codeinst); + auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); + if (params.cache && invoke != NULL) { + StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, codeinst); theFunc = cast( M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(jl_LLVMContext)).getCallee()); theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst); @@ -5726,7 +5731,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // some sanity checking and check whether there's a vararg size_t nargt = jl_svec_len(argt); bool isVa = (nargt > 0 && jl_is_vararg(jl_svecref(argt, nargt - 1))); - assert(!isVa); + assert(!isVa); (void)isVa; jl_array_t *closure_types = NULL; jl_value_t *sigt = NULL; // dispatch-sig = type signature with Ref{} annotations removed and applied to the env @@ -7803,12 +7808,14 @@ void jl_compile_workqueue( "invalid world for code-instance"); StringRef preal_decl = ""; bool preal_specsig = false; - if (params.cache && codeinst->invoke != NULL) { - if (codeinst->invoke == jl_fptr_args) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); + if (params.cache && invoke != NULL) { + auto fptr = jl_atomic_load_relaxed(&codeinst->specptr.fptr); + if (invoke == jl_fptr_args) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, codeinst); } else if (codeinst->isspecsig) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, codeinst); preal_specsig = true; } } diff --git a/src/datatype.c b/src/datatype.c index cf4c0cd94b5f2..31b6bc93cd014 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -48,9 +48,9 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo jl_methtable_type); mt->name = jl_demangle_typename(name); mt->module = module; - mt->defs = jl_nothing; - mt->leafcache = (jl_array_t*)jl_an_empty_vec_any; - mt->cache = jl_nothing; + jl_atomic_store_relaxed(&mt->defs, jl_nothing); + jl_atomic_store_relaxed(&mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&mt->cache, jl_nothing); mt->max_args = 0; mt->kwsorter = NULL; mt->backedges = NULL; @@ -69,8 +69,8 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->name = name; tn->module = module; tn->wrapper = NULL; - tn->cache = jl_emptysvec; - tn->linearcache = jl_emptysvec; + jl_atomic_store_relaxed(&tn->cache, jl_emptysvec); + jl_atomic_store_relaxed(&tn->linearcache, jl_emptysvec); tn->names = NULL; tn->hash = bitmix(bitmix(module ? module->build_id : 0, name->hash), 0xa1ada1da); tn->abstract = abstract; @@ -727,23 +727,23 @@ JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt) #if MAX_ATOMIC_SIZE > MAX_POINTERATOMIC_SIZE #error MAX_ATOMIC_SIZE too large #endif +#if MAX_ATOMIC_SIZE >= 16 && !defined(_P64) +#error 12 byte GC pool size alignment unimplemented for 32-bit +#endif #if MAX_POINTERATOMIC_SIZE > 16 #error MAX_POINTERATOMIC_SIZE too large #endif -#if MAX_POINTERATOMIC_SIZE >= 16 -#ifndef _P64 -#error 12 byte GC pool size not implemented for 32-bit -#endif -typedef __uint128_t uint128_t; -typedef uint128_t jl_uatomicmax_t; -#else -typedef uint64_t jl_uatomicmax_t; -#endif - #if BYTE_ORDER != LITTLE_ENDIAN #error using masks for atomics (instead of memcpy like nb == 16) assumes little endian #endif +#if MAX_POINTERATOMIC_SIZE >= 16 +typedef struct _jl_uint128_t { + uint64_t a; + uint64_t b; +} jl_uint128_t; +#endif + static inline uint32_t zext_read32(const jl_value_t *x, size_t nb) JL_NOTSAFEPOINT { uint32_t y = *(uint32_t*)x; @@ -769,11 +769,11 @@ static inline uint64_t zext_read64(const jl_value_t *x, size_t nb) JL_NOTSAFEPOI #endif #if MAX_POINTERATOMIC_SIZE >= 16 -static inline uint128_t zext_read128(const jl_value_t *x, size_t nb) JL_NOTSAFEPOINT +static inline jl_uint128_t zext_read128(const jl_value_t *x, size_t nb) JL_NOTSAFEPOINT { - uint128_t y = 0; + jl_uint128_t y = {0}; if (nb == 16) - y = *(uint128_t*)x; + y = *(jl_uint128_t*)x; else memcpy(&y, x, nb); return y; @@ -814,34 +814,34 @@ JL_DLLEXPORT jl_value_t *jl_atomic_new_bits(jl_value_t *dt, const char *data) size_t nb = jl_datatype_size(bt); // some types have special pools to minimize allocations if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance - if (bt == jl_bool_type) return (1 & jl_atomic_load((int8_t*)data)) ? jl_true : jl_false; - if (bt == jl_uint8_type) return jl_box_uint8(jl_atomic_load((uint8_t*)data)); - if (bt == jl_int64_type) return jl_box_int64(jl_atomic_load((int64_t*)data)); - if (bt == jl_int32_type) return jl_box_int32(jl_atomic_load((int32_t*)data)); - if (bt == jl_int8_type) return jl_box_int8(jl_atomic_load((int8_t*)data)); - if (bt == jl_int16_type) return jl_box_int16(jl_atomic_load((int16_t*)data)); - if (bt == jl_uint64_type) return jl_box_uint64(jl_atomic_load((uint64_t*)data)); - if (bt == jl_uint32_type) return jl_box_uint32(jl_atomic_load((uint32_t*)data)); - if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_load((uint16_t*)data)); - if (bt == jl_char_type) return jl_box_char(jl_atomic_load((uint32_t*)data)); + if (bt == jl_bool_type) return (1 & jl_atomic_load((_Atomic(int8_t)*)data)) ? jl_true : jl_false; + if (bt == jl_uint8_type) return jl_box_uint8(jl_atomic_load((_Atomic(uint8_t)*)data)); + if (bt == jl_int64_type) return jl_box_int64(jl_atomic_load((_Atomic(int64_t)*)data)); + if (bt == jl_int32_type) return jl_box_int32(jl_atomic_load((_Atomic(int32_t)*)data)); + if (bt == jl_int8_type) return jl_box_int8(jl_atomic_load((_Atomic(int8_t)*)data)); + if (bt == jl_int16_type) return jl_box_int16(jl_atomic_load((_Atomic(int16_t)*)data)); + if (bt == jl_uint64_type) return jl_box_uint64(jl_atomic_load((_Atomic(uint64_t)*)data)); + if (bt == jl_uint32_type) return jl_box_uint32(jl_atomic_load((_Atomic(uint32_t)*)data)); + if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_load((_Atomic(uint16_t)*)data)); + if (bt == jl_char_type) return jl_box_char(jl_atomic_load((_Atomic(uint32_t)*)data)); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, nb, bt); // data is aligned to the power of two, // we will write too much of v, but the padding should exist if (nb == 1) - *(uint8_t*) v = jl_atomic_load((uint8_t*)data); + *(uint8_t*) v = jl_atomic_load((_Atomic(uint8_t)*)data); else if (nb <= 2) - *(uint16_t*)v = jl_atomic_load((uint16_t*)data); + *(uint16_t*)v = jl_atomic_load((_Atomic(uint16_t)*)data); else if (nb <= 4) - *(uint32_t*)v = jl_atomic_load((uint32_t*)data); + *(uint32_t*)v = jl_atomic_load((_Atomic(uint32_t)*)data); #if MAX_POINTERATOMIC_SIZE >= 8 else if (nb <= 8) - *(uint64_t*)v = jl_atomic_load((uint64_t*)data); + *(uint64_t*)v = jl_atomic_load((_Atomic(uint64_t)*)data); #endif #if MAX_POINTERATOMIC_SIZE >= 16 else if (nb <= 16) - *(uint128_t*)v = jl_atomic_load((uint128_t*)data); + *(jl_uint128_t*)v = jl_atomic_load((_Atomic(jl_uint128_t)*)data); #endif else abort(); @@ -857,18 +857,18 @@ JL_DLLEXPORT void jl_atomic_store_bits(char *dst, const jl_value_t *src, int nb) if (nb == 0) ; else if (nb == 1) - jl_atomic_store((uint8_t*)dst, *(uint8_t*)src); + jl_atomic_store((_Atomic(uint8_t)*)dst, *(uint8_t*)src); else if (nb == 2) - jl_atomic_store((uint16_t*)dst, *(uint16_t*)src); + jl_atomic_store((_Atomic(uint16_t)*)dst, *(uint16_t*)src); else if (nb <= 4) - jl_atomic_store((uint32_t*)dst, zext_read32(src, nb)); + jl_atomic_store((_Atomic(uint32_t)*)dst, zext_read32(src, nb)); #if MAX_POINTERATOMIC_SIZE >= 8 else if (nb <= 8) - jl_atomic_store((uint64_t*)dst, zext_read64(src, nb)); + jl_atomic_store((_Atomic(uint64_t)*)dst, zext_read64(src, nb)); #endif #if MAX_POINTERATOMIC_SIZE >= 16 else if (nb <= 16) - jl_atomic_store((uint128_t*)dst, zext_read128(src, nb)); + jl_atomic_store((_Atomic(jl_uint128_t)*)dst, zext_read128(src, nb)); #endif else abort(); @@ -881,32 +881,32 @@ JL_DLLEXPORT jl_value_t *jl_atomic_swap_bits(jl_value_t *dt, char *dst, const jl jl_datatype_t *bt = (jl_datatype_t*)dt; // some types have special pools to minimize allocations if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance - if (bt == jl_bool_type) return (1 & jl_atomic_exchange((int8_t*)dst, 1 & *(int8_t*)src)) ? jl_true : jl_false; - if (bt == jl_uint8_type) return jl_box_uint8(jl_atomic_exchange((uint8_t*)dst, *(int8_t*)src)); - if (bt == jl_int64_type) return jl_box_int64(jl_atomic_exchange((int64_t*)dst, *(int64_t*)src)); - if (bt == jl_int32_type) return jl_box_int32(jl_atomic_exchange((int32_t*)dst, *(int32_t*)src)); - if (bt == jl_int8_type) return jl_box_int8(jl_atomic_exchange((int8_t*)dst, *(int8_t*)src)); - if (bt == jl_int16_type) return jl_box_int16(jl_atomic_exchange((int16_t*)dst, *(int16_t*)src)); - if (bt == jl_uint64_type) return jl_box_uint64(jl_atomic_exchange((uint64_t*)dst, *(uint64_t*)src)); - if (bt == jl_uint32_type) return jl_box_uint32(jl_atomic_exchange((uint32_t*)dst, *(uint32_t*)src)); - if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_exchange((uint16_t*)dst, *(uint16_t*)src)); - if (bt == jl_char_type) return jl_box_char(jl_atomic_exchange((uint32_t*)dst, *(uint32_t*)src)); + if (bt == jl_bool_type) return (1 & jl_atomic_exchange((_Atomic(int8_t)*)dst, 1 & *(int8_t*)src)) ? jl_true : jl_false; + if (bt == jl_uint8_type) return jl_box_uint8(jl_atomic_exchange((_Atomic(uint8_t)*)dst, *(int8_t*)src)); + if (bt == jl_int64_type) return jl_box_int64(jl_atomic_exchange((_Atomic(int64_t)*)dst, *(int64_t*)src)); + if (bt == jl_int32_type) return jl_box_int32(jl_atomic_exchange((_Atomic(int32_t)*)dst, *(int32_t*)src)); + if (bt == jl_int8_type) return jl_box_int8(jl_atomic_exchange((_Atomic(int8_t)*)dst, *(int8_t*)src)); + if (bt == jl_int16_type) return jl_box_int16(jl_atomic_exchange((_Atomic(int16_t)*)dst, *(int16_t*)src)); + if (bt == jl_uint64_type) return jl_box_uint64(jl_atomic_exchange((_Atomic(uint64_t)*)dst, *(uint64_t*)src)); + if (bt == jl_uint32_type) return jl_box_uint32(jl_atomic_exchange((_Atomic(uint32_t)*)dst, *(uint32_t*)src)); + if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_exchange((_Atomic(uint16_t)*)dst, *(uint16_t*)src)); + if (bt == jl_char_type) return jl_box_char(jl_atomic_exchange((_Atomic(uint32_t)*)dst, *(uint32_t*)src)); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, jl_datatype_size(bt), bt); if (nb == 1) - *(uint8_t*)v = jl_atomic_exchange((uint8_t*)dst, *(uint8_t*)src); + *(uint8_t*)v = jl_atomic_exchange((_Atomic(uint8_t)*)dst, *(uint8_t*)src); else if (nb == 2) - *(uint16_t*)v = jl_atomic_exchange((uint16_t*)dst, *(uint16_t*)src); + *(uint16_t*)v = jl_atomic_exchange((_Atomic(uint16_t)*)dst, *(uint16_t*)src); else if (nb <= 4) - *(uint32_t*)v = jl_atomic_exchange((uint32_t*)dst, zext_read32(src, nb)); + *(uint32_t*)v = jl_atomic_exchange((_Atomic(uint32_t)*)dst, zext_read32(src, nb)); #if MAX_POINTERATOMIC_SIZE >= 8 else if (nb <= 8) - *(uint64_t*)v = jl_atomic_exchange((uint64_t*)dst, zext_read64(src, nb)); + *(uint64_t*)v = jl_atomic_exchange((_Atomic(uint64_t)*)dst, zext_read64(src, nb)); #endif #if MAX_POINTERATOMIC_SIZE >= 16 else if (nb <= 16) - *(uint128_t*)v = jl_atomic_exchange((uint128_t*)dst, zext_read128(src, nb)); + *(jl_uint128_t*)v = jl_atomic_exchange((_Atomic(jl_uint128_t)*)dst, zext_read128(src, nb)); #endif else abort(); @@ -923,29 +923,29 @@ JL_DLLEXPORT int jl_atomic_bool_cmpswap_bits(char *dst, const jl_value_t *expect } else if (nb == 1) { uint8_t y = *(uint8_t*)expected; - success = jl_atomic_cmpswap((uint8_t*)dst, &y, *(uint8_t*)src); + success = jl_atomic_cmpswap((_Atomic(uint8_t)*)dst, &y, *(uint8_t*)src); } else if (nb == 2) { uint16_t y = *(uint16_t*)expected; - success = jl_atomic_cmpswap((uint16_t*)dst, &y, *(uint16_t*)src); + success = jl_atomic_cmpswap((_Atomic(uint16_t)*)dst, &y, *(uint16_t*)src); } else if (nb <= 4) { uint32_t y = zext_read32(expected, nb); uint32_t z = zext_read32(src, nb); - success = jl_atomic_cmpswap((uint32_t*)dst, &y, z); + success = jl_atomic_cmpswap((_Atomic(uint32_t)*)dst, &y, z); } #if MAX_POINTERATOMIC_SIZE >= 8 else if (nb <= 8) { uint64_t y = zext_read64(expected, nb); uint64_t z = zext_read64(src, nb); - success = jl_atomic_cmpswap((uint64_t*)dst, &y, z); + success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, &y, z); } #endif #if MAX_POINTERATOMIC_SIZE >= 16 else if (nb <= 16) { - uint128_t y = zext_read128(expected, nb); - uint128_t z = zext_read128(src, nb); - success = jl_atomic_cmpswap((uint128_t*)dst, &y, z); + jl_uint128_t y = zext_read128(expected, nb); + jl_uint128_t z = zext_read128(src, nb); + success = jl_atomic_cmpswap((_Atomic(jl_uint128_t)*)dst, &y, z); } #endif else { @@ -972,10 +972,10 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t if (dt == et) { *y8 = *(uint8_t*)expected; uint8_t z8 = *(uint8_t*)src; - success = jl_atomic_cmpswap((uint8_t*)dst, y8, z8); + success = jl_atomic_cmpswap((_Atomic(uint8_t)*)dst, y8, z8); } else { - *y8 = jl_atomic_load((uint8_t*)dst); + *y8 = jl_atomic_load((_Atomic(uint8_t)*)dst); success = 0; } } @@ -985,10 +985,10 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t if (dt == et) { *y16 = *(uint16_t*)expected; uint16_t z16 = *(uint16_t*)src; - success = jl_atomic_cmpswap((uint16_t*)dst, y16, z16); + success = jl_atomic_cmpswap((_Atomic(uint16_t)*)dst, y16, z16); } else { - *y16 = jl_atomic_load((uint16_t*)dst); + *y16 = jl_atomic_load((_Atomic(uint16_t)*)dst); success = 0; } } @@ -998,13 +998,13 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t *y32 = zext_read32(expected, nb); uint32_t z32 = zext_read32(src, nb); while (1) { - success = jl_atomic_cmpswap((uint32_t*)dst, y32, z32); + success = jl_atomic_cmpswap((_Atomic(uint32_t)*)dst, y32, z32); if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt)) break; } } else { - *y32 = jl_atomic_load((uint32_t*)dst); + *y32 = jl_atomic_load((_Atomic(uint32_t)*)dst); success = 0; } } @@ -1015,31 +1015,31 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t *y64 = zext_read64(expected, nb); uint64_t z64 = zext_read64(src, nb); while (1) { - success = jl_atomic_cmpswap((uint64_t*)dst, y64, z64); + success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, y64, z64); if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt)) break; } } else { - *y64 = jl_atomic_load((uint64_t*)dst); + *y64 = jl_atomic_load((_Atomic(uint64_t)*)dst); success = 0; } } #endif #if MAX_POINTERATOMIC_SIZE >= 16 else if (nb <= 16) { - uint128_t *y128 = (uint128_t*)y; + jl_uint128_t *y128 = (jl_uint128_t*)y; if (dt == et) { *y128 = zext_read128(expected, nb); - uint128_t z128 = zext_read128(src, nb); + jl_uint128_t z128 = zext_read128(src, nb); while (1) { - success = jl_atomic_cmpswap((uint128_t*)dst, y128, z128); + success = jl_atomic_cmpswap((_Atomic(jl_uint128_t)*)dst, y128, z128); if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt)) break; } } else { - *y128 = jl_atomic_load((uint128_t*)dst); + *y128 = jl_atomic_load((_Atomic(jl_uint128_t)*)dst); success = 0; } } @@ -1394,7 +1394,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i) jl_bounds_error_int(v, i + 1); size_t offs = jl_field_offset(st, i); if (jl_field_isptr(st, i)) { - return jl_atomic_load_relaxed((jl_value_t**)((char*)v + offs)); + return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs)); } jl_value_t *ty = jl_field_type_concrete(st, i); int isatomic = jl_field_isatomic(st, i); @@ -1431,7 +1431,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field_noalloc(jl_value_t *v JL_PROPAGATES_RO assert(i < jl_datatype_nfields(st)); size_t offs = jl_field_offset(st,i); assert(jl_field_isptr(st,i)); - return jl_atomic_load_relaxed((jl_value_t**)((char*)v + offs)); + return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs)); } JL_DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) @@ -1472,7 +1472,7 @@ void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, return; } if (jl_field_isptr(st, i)) { - jl_atomic_store_relaxed((jl_value_t**)((char*)v + offs), rhs); + jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs), rhs); jl_gc_wb(v, rhs); } else { @@ -1522,9 +1522,9 @@ jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_ jl_value_t *r; if (jl_field_isptr(st, i)) { if (isatomic) - r = jl_atomic_exchange((jl_value_t**)((char*)v + offs), rhs); + r = jl_atomic_exchange((_Atomic(jl_value_t*)*)((char*)v + offs), rhs); else - r = jl_atomic_exchange_relaxed((jl_value_t**)((char*)v + offs), rhs); + r = jl_atomic_exchange_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs), rhs); jl_gc_wb(v, rhs); } else { @@ -1594,7 +1594,7 @@ jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_valu if (!jl_isa(y, ty)) jl_type_error("modifyfield!", ty, y); if (jl_field_isptr(st, i)) { - jl_value_t **p = (jl_value_t**)((char*)v + offs); + _Atomic(jl_value_t*) *p = (_Atomic(jl_value_t*)*)((char*)v + offs); if (isatomic ? jl_atomic_cmpswap(p, &r, y) : jl_atomic_cmpswap_relaxed(p, &r, y)) break; } @@ -1673,7 +1673,7 @@ jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val jl_datatype_t *rettyp = jl_apply_cmpswap_type(ty); JL_GC_PROMISE_ROOTED(rettyp); // (JL_ALWAYS_LEAFTYPE) if (jl_field_isptr(st, i)) { - jl_value_t **p = (jl_value_t**)((char*)v + offs); + _Atomic(jl_value_t*) *p = (_Atomic(jl_value_t*)*)((char*)v + offs); int success; while (1) { success = isatomic ? jl_atomic_cmpswap(p, &r, rhs) : jl_atomic_cmpswap_relaxed(p, &r, rhs); @@ -1759,7 +1759,7 @@ JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i) JL_NOTSAFEPOINT { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); size_t offs = jl_field_offset(st, i); - jl_value_t **fld = (jl_value_t**)((char*)v + offs); + _Atomic(jl_value_t*) *fld = (_Atomic(jl_value_t*)*)((char*)v + offs); if (!jl_field_isptr(st, i)) { jl_datatype_t *ft = (jl_datatype_t*)jl_field_type_concrete(st, i); if (!jl_is_datatype(ft) || ft->layout->first_ptr < 0) diff --git a/src/dlload.c b/src/dlload.c index 900fbe94b39b0..920bd3d00d43b 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -324,7 +324,7 @@ JL_DLLEXPORT int jl_dlsym(void *handle, const char *symbol, void ** value, int t char err[256]; win32_formatmessage(GetLastError(), err, sizeof(err)); #endif -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ // Hide the error throwing from the analyser since there isn't a way to express // "safepoint only when throwing error" currently. jl_errorf("could not load symbol \"%s\":\n%s", symbol, err); diff --git a/src/dump.c b/src/dump.c index f7a0ced4a6ab6..c88889027720c 100644 --- a/src/dump.c +++ b/src/dump.c @@ -349,13 +349,13 @@ static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m) jl_serialize_value(s, (jl_value_t*)table[i]); jl_binding_t *b = (jl_binding_t*)table[i+1]; jl_serialize_value(s, b->name); - jl_value_t *e = b->value; + jl_value_t *e = jl_atomic_load_relaxed(&b->value); if (!b->constp && e && jl_is_cpointer(e) && jl_unbox_voidpointer(e) != (void*)-1 && jl_unbox_voidpointer(e) != NULL) // reset Ptr fields to C_NULL (but keep MAP_FAILED / INVALID_HANDLE) jl_serialize_cnull(s, jl_typeof(e)); else jl_serialize_value(s, e); - jl_serialize_value(s, b->globalref); + jl_serialize_value(s, jl_atomic_load_relaxed(&b->globalref)); jl_serialize_value(s, b->owner); write_int8(s->s, (b->deprecated<<3) | (b->constp<<2) | (b->exportp<<1) | (b->imported)); } @@ -659,7 +659,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li if (!(serialization_mode & METHOD_INTERNAL)) return; jl_serialize_value(s, m->specializations); - jl_serialize_value(s, m->speckeyset); + jl_serialize_value(s, jl_atomic_load_relaxed(&m->speckeyset)); jl_serialize_value(s, (jl_value_t*)m->name); jl_serialize_value(s, (jl_value_t*)m->file); write_int32(s->s, m->line); @@ -1509,8 +1509,9 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_ } m->specializations = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->specializations); jl_gc_wb(m, m->specializations); - m->speckeyset = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->speckeyset); - jl_gc_wb(m, m->speckeyset); + jl_array_t *speckeyset = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->speckeyset); + jl_atomic_store_relaxed(&m->speckeyset, speckeyset); + jl_gc_wb(m, speckeyset); m->name = (jl_sym_t*)jl_deserialize_value(s, NULL); jl_gc_wb(m, m->name); m->file = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1647,10 +1648,12 @@ static jl_value_t *jl_deserialize_value_module(jl_serializer_state *s) JL_GC_DIS break; jl_binding_t *b = jl_get_binding_wr(m, asname, 1); b->name = (jl_sym_t*)jl_deserialize_value(s, (jl_value_t**)&b->name); - b->value = jl_deserialize_value(s, &b->value); - if (b->value != NULL) jl_gc_wb(m, b->value); - b->globalref = jl_deserialize_value(s, &b->globalref); - if (b->globalref != NULL) jl_gc_wb(m, b->globalref); + jl_value_t *bvalue = jl_deserialize_value(s, (jl_value_t**)&b->value); + *(jl_value_t**)&b->value = bvalue; + if (bvalue != NULL) jl_gc_wb(m, bvalue); + jl_value_t *bglobalref = jl_deserialize_value(s, (jl_value_t**)&b->globalref); + *(jl_value_t**)&b->globalref = bglobalref; + if (bglobalref != NULL) jl_gc_wb(m, bglobalref); b->owner = (jl_module_t*)jl_deserialize_value(s, (jl_value_t**)&b->owner); if (b->owner != NULL) jl_gc_wb(m, b->owner); int8_t flags = read_int8(s->s); diff --git a/src/gc-stacks.c b/src/gc-stacks.c index 3708531e9b405..a621bd96ab3d4 100644 --- a/src/gc-stacks.c +++ b/src/gc-stacks.c @@ -23,7 +23,7 @@ #define MIN_STACK_MAPPINGS_PER_POOL 5 const size_t jl_guard_size = (4096 * 8); -static uint32_t num_stack_mappings = 0; +static _Atomic(uint32_t) num_stack_mappings = 0; #ifdef _OS_WINDOWS_ #define MAP_FAILED NULL diff --git a/src/gc.c b/src/gc.c index 8b9688833c5dd..1ba8f65c09efd 100644 --- a/src/gc.c +++ b/src/gc.c @@ -132,7 +132,7 @@ static jl_mutex_t gc_cache_lock; // Flag that tells us whether we need to support conservative marking // of objects. -static int support_conservative_marking = 0; +static _Atomic(int) support_conservative_marking = 0; /** * Note about GC synchronization: @@ -166,7 +166,7 @@ static int support_conservative_marking = 0; * finalizers in unmanaged (GC safe) mode. */ -jl_gc_num_t gc_num = {0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +jl_gc_num_t gc_num = {0}; static size_t last_long_collect_interval; pagetable_t memory_map; @@ -181,7 +181,7 @@ bigval_t *big_objects_marked = NULL; // `to_finalize` should not have tagged pointers. arraylist_t finalizer_list_marked; arraylist_t to_finalize; -int jl_gc_have_pending_finalizers = 0; +JL_DLLEXPORT _Atomic(int) jl_gc_have_pending_finalizers = 0; NOINLINE uintptr_t gc_get_stack_ptr(void) { @@ -298,7 +298,7 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, // This way, the mutation should not conflict with the owning thread, // which only writes to locations later than `len` // and will not resize the buffer without acquiring the lock. - size_t len = need_sync ? jl_atomic_load_acquire(&list->len) : list->len; + size_t len = need_sync ? jl_atomic_load_acquire((_Atomic(size_t)*)&list->len) : list->len; size_t oldlen = len; void **items = list->items; size_t j = 0; @@ -331,7 +331,7 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, // The `memset` (like any other content mutation) has to be done // **before** the `cmpxchg` which publishes the length. memset(&items[len], 0, (oldlen - len) * sizeof(void*)); - jl_atomic_cmpswap(&list->len, &oldlen, len); + jl_atomic_cmpswap((_Atomic(size_t)*)&list->len, &oldlen, len); } else { list->len = len; @@ -484,7 +484,7 @@ void jl_gc_run_all_finalizers(jl_task_t *ct) static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT { - assert(ptls->gc_state == 0); + assert(jl_atomic_load_relaxed(&ptls->gc_state) == 0); arraylist_t *a = &ptls->finalizers; // This acquire load and the release store at the end are used to // synchronize with `finalize_object` on another thread. Apart from the GC, @@ -493,7 +493,7 @@ static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT // (only one thread since it needs to acquire the finalizer lock). // Similar to `finalize_object`, all content mutation has to be done // between the acquire and the release of the length. - size_t oldlen = jl_atomic_load_acquire(&a->len); + size_t oldlen = jl_atomic_load_acquire((_Atomic(size_t)*)&a->len); if (__unlikely(oldlen + 2 > a->max)) { JL_LOCK_NOGC(&finalizers_lock); // `a->len` might have been modified. @@ -507,7 +507,7 @@ static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT void **items = a->items; items[oldlen] = v; items[oldlen + 1] = f; - jl_atomic_store_release(&a->len, oldlen + 2); + jl_atomic_store_release((_Atomic(size_t)*)&a->len, oldlen + 2); } JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f) JL_NOTSAFEPOINT @@ -738,7 +738,7 @@ STATIC_INLINE int gc_setmark_tag(jl_taggedvalue_t *o, uint8_t mark_mode, assert((tag & 0x3) == mark_mode); } *bits = mark_mode; - tag = jl_atomic_exchange_relaxed(&o->header, tag); + tag = jl_atomic_exchange_relaxed((_Atomic(uintptr_t)*)&o->header, tag); verify_val(jl_valueof(o)); return !gc_marked(tag); } @@ -781,7 +781,8 @@ STATIC_INLINE void gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o, jl_assume(page); if (mark_mode == GC_OLD_MARKED) { ptls->gc_cache.perm_scanned_bytes += page->osize; - jl_atomic_fetch_add_relaxed(&page->nold, 1); + static_assert(sizeof(_Atomic(uint16_t)) == sizeof(page->nold), ""); + jl_atomic_fetch_add_relaxed((_Atomic(uint16_t)*)&page->nold, 1); } else { ptls->gc_cache.scanned_bytes += page->osize; @@ -790,7 +791,7 @@ STATIC_INLINE void gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o, char *page_begin = gc_page_data(o) + GC_PAGE_OFFSET; int obj_id = (((char*)o) - page_begin) / page->osize; uint8_t *ages = page->ages + obj_id / 8; - jl_atomic_fetch_and_relaxed(ages, ~(1 << (obj_id % 8))); + jl_atomic_fetch_and_relaxed((_Atomic(uint8_t)*)ages, ~(1 << (obj_id % 8))); } } objprofile_count(jl_typeof(jl_valueof(o)), @@ -877,7 +878,7 @@ void jl_gc_force_mark_old(jl_ptls_t ptls, jl_value_t *v) JL_NOTSAFEPOINT static inline void maybe_collect(jl_ptls_t ptls) { - if (ptls->gc_num.allocd >= 0 || gc_debug_check_other()) { + if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || gc_debug_check_other()) { jl_gc_collect(JL_GC_AUTO); } else { @@ -956,8 +957,10 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) jl_throw(jl_memory_exception); gc_invoke_callbacks(jl_gc_cb_notify_external_alloc_t, gc_cblist_notify_external_alloc, (v, allocsz)); - ptls->gc_num.allocd += allocsz; - ptls->gc_num.bigalloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.bigalloc, + jl_atomic_load_relaxed(&ptls->gc_num.bigalloc) + 1); #ifdef MEMDEBUG memset(v, 0xee, allocsz); #endif @@ -1050,7 +1053,8 @@ void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT { jl_ptls_t ptls = jl_current_task->ptls; - ptls->gc_num.allocd += sz; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); } static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT @@ -1074,8 +1078,8 @@ static void reset_thread_gc_counts(void) JL_NOTSAFEPOINT for (int i = 0; i < jl_n_threads; i++) { jl_ptls_t ptls = jl_all_tls_states[i]; if (ptls) { - memset(&ptls->gc_num, 0, sizeof(jl_thread_gc_num_t)); - ptls->gc_num.allocd = -(int64_t)gc_num.interval; + memset(&ptls->gc_num, 0, sizeof(ptls->gc_num)); + jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval); } } } @@ -1198,13 +1202,15 @@ JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, // to workaround a llvm bug. // Ref https://llvm.org/bugs/show_bug.cgi?id=27190 jl_gc_pool_t *p = (jl_gc_pool_t*)((char*)ptls + pool_offset); - assert(ptls->gc_state == 0); + assert(jl_atomic_load_relaxed(&ptls->gc_state) == 0); #ifdef MEMDEBUG return jl_gc_big_alloc(ptls, osize); #endif maybe_collect(ptls); - ptls->gc_num.allocd += osize; - ptls->gc_num.poolalloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + osize); + jl_atomic_store_relaxed(&ptls->gc_num.poolalloc, + jl_atomic_load_relaxed(&ptls->gc_num.poolalloc) + 1); // first try to use the freelist jl_taggedvalue_t *v = p->freelist; if (v) { @@ -2398,8 +2404,8 @@ module_binding: { void *vb = jl_astaggedvalue(b); verify_parent1("module", binding->parent, &vb, "binding_buff"); (void)vb; - jl_value_t *value = b->value; - jl_value_t *globalref = b->globalref; + jl_value_t *value = jl_atomic_load_relaxed(&b->value); + jl_value_t *globalref = jl_atomic_load_relaxed(&b->globalref); if (value) { verify_parent2("module", binding->parent, &b->value, "binding(%s)", jl_symbol_name(b->name)); @@ -2800,9 +2806,11 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp) } } gc_mark_queue_obj(gc_cache, sp, jl_anytuple_type_type); - for (size_t i = 0; i < N_CALL_CACHE; i++) - if (call_cache[i]) - gc_mark_queue_obj(gc_cache, sp, call_cache[i]); + for (size_t i = 0; i < N_CALL_CACHE; i++) { + jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); + if (v != NULL) + gc_mark_queue_obj(gc_cache, sp, v); + } if (jl_all_methods != NULL) gc_mark_queue_obj(gc_cache, sp, jl_all_methods); if (_jl_debug_method_invalidation != NULL) @@ -2858,7 +2866,7 @@ static void sweep_finalizer_list(arraylist_t *list) } // collector entry point and control -static volatile uint32_t jl_gc_disable_counter = 1; +static _Atomic(uint32_t) jl_gc_disable_counter = 1; JL_DLLEXPORT int jl_gc_enable(int on) { @@ -2969,8 +2977,8 @@ static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp jl_binding_t *ptr = (jl_binding_t*)items[i]; // A null pointer can happen here when the binding is cleaned up // as an exception is thrown after it was already queued (#10221) - if (!ptr->value) continue; - if (gc_mark_queue_obj(gc_cache, sp, ptr->value)) { + jl_value_t *v = jl_atomic_load_relaxed(&ptr->value); + if (v != NULL && gc_mark_queue_obj(gc_cache, sp, v)) { items[n_bnd_refyoung] = ptr; n_bnd_refyoung++; } @@ -3204,15 +3212,16 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - if (jl_gc_disable_counter) { - size_t localbytes = ptls->gc_num.allocd + gc_num.interval; - ptls->gc_num.allocd = -(int64_t)gc_num.interval; - jl_atomic_add_fetch(&gc_num.deferred_alloc, localbytes); + if (jl_atomic_load_relaxed(&jl_gc_disable_counter)) { + size_t localbytes = jl_atomic_load_relaxed(&ptls->gc_num.allocd) + gc_num.interval; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval); + static_assert(sizeof(_Atomic(uint64_t)) == sizeof(gc_num.deferred_alloc), ""); + jl_atomic_fetch_add((_Atomic(uint64_t)*)&gc_num.deferred_alloc, localbytes); return; } gc_debug_print(); - int8_t old_state = ptls->gc_state; + int8_t old_state = jl_atomic_load_relaxed(&ptls->gc_state); jl_atomic_store_release(&ptls->gc_state, JL_GC_STATE_WAITING); // `jl_safepoint_start_gc()` makes sure only one thread can // run the GC. @@ -3234,7 +3243,7 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) gc_invoke_callbacks(jl_gc_cb_pre_gc_t, gc_cblist_pre_gc, (collection)); - if (!jl_gc_disable_counter) { + if (!jl_atomic_load_relaxed(&jl_gc_disable_counter)) { JL_LOCK_NOGC(&finalizers_lock); if (_jl_gc_collect(ptls, collection)) { // recollect @@ -3319,9 +3328,9 @@ void jl_init_thread_heap(jl_ptls_t ptls) gc_cache->pc_stack_end = gc_cache->pc_stack + init_size; gc_cache->data_stack = (jl_gc_mark_data_t *)malloc_s(init_size * sizeof(jl_gc_mark_data_t)); - memset(&ptls->gc_num, 0, sizeof(jl_thread_gc_num_t)); + memset(&ptls->gc_num, 0, sizeof(ptls->gc_num)); assert(gc_num.interval == default_collect_interval); - ptls->gc_num.allocd = -(int64_t)gc_num.interval; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval); } // System-wide initializations @@ -3366,8 +3375,10 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) if (pgcstack && ct->world_age) { jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); - ptls->gc_num.allocd += sz; - ptls->gc_num.malloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); + jl_atomic_store_relaxed(&ptls->gc_num.malloc, + jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); } return malloc(sz); } @@ -3379,8 +3390,10 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) if (pgcstack && ct->world_age) { jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); - ptls->gc_num.allocd += nm*sz; - ptls->gc_num.malloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz); + jl_atomic_store_relaxed(&ptls->gc_num.malloc, + jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); } return calloc(nm, sz); } @@ -3392,8 +3405,10 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz) free(p); if (pgcstack && ct->world_age) { jl_ptls_t ptls = ct->ptls; - ptls->gc_num.freed += sz; - ptls->gc_num.freecall++; + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz); + jl_atomic_store_relaxed(&ptls->gc_num.freecall, + jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1); } } @@ -3405,10 +3420,13 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); if (sz < old) - ptls->gc_num.freed += (old - sz); + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz)); else - ptls->gc_num.allocd += (sz - old); - ptls->gc_num.realloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old)); + jl_atomic_store_relaxed(&ptls->gc_num.realloc, + jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); } return realloc(p, sz); } @@ -3472,8 +3490,10 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) size_t allocsz = LLT_ALIGN(sz, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); - ptls->gc_num.allocd += allocsz; - ptls->gc_num.malloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.malloc, + jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -3503,10 +3523,13 @@ static void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t olds live_bytes += allocsz - oldsz; } else if (allocsz < oldsz) - ptls->gc_num.freed += (oldsz - allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + (oldsz - allocsz)); else - ptls->gc_num.allocd += (allocsz - oldsz); - ptls->gc_num.realloc++; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (allocsz - oldsz)); + jl_atomic_store_relaxed(&ptls->gc_num.realloc, + jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); int last_errno = errno; #ifdef _OS_WINDOWS_ @@ -3576,7 +3599,7 @@ jl_value_t *jl_gc_realloc_string(jl_value_t *s, size_t sz) #define GC_PERM_POOL_SIZE (2 * 1024 * 1024) // 20k limit for pool allocation. At most 1% fragmentation #define GC_PERM_POOL_LIMIT (20 * 1024) -jl_mutex_t gc_perm_lock = {0, 0}; +jl_mutex_t gc_perm_lock; static uintptr_t gc_perm_pool = 0; static uintptr_t gc_perm_end = 0; @@ -3721,8 +3744,8 @@ JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void) } return result; } else { - int result = support_conservative_marking; - support_conservative_marking = 1; + int result = jl_atomic_load(&support_conservative_marking); + jl_atomic_store(&support_conservative_marking, 1); return result; } } diff --git a/src/gc.h b/src/gc.h index 06faa64a8b07f..19fe3401665d1 100644 --- a/src/gc.h +++ b/src/gc.h @@ -371,7 +371,7 @@ typedef struct { int ub; } pagetable_t; -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ unsigned ffs_u32(uint32_t bitvec) JL_NOTSAFEPOINT; #else STATIC_INLINE unsigned ffs_u32(uint32_t bitvec) diff --git a/src/gf.c b/src/gf.c index 3f742af4ca1f6..b36bc9086fe2a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -118,7 +118,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m J } } else { - jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations); + _Atomic(jl_method_instance_t*) *data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(specializations); JL_GC_PUSH1(&specializations); // clang-sa doesn't realize this loop uses specializations for (i = cl; i > 0; i--) { jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i - 1]); @@ -140,7 +140,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m J } else { if (hv) { - jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations); + _Atomic(jl_method_instance_t*) *data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(specializations); for (i = 0; i < cl; i++) { jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); if ((jl_value_t*)mi == jl_nothing) @@ -992,7 +992,7 @@ static inline jl_typemap_entry_t *lookup_leafcache(jl_array_t *leafcache JL_PROP } static jl_method_instance_t *cache_method( - jl_methtable_t *mt, jl_typemap_t **cache, jl_value_t *parent JL_PROPAGATES_ROOT, + jl_methtable_t *mt, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, jl_tupletype_t *tt, // the original tupletype of the signature jl_method_t *definition, size_t world, size_t min_valid, size_t max_valid, @@ -1009,7 +1009,7 @@ static jl_method_instance_t *cache_method( return entry->func.linfo; } struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL, 0, ~(size_t)0}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(*cache, &search, offs, /*subtype*/1); + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(cache), &search, offs, /*subtype*/1); if (entry && entry->func.value) return entry->func.linfo; } @@ -1135,7 +1135,7 @@ static jl_method_instance_t *cache_method( // that satisfies our requirements if (cachett != tt) { struct jl_typemap_assoc search = {(jl_value_t*)cachett, world, NULL, 0, ~(size_t)0}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(*cache, &search, offs, /*subtype*/1); + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(cache), &search, offs, /*subtype*/1); if (entry && jl_egal((jl_value_t*)entry->simplesig, simplett ? (jl_value_t*)simplett : jl_nothing) && jl_egal((jl_value_t*)guardsigs, (jl_value_t*)entry->guardsigs)) { JL_GC_POP(); @@ -1179,7 +1179,7 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt JL_PROPAGATE // caller must hold the mt->writelock assert(tt->isdispatchtuple || tt->hasfreetypevars); if (tt->isdispatchtuple) { - jl_array_t *leafcache = mt->leafcache; + jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); if (entry) return entry->func.linfo; @@ -1527,7 +1527,7 @@ static int typemap_search(jl_typemap_entry_t *entry, void *closure) static jl_typemap_entry_t *do_typemap_search(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_method_t *method) JL_NOTSAFEPOINT; -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ static jl_typemap_entry_t *do_typemap_search(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_method_t *method) JL_NOTSAFEPOINT { jl_value_t *closure = (jl_value_t*)(method); if (jl_typemap_visitor(mt->defs, typemap_search, &closure)) @@ -1547,7 +1547,7 @@ static void jl_method_table_invalidate(jl_methtable_t *mt, jl_typemap_entry_t *m mt_cache_env.shadowed = NULL; mt_cache_env.invalidated = 0; jl_typemap_visitor(mt->cache, disable_mt_cache, (void*)&mt_cache_env); - jl_array_t *leafcache = mt->leafcache; + jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); size_t i, l = jl_array_len(leafcache); for (i = 1; i < l; i += 2) { jl_typemap_entry_t *oldentry = (jl_typemap_entry_t*)jl_array_ptr_ref(leafcache, i); @@ -1727,7 +1727,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method if (morespec[j] == (char)morespec_is) continue; jl_svec_t *specializations = jl_atomic_load_acquire(&m->specializations); - jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations); + _Atomic(jl_method_instance_t*) *data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(specializations); size_t i, l = jl_svec_len(specializations); enum morespec_options ambig = morespec_unknown; for (i = 0; i < l; i++) { @@ -1783,7 +1783,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method mt_cache_env.invalidated = 0; jl_typemap_visitor(mt->cache, invalidate_mt_cache, (void*)&mt_cache_env); - jl_array_t *leafcache = mt->leafcache; + jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); size_t i, l = jl_array_len(leafcache); for (i = 1; i < l; i += 2) { jl_value_t *entry = jl_array_ptr_ref(leafcache, i); @@ -2274,8 +2274,8 @@ STATIC_INLINE int sig_match_fast(jl_value_t *arg1t, jl_value_t **args, jl_value_ return 1; } -jl_typemap_entry_t *call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; -static uint8_t pick_which[N_CALL_CACHE]; +_Atomic(jl_typemap_entry_t*) call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; +static _Atomic(uint8_t) pick_which[N_CALL_CACHE]; #ifdef JL_GF_PROFILE size_t ncalls; void call_cache_stats() diff --git a/src/iddict.c b/src/iddict.c index 0d67a2b4c82c1..73c2999491c28 100644 --- a/src/iddict.c +++ b/src/iddict.c @@ -43,7 +43,7 @@ static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_ *pa = a; } size_t maxprobe = max_probe(sz); - void **tab = (void **)a->data; + _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*)a->data; hv = keyhash(key); while (1) { @@ -54,7 +54,7 @@ static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_ empty_slot = -1; do { - jl_value_t *k2 = (jl_value_t*)tab[index]; + jl_value_t *k2 = tab[index]; if (k2 == NULL) { if (empty_slot == -1) empty_slot = index; @@ -102,20 +102,20 @@ static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_ *pa = jl_idtable_rehash(*pa, newsz); a = *pa; - tab = (void **)a->data; + tab = (_Atomic(jl_value_t*)*)a->data; sz = hash_size(a); maxprobe = max_probe(sz); } } /* returns bp if key is in hash, otherwise NULL */ -inline jl_value_t **jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT +inline _Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT { size_t sz = hash_size(a); if (sz == 0) return NULL; size_t maxprobe = max_probe(sz); - void **tab = (void **)a->data; + _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*)a->data; uint_t hv = keyhash(key); size_t index = h2index(hv, sz); sz *= 2; @@ -123,12 +123,12 @@ inline jl_value_t **jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEP size_t iter = 0; do { - jl_value_t *k2 = (jl_value_t*)jl_atomic_load_relaxed(&tab[index]); // just to ensure the load doesn't get duplicated + jl_value_t *k2 = jl_atomic_load_relaxed(&tab[index]); // just to ensure the load doesn't get duplicated if (k2 == NULL) return NULL; if (jl_egal(key, k2)) { if (jl_atomic_load_relaxed(&tab[index + 1]) != NULL) - return (jl_value_t**)&tab[index + 1]; + return &tab[index + 1]; // `nothing` is our sentinel value for deletion, so need to keep searching if it's also our search key if (key != jl_nothing) return NULL; // concurrent insertion hasn't completed yet @@ -155,21 +155,21 @@ jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT { - jl_value_t **bp = jl_table_peek_bp(h, key); + _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp); } JL_DLLEXPORT jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found) { - jl_value_t **bp = jl_table_peek_bp(h, key); + _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); if (found) *found = (bp != NULL); if (bp == NULL) return deflt; - jl_value_t *val = *bp; - *(bp - 1) = jl_nothing; // clear the key - *bp = NULL; + jl_value_t *val = jl_atomic_load_relaxed(bp); + jl_atomic_store_relaxed(bp - 1, jl_nothing); // clear the key + jl_atomic_store_relaxed(bp, NULL); // and the value (briefly corrupting the table) return val; } diff --git a/src/interpreter.c b/src/interpreter.c index 2bfa6482aa9f7..9aab434f1d8a5 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -37,7 +37,7 @@ typedef struct { JL_GCC_IGNORE_STOP #endif -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ extern void JL_GC_ENABLEFRAME(interpreter_state*) JL_NOTSAFEPOINT; @@ -93,7 +93,7 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) } jl_value_t *bp_owner = (jl_value_t*)modu; jl_binding_t *b = jl_get_binding_for_method_def(modu, fname); - jl_value_t **bp = &b->value; + _Atomic(jl_value_t*) *bp = &b->value; jl_value_t *gf = jl_generic_function_def(b->name, b->owner, bp, bp_owner, b); return gf; } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 1847fc5c60e37..e566c6b6bd731 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -305,8 +305,7 @@ static Value *emit_unboxed_coercion(jl_codectx_t &ctx, Type *to, Value *unboxed) } else if (!ty->isIntOrPtrTy() && !ty->isFloatingPointTy()) { const DataLayout &DL = jl_data_layout; - unsigned nb = DL.getTypeSizeInBits(ty); - assert(nb == DL.getTypeSizeInBits(to)); + assert(DL.getTypeSizeInBits(ty) == DL.getTypeSizeInBits(to)); AllocaInst *cast = ctx.builder.CreateAlloca(ty); ctx.builder.CreateStore(unboxed, cast); unboxed = ctx.builder.CreateLoad(to, ctx.builder.CreateBitCast(cast, to->getPointerTo())); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index a87e14f7a76a4..792250490544a 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -175,14 +175,14 @@ static jl_callptr_t _jl_compile_codeinst( // once set, don't change invoke-ptr, as that leads to race conditions // with the (not) simultaneous updates to invoke and specptr if (!decls.specFunctionObject.empty()) { - this_code->specptr.fptr = (void*)getAddressForFunction(decls.specFunctionObject); + jl_atomic_store_release(&this_code->specptr.fptr, (void*)getAddressForFunction(decls.specFunctionObject)); this_code->isspecsig = isspecsig; } jl_atomic_store_release(&this_code->invoke, addr); } else if (this_code->invoke == jl_fptr_const_return && !decls.specFunctionObject.empty()) { - // hack to export this pointer value to jl_dump_method_asm - this_code->specptr.fptr = (void*)getAddressForFunction(decls.specFunctionObject); + // hack to export this pointer value to jl_dump_method_disasm + jl_atomic_store_release(&this_code->specptr.fptr, (void*)getAddressForFunction(decls.specFunctionObject)); } if (this_code== codeinst) fptr = addr; @@ -412,10 +412,10 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, // printing via disassembly jl_code_instance_t *codeinst = jl_generate_fptr(mi, world); if (codeinst) { - uintptr_t fptr = (uintptr_t)codeinst->invoke; + uintptr_t fptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->invoke); if (getwrapper) return jl_dump_fptr_asm(fptr, raw_mc, asm_variant, debuginfo, binary); - uintptr_t specfptr = (uintptr_t)codeinst->specptr.fptr; + uintptr_t specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr); if (fptr == (uintptr_t)&jl_fptr_const_return && specfptr == 0) { // normally we prevent native code from being generated for these functions, // (using sentinel value `1` instead) @@ -425,7 +425,7 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled); if (measure_compile_time_enabled) compiler_start_time = jl_hrtime(); - specfptr = (uintptr_t)codeinst->specptr.fptr; + specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr); if (specfptr == 0) { jl_code_info_t *src = jl_type_infer(mi, world, 0); JL_GC_PUSH1(&src); @@ -438,12 +438,12 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, if (src && (jl_value_t*)src != jl_nothing) src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src); } - fptr = (uintptr_t)codeinst->invoke; - specfptr = (uintptr_t)codeinst->specptr.fptr; + fptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->invoke); + specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr); if (src && jl_is_code_info(src)) { if (fptr == (uintptr_t)&jl_fptr_const_return && specfptr == 0) { fptr = (uintptr_t)_jl_compile_codeinst(codeinst, src, world); - specfptr = (uintptr_t)codeinst->specptr.fptr; + specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr); } } JL_GC_POP(); @@ -830,13 +830,14 @@ StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *cod std::string string_fname; raw_string_ostream stream_fname(string_fname); // try to pick an appropriate name that describes it - if (Addr == (uintptr_t)codeinst->invoke) { + jl_callptr_t invoke = jl_atomic_load_relaxed(&codeinst->invoke); + if (Addr == (uintptr_t)invoke) { stream_fname << "jsysw_"; } - else if (codeinst->invoke == &jl_fptr_args) { + else if (invoke == &jl_fptr_args) { stream_fname << "jsys1_"; } - else if (codeinst->invoke == &jl_fptr_sparam) { + else if (invoke == &jl_fptr_sparam) { stream_fname << "jsys3_"; } else { diff --git a/src/jl_uv.c b/src/jl_uv.c index 18733926e8c54..5795cd8f7e3ba 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -57,7 +57,7 @@ void jl_init_uv(void) JL_MUTEX_INIT(&jl_uv_mutex); // a file-scope initializer can be used instead } -int jl_uv_n_waiters = 0; +_Atomic(int) jl_uv_n_waiters = 0; void JL_UV_LOCK(void) { diff --git a/src/jlapi.c b/src/jlapi.c index 2192d4cee665f..fa480414a83f1 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -433,7 +433,7 @@ JL_DLLEXPORT jl_value_t *(jl_get_fieldtypes)(jl_value_t *v) } -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ JL_DLLEXPORT int8_t (jl_gc_unsafe_enter)(void) { jl_task_t *ct = jl_current_task; diff --git a/src/jltypes.c b/src/jltypes.c index 1752cfc8c2196..16db7bcc0ce11 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -19,7 +19,7 @@ extern "C" { #endif -jl_value_t *cmpswap_names JL_GLOBALLY_ROOTED; +_Atomic(jl_value_t*) cmpswap_names JL_GLOBALLY_ROOTED; // compute empirical max-probe for a given size #define max_probe(size) ((size) <= 1024 ? 16 : (size) >> 6) @@ -625,7 +625,7 @@ static jl_datatype_t *lookup_type_set(jl_svec_t *cache, jl_value_t **key, size_t if (sz == 0) return NULL; size_t maxprobe = max_probe(sz); - jl_datatype_t **tab = (jl_datatype_t**)jl_svec_data(cache); + _Atomic(jl_datatype_t*) *tab = (_Atomic(jl_datatype_t*)*)jl_svec_data(cache); size_t index = h2index(hv, sz); size_t orig = index; size_t iter = 0; @@ -648,7 +648,7 @@ static jl_datatype_t *lookup_type_setvalue(jl_svec_t *cache, jl_value_t *key1, j if (sz == 0) return NULL; size_t maxprobe = max_probe(sz); - jl_datatype_t **tab = (jl_datatype_t**)jl_svec_data(cache); + _Atomic(jl_datatype_t*) *tab = (_Atomic(jl_datatype_t*)*)jl_svec_data(cache); size_t index = h2index(hv, sz); size_t orig = index; size_t iter = 0; @@ -671,7 +671,7 @@ static ssize_t lookup_type_idx_linear(jl_svec_t *cache, jl_value_t **key, size_t { if (n == 0) return -1; - jl_datatype_t **data = (jl_datatype_t**)jl_svec_data(cache); + _Atomic(jl_datatype_t*) *data = (_Atomic(jl_datatype_t*)*)jl_svec_data(cache); size_t cl = jl_svec_len(cache); ssize_t i; for (i = 0; i < cl; i++) { @@ -688,7 +688,7 @@ static ssize_t lookup_type_idx_linearvalue(jl_svec_t *cache, jl_value_t *key1, j { if (n == 0) return -1; - jl_datatype_t **data = (jl_datatype_t**)jl_svec_data(cache); + _Atomic(jl_datatype_t*) *data = (_Atomic(jl_datatype_t*)*)jl_svec_data(cache); size_t cl = jl_svec_len(cache); ssize_t i; for (i = 0; i < cl; i++) { @@ -734,7 +734,7 @@ static jl_value_t *lookup_typevalue(jl_typename_t *tn, jl_value_t *key1, jl_valu static int cache_insert_type_set_(jl_svec_t *a, jl_datatype_t *val, uint_t hv) { - jl_datatype_t **tab = (jl_datatype_t**)jl_svec_data(a); + _Atomic(jl_datatype_t*) *tab = (_Atomic(jl_datatype_t*)*)jl_svec_data(a); size_t sz = jl_svec_len(a); if (sz <= 1) return 0; diff --git a/src/julia.h b/src/julia.h index d05dda984da8f..55f14e86f2b2d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -97,7 +97,7 @@ typedef struct _jl_taggedvalue_t jl_taggedvalue_t; -#include "atomics.h" +#include "julia_atomics.h" #include "julia_threads.h" #include "julia_assert.h" @@ -127,7 +127,7 @@ JL_EXTENSION struct _jl_taggedvalue_t { // jl_value_t value; }; -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ JL_DLLEXPORT jl_taggedvalue_t *_jl_astaggedvalue(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; #define jl_astaggedvalue(v) _jl_astaggedvalue((jl_value_t*)(v)) jl_value_t *_jl_valueof(jl_taggedvalue_t *tv JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; @@ -146,7 +146,7 @@ static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT { // Do not call this on a value that is already initialized. jl_taggedvalue_t *tag = jl_astaggedvalue(v); - jl_atomic_store_relaxed(&tag->type, (jl_value_t*)t); + jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)&tag->type, (jl_value_t*)t); } #define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) @@ -154,8 +154,8 @@ static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT // The string data is nul-terminated and hangs off the end of the struct. typedef struct _jl_sym_t { JL_DATA_TYPE - struct _jl_sym_t *left; - struct _jl_sym_t *right; + _Atomic(struct _jl_sym_t*) left; + _Atomic(struct _jl_sym_t*) right; uintptr_t hash; // precomputed hash value // JL_ATTRIBUTE_ALIGN_PTRSIZE(char name[]); } jl_sym_t; @@ -248,14 +248,6 @@ typedef jl_value_t *(*jl_fptr_sparam_t)(jl_value_t*, jl_value_t**, uint32_t, jl_ JL_DLLEXPORT extern jl_call_t jl_fptr_interpret_call; -JL_EXTENSION typedef union { - void* fptr; - jl_fptr_args_t fptr1; - // 2 constant - jl_fptr_sparam_t fptr3; - // 4 interpreter -} jl_generic_specptr_t; - typedef struct _jl_method_instance_t jl_method_instance_t; typedef struct _jl_line_info_node_t { @@ -314,13 +306,13 @@ typedef struct _jl_method_t { jl_value_t *sig; // table of all jl_method_instance_t specializations we have - jl_svec_t *specializations; // allocated as [hashable, ..., NULL, linear, ....] - jl_array_t *speckeyset; // index lookup by hash into specializations + _Atomic(jl_svec_t*) specializations; // allocated as [hashable, ..., NULL, linear, ....] + _Atomic(jl_array_t*) speckeyset; // index lookup by hash into specializations jl_value_t *slot_syms; // compacted list of slot names (String) jl_value_t *external_mt; // reference to the method table this method is part of, null if part of the internal table jl_value_t *source; // original code template (jl_code_info_t, but may be compressed), null for builtins - struct _jl_method_instance_t *unspecialized; // unspecialized executable method instance, or null + _Atomic(struct _jl_method_instance_t*) unspecialized; // unspecialized executable method instance, or null jl_value_t *generator; // executable code-generating function if available jl_array_t *roots; // pointers in generated code (shared to reduce memory), or null jl_svec_t *ccallable; // svec(rettype, sig) if a ccallable entry point is requested for this @@ -328,7 +320,7 @@ typedef struct _jl_method_t { // cache of specializations of this method for invoke(), i.e. // cases where this method was called even though it was not necessarily // the most specific for the argument types. - jl_typemap_t *invokes; + _Atomic(jl_typemap_t*) invokes; // A function that compares two specializations of this method, returning // `true` if the first signature is to be considered "smaller" than the @@ -366,7 +358,7 @@ struct _jl_method_instance_t { jl_value_t *uninferred; // cached uncompressed code, for generated functions, top-level thunks, or the interpreter jl_array_t *backedges; // list of method-instances which contain a call into this method-instance jl_array_t *callbacks; // list of callback functions to inform external caches about invalidations - struct _jl_code_instance_t *cache; + _Atomic(struct _jl_code_instance_t*) cache; uint8_t inInference; // flags to tell if inference is running on this object }; @@ -385,7 +377,7 @@ typedef struct jl_opaque_closure_t { typedef struct _jl_code_instance_t { JL_DATA_TYPE jl_method_instance_t *def; // method this is specialized from - struct _jl_code_instance_t *next; // pointer to the next cache entry + _Atomic(struct _jl_code_instance_t*) next; // pointer to the next cache entry // world range for which this object is valid to use size_t min_world; @@ -400,9 +392,15 @@ typedef struct _jl_code_instance_t { // compilation state cache uint8_t isspecsig; // if specptr is a specialized function signature for specTypes->rettype - uint8_t precompile; // if set, this will be added to the output system image - jl_callptr_t invoke; // jlcall entry point - jl_generic_specptr_t specptr; // private data for `jlcall entry point` + _Atomic(uint8_t) precompile; // if set, this will be added to the output system image + _Atomic(jl_callptr_t) invoke; // jlcall entry point + union _jl_generic_specptr_t { + _Atomic(void*) fptr; + _Atomic(jl_fptr_args_t) fptr1; + // 2 constant + _Atomic(jl_fptr_sparam_t) fptr3; + // 4 interpreter + } specptr; // private data for `jlcall entry point } jl_code_instance_t; // all values are callable as Functions @@ -436,8 +434,8 @@ typedef struct { // `wrapper` is either the only instantiation of the type (if no parameters) // or a UnionAll accepting parameters to make an instantiation. jl_value_t *wrapper; - jl_svec_t *cache; // sorted array - jl_svec_t *linearcache; // unsorted array + _Atomic(jl_svec_t*) cache; // sorted array + _Atomic(jl_svec_t*) linearcache; // unsorted array struct _jl_methtable_t *mt; jl_array_t *partial; // incomplete instantiations of this type intptr_t hash; @@ -526,10 +524,10 @@ typedef struct { typedef struct { // not first-class jl_sym_t *name; - jl_value_t *value; - jl_value_t *globalref; // cached GlobalRef for this binding - struct _jl_module_t *owner; // for individual imported bindings - uint8_t constp; + _Atomic(jl_value_t*) value; + _Atomic(jl_value_t*) globalref; // cached GlobalRef for this binding + struct _jl_module_t* owner; // for individual imported bindings -- TODO: make _Atomic + uint8_t constp:1; uint8_t exportp:1; uint8_t imported:1; uint8_t deprecated:2; // 0=not deprecated, 1=renamed, 2=moved to another package @@ -550,7 +548,7 @@ typedef struct _jl_module_t { uint64_t build_id; jl_uuid_t uuid; size_t primary_world; - uint32_t counter; + _Atomic(uint32_t) counter; int32_t nospecialize; // global bit flags: initialization for new methods int8_t optlevel; int8_t compile; @@ -562,7 +560,7 @@ typedef struct _jl_module_t { // one Type-to-Value entry typedef struct _jl_typemap_entry_t { JL_DATA_TYPE - struct _jl_typemap_entry_t *next; // invasive linked list + _Atomic(struct _jl_typemap_entry_t*) next; // invasive linked list jl_tupletype_t *sig; // the type signature for this entry jl_tupletype_t *simplesig; // a simple signature for fast rejection jl_svec_t *guardsigs; @@ -587,23 +585,23 @@ typedef struct _jl_typemap_level_t { // next split may be on Type{T} as LeafTypes then TypeName's parents up to Any // next split may be on LeafType // next split may be on TypeName - jl_array_t *arg1; // contains LeafType - jl_array_t *targ; // contains Type{LeafType} - jl_array_t *name1; // contains non-abstract TypeName, for parents up to (excluding) Any - jl_array_t *tname; // contains a dict of Type{TypeName}, for parents up to Any + _Atomic(jl_array_t*) arg1; // contains LeafType + _Atomic(jl_array_t*) targ; // contains Type{LeafType} + _Atomic(jl_array_t*) name1; // contains non-abstract TypeName, for parents up to (excluding) Any + _Atomic(jl_array_t*) tname; // contains a dict of Type{TypeName}, for parents up to Any // next a linear list of things too complicated at this level for analysis (no more levels) - jl_typemap_entry_t *linear; + _Atomic(jl_typemap_entry_t*) linear; // finally, start a new level if the type at offs is Any - jl_typemap_t *any; + _Atomic(jl_typemap_t*) any; } jl_typemap_level_t; // contains the TypeMap for one Type typedef struct _jl_methtable_t { JL_DATA_TYPE jl_sym_t *name; // sometimes a hack used by serialization to handle kwsorter - jl_typemap_t *defs; - jl_array_t *leafcache; - jl_typemap_t *cache; + _Atomic(jl_typemap_t*) defs; + _Atomic(jl_array_t*) leafcache; + _Atomic(jl_typemap_t*) cache; intptr_t max_args; // max # of non-vararg arguments in a signature jl_value_t *kwsorter; // keyword argument sorter function jl_module_t *module; // used for incremental serialization to locate original binding @@ -777,7 +775,7 @@ struct _jl_gcframe_t { #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<2) #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<2)|1) -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ // When running with the analyzer make these real function calls, that are // easier to detect in the analyzer @@ -908,7 +906,7 @@ JL_DLLEXPORT void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, #define jl_svec_set_len_unsafe(t,n) (((jl_svec_t*)(t))->length=(n)) #define jl_svec_data(t) ((jl_value_t**)((char*)(t) + sizeof(jl_svec_t))) -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ STATIC_INLINE jl_value_t *jl_svecref(void *t JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; STATIC_INLINE jl_value_t *jl_svecset( void *t JL_ROOTING_ARGUMENT JL_PROPAGATES_ROOT, @@ -920,7 +918,7 @@ STATIC_INLINE jl_value_t *jl_svecref(void *t JL_PROPAGATES_ROOT, size_t i) JL_NO assert(i < jl_svec_len(t)); // while svec is supposedly immutable, in practice we sometimes publish it first // and set the values lazily - return jl_atomic_load_relaxed(jl_svec_data(t) + i); + return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)jl_svec_data(t) + i); } STATIC_INLINE jl_value_t *jl_svecset( void *t JL_ROOTING_ARGUMENT JL_PROPAGATES_ROOT, @@ -952,7 +950,7 @@ JL_DLLEXPORT size_t jl_array_len_(jl_array_t *a); JL_DLLEXPORT char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT; -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ jl_value_t **jl_array_ptr_data(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; STATIC_INLINE jl_value_t *jl_array_ptr_ref(void *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; STATIC_INLINE jl_value_t *jl_array_ptr_set( @@ -964,7 +962,7 @@ STATIC_INLINE jl_value_t *jl_array_ptr_ref(void *a JL_PROPAGATES_ROOT, size_t i) { assert(((jl_array_t*)a)->flags.ptrarray); assert(i < jl_array_len(a)); - return jl_atomic_load_relaxed(((jl_value_t**)(jl_array_data(a))) + i); + return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)(jl_array_data(a))) + i); } STATIC_INLINE jl_value_t *jl_array_ptr_set( void *a JL_ROOTING_ARGUMENT, size_t i, @@ -972,7 +970,7 @@ STATIC_INLINE jl_value_t *jl_array_ptr_set( { assert(((jl_array_t*)a)->flags.ptrarray); assert(i < jl_array_len(a)); - jl_atomic_store_relaxed(((jl_value_t**)(jl_array_data(a))) + i, (jl_value_t*)x); + jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)(jl_array_data(a))) + i, (jl_value_t*)x); if (x) { if (((jl_array_t*)a)->flags.how == 3) { a = jl_array_data_owner(a); @@ -1428,7 +1426,7 @@ JL_DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, size_t len); JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void); JL_DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_module_t *module, - jl_value_t **bp, jl_value_t *bp_owner, + _Atomic(jl_value_t*) *bp, jl_value_t *bp_owner, jl_binding_t *bnd); JL_DLLEXPORT jl_method_t *jl_method_def(jl_svec_t *argdata, jl_methtable_t *mt, jl_code_info_t *f, jl_module_t *module); JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo); @@ -1836,13 +1834,13 @@ typedef struct _jl_task_t { uint64_t rngState1; uint64_t rngState2; uint64_t rngState3; - uint8_t _state; + _Atomic(uint8_t) _state; uint8_t sticky; // record whether this Task can be migrated to a new thread - uint8_t _isexception; // set if `result` is an exception to throw or that we exited with + _Atomic(uint8_t) _isexception; // set if `result` is an exception to throw or that we exited with // hidden state: // id of owning thread - does not need to be defined until the task runs - int16_t tid; + _Atomic(int16_t) tid; // multiqueue priority int16_t prio; // saved gc stack top for context switches @@ -1887,7 +1885,7 @@ JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e); JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; #define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) -#include "locks.h" // requires jl_task_t definition +#include "julia_locks.h" // requires jl_task_t definition JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh); JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh); @@ -1926,7 +1924,7 @@ void (jl_longjmp)(jmp_buf _Buf, int _Value); #endif -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ // This is hard. Ideally we'd teach the static analyzer about the extra control // flow edges. But for now, just hide this as best we can diff --git a/src/julia_atomics.h b/src/julia_atomics.h new file mode 100644 index 0000000000000..0f3d31176a0c3 --- /dev/null +++ b/src/julia_atomics.h @@ -0,0 +1,311 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#ifndef JL_ATOMICS_H +#define JL_ATOMICS_H + +#if defined(__i386__) && defined(__GNUC__) && !defined(__SSE2__) +# error Julia can only be built for architectures above Pentium 4. Pass -march=pentium4, or set MARCH=pentium4 and ensure that -march is not passed separately with an older architecture. +#endif + +// Low-level atomic operations +#ifdef __cplusplus +#include +using std::memory_order_relaxed; +using std::memory_order_consume; +using std::memory_order_acquire; +using std::memory_order_release; +using std::memory_order_acq_rel; +using std::memory_order_seq_cst; +using std::atomic_thread_fence; +using std::atomic_signal_fence; +using std::atomic_load; +using std::atomic_load_explicit; +using std::atomic_store; +using std::atomic_store_explicit; +using std::atomic_fetch_add; +using std::atomic_fetch_add_explicit; +using std::atomic_fetch_and; +using std::atomic_fetch_and_explicit; +using std::atomic_fetch_or; +using std::atomic_fetch_or_explicit; +using std::atomic_compare_exchange_strong; +using std::atomic_compare_exchange_strong_explicit; +using std::atomic_exchange; +using std::atomic_exchange_explicit; +extern "C" { +#define _Atomic(T) std::atomic +#else +#include +#endif +#include // for sig_atomic_t + +#if defined(_CPU_X86_64_) || defined(_CPU_X86_) +# include +#endif + +enum jl_memory_order { + jl_memory_order_unspecified = -2, + jl_memory_order_invalid = -1, + jl_memory_order_notatomic = 0, + jl_memory_order_unordered, + jl_memory_order_monotonic, + jl_memory_order_consume, + jl_memory_order_acquire, + jl_memory_order_release, + jl_memory_order_acq_rel, + jl_memory_order_seq_cst +}; + +/** + * Thread synchronization primitives: + * + * These roughly follows the c11/c++11 memory model and the act as memory + * barriers at both the compiler level and the hardware level. + * The only exception is the GC safepoint and GC state transitions for which + * we use only a compiler (signal) barrier and use the signal handler to do the + * synchronization in order to lower the mutator overhead as much as possible. + * + * We use the compiler intrinsics to implement a similar API to the c11/c++11 + * one instead of using it directly because, we need interoperability between + * code written in different languages. The current c++ standard (c++14) does + * not allow using c11 atomic functions or types and there's currently no + * guarantee that the two types are compatible (although most of them probably + * are). We also need to access these atomic variables from the LLVM JIT code + * which is very hard unless the layout of the object is fully specified. + */ +#define jl_fence() atomic_thread_fence(memory_order_seq_cst) +#define jl_fence_release() atomic_thread_fence(memory_order_release) +#define jl_signal_fence() atomic_signal_fence(memory_order_seq_cst) + +#ifdef __cplusplus +} +// implicit conversion wasn't correctly specified 2017, so many compilers get +// this wrong thus we include the correct definitions here (with implicit +// conversion), instead of using the macro version +template +T jl_atomic_load(std::atomic *ptr) +{ + return std::atomic_load(ptr); +} +template +T jl_atomic_load_explicit(std::atomic *ptr, std::memory_order order) +{ + return std::atomic_load_explicit(ptr, order); +} +#define jl_atomic_load_relaxed(ptr) jl_atomic_load_explicit(ptr, memory_order_relaxed) +#define jl_atomic_load_acquire(ptr) jl_atomic_load_explicit(ptr, memory_order_acquire) +template +void jl_atomic_store(std::atomic *ptr, S desired) +{ + std::atomic_store(ptr, desired); +} +template +void jl_atomic_store_explicit(std::atomic *ptr, S desired, std::memory_order order) +{ + std::atomic_store_explicit(ptr, desired, order); +} +#define jl_atomic_store_relaxed(ptr, val) jl_atomic_store_explicit(ptr, val, memory_order_relaxed) +#define jl_atomic_store_release(ptr, val) jl_atomic_store_explicit(ptr, val, memory_order_release) +template +T jl_atomic_fetch_add(std::atomic *ptr, S val) +{ + return std::atomic_fetch_add(ptr, val); +} +template +T jl_atomic_fetch_add_explicit(std::atomic *ptr, S val, std::memory_order order) +{ + return std::atomic_fetch_add_explicit(ptr, val, order); +} +#define jl_atomic_fetch_add_relaxed(ptr, val) jl_atomic_fetch_add_explicit(ptr, val, memory_order_relaxed) +template +T jl_atomic_fetch_and(std::atomic *ptr, S val) +{ + return std::atomic_fetch_and(ptr, val); +} +template +T jl_atomic_fetch_and_explicit(std::atomic *ptr, S val, std::memory_order order) +{ + return std::atomic_fetch_and_explicit(ptr, val, order); +} +#define jl_atomic_fetch_and_relaxed(ptr, val) jl_atomic_fetch_and_explicit(ptr, val, memory_order_relaxed) +template +T jl_atomic_fetch_or(std::atomic *ptr, S val) +{ + return std::atomic_fetch_or(ptr, val); +} +template +T jl_atomic_fetch_or_explicit(std::atomic *ptr, S val, std::memory_order order) +{ + return std::atomic_fetch_or_explicit(ptr, val, order); +} +#define jl_atomic_fetch_or_relaxed(ptr, val) jl_atomic_fetch_or_explicit(ptr, val, memory_order_relaxed) +template +bool jl_atomic_cmpswap(std::atomic *ptr, T *expected, S val) +{ + return std::atomic_compare_exchange_strong(ptr, expected, val); +} +template +bool jl_atomic_cmpswap_explicit(std::atomic *ptr, T *expected, S val, std::memory_order order) +{ + return std::atomic_compare_exchange_strong_explicit(ptr, expected, val, order, order); +} +#define jl_atomic_cmpswap_relaxed(ptr, val) jl_atomic_cmpswap_explicit(ptr, val, memory_order_relaxed) +template +T jl_atomic_exchange(std::atomic *ptr, S desired) +{ + return std::atomic_exchange(ptr, desired); +} +template +T jl_atomic_exchange_explicit(std::atomic *ptr, S desired, std::memory_order order) +{ + return std::atomic_exchange_explicit(ptr, desired, order); +} +#define jl_atomic_exchange_relaxed(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_relaxed) +extern "C" { +#else + +# define jl_atomic_fetch_add_relaxed(obj, arg) \ + atomic_fetch_add_explicit(obj, arg, memory_order_relaxed) +# define jl_atomic_fetch_add(obj, arg) \ + atomic_fetch_add(obj, arg) +# define jl_atomic_fetch_and_relaxed(obj, arg) \ + atomic_fetch_and_explicit(obj, arg, memory_order_relaxed) +# define jl_atomic_fetch_and(obj, arg) \ + atomic_fetch_and(obj, arg) +# define jl_atomic_fetch_or_relaxed(obj, arg) \ + atomic_fetch_or_explicit(obj, arg, __ATOMIC_RELAXED) +# define jl_atomic_fetch_or(obj, arg) \ + atomic_fetch_or(obj, arg) +# define jl_atomic_cmpswap(obj, expected, desired) \ + atomic_compare_exchange_strong(obj, expected, desired) +# define jl_atomic_cmpswap_relaxed(obj, expected, desired) \ + atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_relaxed, memory_order_relaxed) +// TODO: Maybe add jl_atomic_cmpswap_weak for spin lock +# define jl_atomic_exchange(obj, desired) \ + atomic_exchange(obj, desired) +# define jl_atomic_exchange_relaxed(obj, desired) \ + atomic_exchange_explicit(obj, desired, memory_order_relaxed) +# define jl_atomic_store(obj, val) \ + atomic_store(obj, val) +# define jl_atomic_store_relaxed(obj, val) \ + atomic_store_explicit(obj, val, memory_order_relaxed) + +# if defined(__clang__) || defined(__ICC) || defined(__INTEL_COMPILER) || \ + !(defined(_CPU_X86_) || defined(_CPU_X86_64_)) +// ICC and Clang doesn't have this bug... +# define jl_atomic_store_release(obj, val) \ + atomic_store_explicit(obj, val, memory_order_release) +# else +// Workaround a GCC bug when using store with release order by using the +// stronger version instead. +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67458 +// fixed in https://gcc.gnu.org/git/?p=gcc.git&a=commit;h=d8c40eff56f69877b33c697ded756d50fde90c27 +# define jl_atomic_store_release(obj, val) do { \ + jl_signal_fence(); \ + atomic_store_explicit(obj, val, memory_order_release); \ + } while (0) +# endif +# define jl_atomic_load(obj) \ + atomic_load(obj) +# define jl_atomic_load_acquire(obj) \ + atomic_load_explicit(obj, memory_order_acquire) +#ifdef JL_TSAN_ENABLED +// For the sake of tsan, call these loads consume ordering since they will act +// as such on the processors we support while normally, the compiler would +// upgrade this to acquire ordering, which is strong (and slower) than we want. +# define jl_atomic_load_relaxed(obj) \ + atomic_load_explicit(obj, memory_order_consume) +#else +# define jl_atomic_load_relaxed(obj) \ + atomic_load_explicit(obj, memory_order_relaxed) +#endif +#endif + +#ifdef __clang_gcanalyzer__ +// for the purposes of the GC analyzer, we can turn these into non-atomic +// expressions with similar properties (for the sake of the analyzer, we don't +// care if it is an exact match for behavior) + +#undef _Atomic +#define _Atomic(T) T + +#undef jl_atomic_exchange +#undef jl_atomic_exchange_relaxed +#define jl_atomic_exchange(obj, desired) \ + (__extension__({ \ + __typeof__((obj)) p__analyzer__ = (obj); \ + __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ + *p__analyzer__ = (desired); \ + temp__analyzer__; \ + })) +#define jl_atomic_exchange_relaxed jl_atomic_exchange + +#undef jl_atomic_cmpswap +#undef jl_atomic_cmpswap_relaxed +#define jl_atomic_cmpswap(obj, expected, desired) \ + (__extension__({ \ + __typeof__((obj)) p__analyzer__ = (obj); \ + __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ + __typeof__((expected)) x__analyzer__ = (expected); \ + int eq__analyzer__ = memcmp(&temp__analyzer__, x__analyzer__, sizeof(temp__analyzer__)) == 0; \ + if (eq__analyzer__) \ + *p__analyzer__ = (desired); \ + else \ + *x__analyzer__ = temp__analyzer__; \ + eq__analyzer__; \ + })) +#define jl_atomic_cmpswap_relaxed jl_atomic_cmpswap + +#undef jl_atomic_store +#undef jl_atomic_store_release +#undef jl_atomic_store_relaxed +#define jl_atomic_store(obj, val) (*(obj) = (val)) +#define jl_atomic_store_release jl_atomic_store +#define jl_atomic_store_relaxed jl_atomic_store + +#undef jl_atomic_load +#undef jl_atomic_load_acquire +#undef jl_atomic_load_relaxed +#define jl_atomic_load(obj) (*(obj)) +#define jl_atomic_load_acquire jl_atomic_load +#define jl_atomic_load_relaxed jl_atomic_load + +#undef jl_atomic_fetch_add +#undef jl_atomic_fetch_and +#undef jl_atomic_fetch_or +#undef jl_atomic_fetch_add_relaxed +#undef jl_atomic_fetch_and_relaxed +#undef jl_atomic_fetch_or_relaxed +#define jl_atomic_fetch_add(obj, val) \ + (__extension__({ \ + __typeof__((obj)) p__analyzer__ = (obj); \ + __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ + *(p__analyzer__) = temp__analyzer__ + (val); \ + temp__analyzer__; \ + })) +#define jl_atomic_fetch_and(obj, val) \ + (__extension__({ \ + __typeof__((obj)) p__analyzer__ = (obj); \ + __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ + *(p__analyzer__) = temp__analyzer__ & (val); \ + temp__analyzer__; \ + })) +#define jl_atomic_fetch_or(obj, val) \ + (__extension__({ \ + __typeof__((obj)) p__analyzer__ = (obj); \ + __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \ + *(p__analyzer__) = temp__analyzer__ | (val); \ + temp__analyzer__; \ + })) +#define jl_atomic_fetch_add_relaxed jl_atomic_fetch_add +#define jl_atomic_fetch_and_relaxed jl_atomic_fetch_and +#define jl_atomic_fetch_or_relaxed jl_atomic_fetch_or + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif // JL_ATOMICS_H diff --git a/src/julia_internal.h b/src/julia_internal.h index 2605da0cad312..7d3549ffc88ba 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -4,7 +4,7 @@ #define JL_INTERNAL_H #include "options.h" -#include "locks.h" +#include "julia_locks.h" #include #if !defined(_WIN32) #include @@ -105,7 +105,7 @@ void __tsan_switch_to_fiber(void *fiber, unsigned flags); static uv_loop_t *const unused_uv_loop_arg = (uv_loop_t *)0xBAD10; extern jl_mutex_t jl_uv_mutex; -extern int jl_uv_n_waiters; +extern _Atomic(int) jl_uv_n_waiters; void JL_UV_LOCK(void); #define JL_UV_UNLOCK() JL_UNLOCK(&jl_uv_mutex) @@ -155,8 +155,8 @@ static inline uint64_t cycleclock(void) #include "timing.h" // Global *atomic* integers controlling *process-wide* measurement of compilation time. -extern uint8_t jl_measure_compile_time_enabled; -extern uint64_t jl_cumulative_compile_time; +extern _Atomic(uint8_t) jl_measure_compile_time_enabled; +extern _Atomic(uint64_t) jl_cumulative_compile_time; #ifdef _COMPILER_MICROSOFT_ # define jl_return_address() ((uintptr_t)_ReturnAddress()) @@ -184,14 +184,16 @@ STATIC_INLINE uint32_t jl_int32hash_fast(uint32_t a) static inline void memmove_refs(void **dstp, void *const *srcp, size_t n) JL_NOTSAFEPOINT { size_t i; + _Atomic(void*) *srcpa = (_Atomic(void*)*)srcp; + _Atomic(void*) *dstpa = (_Atomic(void*)*)dstp; if (dstp < srcp || dstp > srcp + n) { for (i = 0; i < n; i++) { - jl_atomic_store_relaxed(dstp + i, jl_atomic_load_relaxed(srcp + i)); + jl_atomic_store_relaxed(dstpa + i, jl_atomic_load_relaxed(srcpa + i)); } } else { for (i = 0; i < n; i++) { - jl_atomic_store_relaxed(dstp + n - i - 1, jl_atomic_load_relaxed(srcp + n - i - 1)); + jl_atomic_store_relaxed(dstpa + n - i - 1, jl_atomic_load_relaxed(srcpa + n - i - 1)); } } } @@ -217,7 +219,7 @@ extern jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED; extern size_t jl_page_size; extern jl_function_t *jl_typeinf_func; extern size_t jl_typeinf_world; -extern jl_typemap_entry_t *call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; +extern _Atomic(jl_typemap_entry_t*) call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; extern jl_array_t *jl_all_methods JL_GLOBALLY_ROOTED; JL_DLLEXPORT extern int jl_lineno; @@ -719,7 +721,7 @@ STATIC_INLINE int jl_addr_is_safepoint(uintptr_t addr) uintptr_t safepoint_addr = (uintptr_t)jl_safepoint_pages; return addr >= safepoint_addr && addr < safepoint_addr + jl_page_size * 3; } -extern uint32_t jl_gc_running; +extern _Atomic(uint32_t) jl_gc_running; // All the functions are safe to be called from within a signal handler // provided that the thread will not be interrupted by another asynchronous // signal. @@ -766,13 +768,13 @@ typedef jl_gcframe_t ***(*jl_pgcstack_key_t)(void) JL_NOTSAFEPOINT; #endif void jl_pgcstack_getkey(jl_get_pgcstack_func **f, jl_pgcstack_key_t *k); -#if !defined(__clang_analyzer__) +#if !defined(__clang_gcanalyzer__) static inline void jl_set_gc_and_wait(void) { jl_task_t *ct = jl_current_task; // reading own gc state doesn't need atomic ops since no one else // should store to it. - int8_t state = ct->ptls->gc_state; + int8_t state = jl_atomic_load_relaxed(&ct->ptls->gc_state); jl_atomic_store_release(&ct->ptls->gc_state, JL_GC_STATE_WAITING); jl_safepoint_wait_gc(); jl_atomic_store_release(&ct->ptls->gc_state, state); @@ -798,7 +800,7 @@ void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); -jl_value_t **jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT; +_Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); @@ -1081,7 +1083,7 @@ extern void *jl_winsock_handle; void *jl_get_library_(const char *f_lib, int throw_err); #define jl_get_library(f_lib) jl_get_library_(f_lib, 1) -JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, void **hnd); +JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *hnd); JL_DLLEXPORT void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name); JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill, @@ -1229,11 +1231,11 @@ void jl_mach_gc_end(void); typedef uint_t (*smallintset_hash)(size_t val, jl_svec_t *data); typedef int (*smallintset_eq)(size_t val, const void *key, jl_svec_t *data, uint_t hv); ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv); -void jl_smallintset_insert(jl_array_t **pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data); +void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data); // -- typemap.c -- // -void jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent, +void jl_typemap_insert(_Atomic(jl_typemap_t*) *cache, jl_value_t *parent, jl_typemap_entry_t *newrec, int8_t offs); jl_typemap_entry_t *jl_typemap_alloc( jl_tupletype_t *type, jl_tupletype_t *simpletype, jl_svec_t *guardsigs, @@ -1415,7 +1417,7 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_GCC_IGNORE_STOP #endif // _COMPILER_GCC_ -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ // Not a safepoint (so it dosn't free other values), but an artificial use. // Usually this is unnecessary because the analyzer can see all real uses, // but sometimes real uses are harder for the analyzer to see, or it may diff --git a/src/locks.h b/src/julia_locks.h similarity index 98% rename from src/locks.h rename to src/julia_locks.h index 8aeab33cbc330..8da0fc8ac9537 100644 --- a/src/locks.h +++ b/src/julia_locks.h @@ -40,7 +40,7 @@ static inline void jl_mutex_wait(jl_mutex_t *lock, int safepoint) static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT { -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ // Hide this body from the analyzer, otherwise it complains that we're calling // a non-safepoint from this function. The 0 arguments guarantees that we do // not reach the safepoint, but the analyzer can't figure that out @@ -112,7 +112,7 @@ static inline int jl_mutex_trylock(jl_mutex_t *lock) } static inline void jl_mutex_unlock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT { -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ assert(jl_atomic_load_relaxed(&lock->owner) == jl_current_task && "Unlocking a lock in a different thread."); if (--lock->count == 0) { diff --git a/src/julia_threads.h b/src/julia_threads.h index 27ed5274b75df..57270832123c3 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -4,7 +4,7 @@ #ifndef JL_THREADS_H #define JL_THREADS_H -#include +#include "julia_atomics.h" // threading ------------------------------------------------------------------ #ifdef __cplusplus @@ -97,7 +97,7 @@ typedef pthread_t jl_thread_t; // Recursive spin lock typedef struct { - struct _jl_task_t *owner; + _Atomic(struct _jl_task_t*) owner; uint32_t count; } jl_mutex_t; @@ -108,13 +108,13 @@ typedef struct { } jl_gc_pool_t; typedef struct { - int64_t allocd; - int64_t freed; - uint64_t malloc; - uint64_t realloc; - uint64_t poolalloc; - uint64_t bigalloc; - uint64_t freecall; + _Atomic(int64_t) allocd; + _Atomic(int64_t) freed; + _Atomic(uint64_t) malloc; + _Atomic(uint64_t) realloc; + _Atomic(uint64_t) poolalloc; + _Atomic(uint64_t) bigalloc; + _Atomic(uint64_t) freecall; } jl_thread_gc_num_t; typedef struct { @@ -194,7 +194,7 @@ typedef struct _jl_tls_states_t { int16_t tid; uint64_t rngseed; volatile size_t *safepoint; - int8_t sleep_check_state; // read/write from foreign threads + _Atomic(int8_t) sleep_check_state; // read/write from foreign threads // Whether it is safe to execute GC at the same time. #define JL_GC_STATE_WAITING 1 // gc_state = 1 means the thread is doing GC or is waiting for the GC to @@ -202,7 +202,7 @@ typedef struct _jl_tls_states_t { #define JL_GC_STATE_SAFE 2 // gc_state = 2 means the thread is running unmanaged code that can be // execute at the same time with the GC. - int8_t gc_state; // read from foreign threads + _Atomic(int8_t) gc_state; // read from foreign threads // execution of certain certain impure // statements is prohibited from certain // callbacks (such as generated functions) @@ -217,7 +217,7 @@ typedef struct _jl_tls_states_t { uv_mutex_t sleep_lock; uv_cond_t wake_signal; volatile sig_atomic_t defer_signal; - struct _jl_task_t *current_task; + _Atomic(struct _jl_task_t*) current_task; struct _jl_task_t *next_task; struct _jl_task_t *previous_task; struct _jl_task_t *root_task; @@ -239,7 +239,7 @@ typedef struct _jl_tls_states_t { struct _jl_bt_element_t *bt_data; // JL_MAX_BT_SIZE + 1 elements long size_t bt_size; // Size for backtrace in transit in bt_data // Atomically set by the sender, reset by the handler. - volatile sig_atomic_t signal_request; + volatile _Atomic(sig_atomic_t) signal_request; // TODO: no actual reason for this to be _Atomic // Allow the sigint to be raised asynchronously // this is limited to the few places we do synchronous IO // we can make this more general (similar to defer_signal) if necessary @@ -293,7 +293,7 @@ typedef jl_tls_states_t *jl_ptls_t; JL_DLLEXPORT void (jl_cpu_pause)(void); JL_DLLEXPORT void (jl_cpu_wake)(void); -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ // Note that the sigint safepoint can also trigger GC, albeit less likely void jl_gc_safepoint_(jl_ptls_t tls); void jl_sigint_safepoint(jl_ptls_t tls); @@ -328,9 +328,9 @@ STATIC_INLINE int8_t jl_gc_state_set(jl_ptls_t ptls, int8_t state, STATIC_INLINE int8_t jl_gc_state_save_and_set(jl_ptls_t ptls, int8_t state) { - return jl_gc_state_set(ptls, state, ptls->gc_state); + return jl_gc_state_set(ptls, state, jl_atomic_load_relaxed(&ptls->gc_state)); } -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ int8_t jl_gc_unsafe_enter(jl_ptls_t ptls); // Can be a safepoint int8_t jl_gc_unsafe_leave(jl_ptls_t ptls, int8_t state) JL_NOTSAFEPOINT; int8_t jl_gc_safe_enter(jl_ptls_t ptls) JL_NOTSAFEPOINT; @@ -347,7 +347,7 @@ JL_DLLEXPORT void jl_gc_enable_finalizers(struct _jl_task_t *ct, int on); JL_DLLEXPORT void jl_gc_disable_finalizers_internal(void); JL_DLLEXPORT void jl_gc_enable_finalizers_internal(void); JL_DLLEXPORT void jl_gc_run_pending_finalizers(struct _jl_task_t *ct); -extern JL_DLLEXPORT int jl_gc_have_pending_finalizers; +extern JL_DLLEXPORT _Atomic(int) jl_gc_have_pending_finalizers; JL_DLLEXPORT void jl_wakeup_thread(int16_t tid); diff --git a/src/llvm-version.h b/src/llvm-version.h index f59f7826c334d..fd7b47fc9461e 100644 --- a/src/llvm-version.h +++ b/src/llvm-version.h @@ -21,7 +21,9 @@ #define LLVM_ENABLE_STATS 0 #endif +#ifdef __cplusplus #if defined(__GNUC__) && (__GNUC__ >= 9) // Added in GCC 9, this warning is annoying #pragma GCC diagnostic ignored "-Winit-list-lifetime" #endif +#endif diff --git a/src/method.c b/src/method.c index 22145a4349853..80588897fe6e9 100644 --- a/src/method.c +++ b/src/method.c @@ -658,8 +658,8 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) jl_task_t *ct = jl_current_task; jl_method_t *m = (jl_method_t*)jl_gc_alloc(ct->ptls, sizeof(jl_method_t), jl_method_type); - m->specializations = jl_emptysvec; - m->speckeyset = (jl_array_t*)jl_an_empty_vec_any; + jl_atomic_store_relaxed(&m->specializations, jl_emptysvec); + jl_atomic_store_relaxed(&m->speckeyset, (jl_array_t*)jl_an_empty_vec_any); m->sig = NULL; m->slot_syms = NULL; m->roots = NULL; @@ -675,7 +675,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->called = 0xff; m->nospecialize = module->nospecialize; m->nkw = 0; - m->invokes = NULL; + jl_atomic_store_relaxed(&m->invokes, NULL); m->recursion_relation = NULL; m->isva = 0; m->nargs = 0; @@ -718,7 +718,8 @@ jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name // empty generic function def JL_DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_module_t *module, - jl_value_t **bp, jl_value_t *bp_owner, + _Atomic(jl_value_t*) *bp, + jl_value_t *bp_owner, jl_binding_t *bnd) { jl_value_t *gf = NULL; @@ -726,16 +727,16 @@ JL_DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, assert(name && bp); if (bnd && bnd->value != NULL && !bnd->constp) jl_errorf("cannot define function %s; it already has a value", jl_symbol_name(bnd->name)); - if (*bp != NULL) { - gf = *bp; + gf = jl_atomic_load_relaxed(bp); + if (gf != NULL) { if (!jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(gf)) && !jl_is_type(gf)) jl_errorf("cannot define function %s; it already has a value", jl_symbol_name(name)); } if (bnd) bnd->constp = 1; - if (*bp == NULL) { + if (gf == NULL) { gf = (jl_value_t*)jl_new_generic_function(name, module); - *bp = gf; + jl_atomic_store(bp, gf); // TODO: fix constp assignment data race if (bp_owner) jl_gc_wb(bp_owner, gf); } return gf; diff --git a/src/module.c b/src/module.c index eee3bc8137ed0..25bb98bd8f521 100644 --- a/src/module.c +++ b/src/module.c @@ -181,7 +181,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, // Hash tables don't generically root their contents, but they do for bindings. // Express this to the analyzer. // NOTE: Must hold m->lock while calling these. -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ jl_binding_t *_jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var) JL_NOTSAFEPOINT; jl_binding_t **_jl_get_module_binding_bp(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var) JL_NOTSAFEPOINT; #else @@ -257,7 +257,7 @@ static jl_binding_t *jl_get_binding_(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t static inline jl_module_t *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ // The analyzer doesn't like looking through the arraylist, so just model the // access for it using this function static inline jl_module_t *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT { @@ -375,12 +375,17 @@ JL_DLLEXPORT jl_value_t *jl_module_globalref(jl_module_t *m, jl_sym_t *var) JL_UNLOCK(&m->lock); return jl_new_struct(jl_globalref_type, m, var); } - if (b->globalref == NULL) { - b->globalref = jl_new_struct(jl_globalref_type, m, var); - jl_gc_wb(m, b->globalref); + jl_value_t *globalref = jl_atomic_load_relaxed(&b->globalref); + if (globalref == NULL) { + jl_value_t *newref = jl_new_struct(jl_globalref_type, m, var); + if (jl_atomic_cmpswap_relaxed(&b->globalref, &globalref, newref)) { + JL_GC_PROMISE_ROOTED(newref); + globalref = newref; + jl_gc_wb(m, globalref); + } } - JL_UNLOCK(&m->lock); - return b->globalref; + JL_UNLOCK(&m->lock); // may GC + return globalref; } static int eq_bindings(jl_binding_t *a, jl_binding_t *b) @@ -635,7 +640,8 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var jl_binding_t *bp = jl_get_binding_wr(m, var, 1); if (bp->value == NULL) { uint8_t constp = 0; - if (jl_atomic_cmpswap(&bp->constp, &constp, 1)) { + // if (jl_atomic_cmpswap(&bp->constp, &constp, 1)) { + if (constp = bp->constp, bp->constp = 1, constp == 0) { jl_value_t *old = NULL; if (jl_atomic_cmpswap(&bp->value, &old, val)) { jl_gc_wb_binding(bp, val); @@ -769,7 +775,7 @@ JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_value_t *rhs) JL_NOT if (jl_egal(rhs, old)) return; if (jl_typeof(rhs) != jl_typeof(old) || jl_is_type(rhs) || jl_is_module(rhs)) { -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ jl_errorf("invalid redefinition of constant %s", jl_symbol_name(b->name)); #endif diff --git a/src/partr.c b/src/partr.c index cc94167e0b62a..a3f2dd43165f7 100644 --- a/src/partr.c +++ b/src/partr.c @@ -59,8 +59,8 @@ extern int jl_gc_mark_queue_obj_explicit(jl_gc_mark_cache_t *gc_cache, typedef struct taskheap_tag { uv_mutex_t lock; jl_task_t **tasks; - int32_t ntasks; - int16_t prio; + _Atomic(int32_t) ntasks; + _Atomic(int16_t) prio; } taskheap_t; /* multiqueue parameters */ @@ -287,7 +287,7 @@ int jl_running_under_rr(int recheck) #ifdef _OS_LINUX_ #define RR_CALL_BASE 1000 #define SYS_rrcall_check_presence (RR_CALL_BASE + 8) - static int is_running_under_rr = 0; + static _Atomic(int) is_running_under_rr = 0; int rr = jl_atomic_load_relaxed(&is_running_under_rr); if (rr == 0 || recheck) { int ret = syscall(SYS_rrcall_check_presence, 0, 0, 0, 0, 0, 0); @@ -414,7 +414,7 @@ static int may_sleep(jl_ptls_t ptls) JL_NOTSAFEPOINT return jl_atomic_load_relaxed(&ptls->sleep_check_state) == sleeping; } -extern unsigned _threadedregion; +extern _Atomic(unsigned) _threadedregion; JL_DLLEXPORT jl_task_t *jl_task_get_next(jl_value_t *trypoptask, jl_value_t *q) { diff --git a/src/rtutils.c b/src/rtutils.c index 67d17c39c67ec..99af741993c44 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -222,7 +222,7 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) // Must have no safepoint eh->prev = ct->eh; eh->gcstack = ct->gcstack; - eh->gc_state = ct->ptls->gc_state; + eh->gc_state = jl_atomic_load_relaxed(&ct->ptls->gc_state); eh->locks_len = ct->ptls->locks.len; eh->defer_signal = ct->ptls->defer_signal; eh->world_age = ct->world_age; @@ -250,7 +250,7 @@ JL_DLLEXPORT void jl_eh_restore_state(jl_handler_t *eh) // This function should **NOT** have any safepoint before the ones at the // end. sig_atomic_t old_defer_signal = ct->ptls->defer_signal; - int8_t old_gc_state = ct->ptls->gc_state; + int8_t old_gc_state = jl_atomic_load_relaxed(&ct->ptls->gc_state); ct->eh = eh->prev; ct->gcstack = eh->gcstack; small_arraylist_t *locks = &ct->ptls->locks; diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index ba265eb67be76..ebbc9668bf6a3 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -54,7 +54,7 @@ void *jl_get_library_(const char *f_lib, int throw_err) } extern "C" JL_DLLEXPORT -void *jl_load_and_lookup(const char *f_lib, const char *f_name, void **hnd) +void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *hnd) { void *handle = jl_atomic_load_acquire(hnd); if (!handle) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 741bb5448b847..00c110f7ce1c5 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -83,7 +83,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerref(jl_value_t *p, jl_value_t *order) jl_value_t *ety = jl_tparam0(jl_typeof(p)); char *pp = (char*)jl_unbox_long(p); if (ety == (jl_value_t*)jl_any_type) { - return jl_atomic_load((jl_value_t**)pp); + return jl_atomic_load((_Atomic(jl_value_t*)*)pp); } else { if (!is_valid_intrinsic_elptr(ety)) @@ -103,7 +103,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerset(jl_value_t *p, jl_value_t *x, jl_v jl_value_t *ety = jl_tparam0(jl_typeof(p)); char *pp = (char*)jl_unbox_long(p); if (ety == (jl_value_t*)jl_any_type) { - jl_atomic_store((jl_value_t**)pp, x); + jl_atomic_store((_Atomic(jl_value_t*)*)pp, x); } else { if (!is_valid_intrinsic_elptr(ety)) @@ -127,7 +127,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerswap(jl_value_t *p, jl_value_t *x, jl_ jl_value_t *y; char *pp = (char*)jl_unbox_long(p); if (ety == (jl_value_t*)jl_any_type) { - y = jl_atomic_exchange((jl_value_t**)pp, x); + y = jl_atomic_exchange((_Atomic(jl_value_t*)*)pp, x); } else { if (!is_valid_intrinsic_elptr(ety)) @@ -151,7 +151,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointermodify(jl_value_t *p, jl_value_t *f, j char *pp = (char*)jl_unbox_long(p); jl_value_t *expected; if (ety == (jl_value_t*)jl_any_type) { - expected = jl_atomic_load((jl_value_t**)pp); + expected = jl_atomic_load((_Atomic(jl_value_t*)*)pp); } else { if (!is_valid_intrinsic_elptr(ety)) @@ -169,7 +169,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointermodify(jl_value_t *p, jl_value_t *f, j jl_value_t *y = jl_apply_generic(f, args, 2); args[1] = y; if (ety == (jl_value_t*)jl_any_type) { - if (jl_atomic_cmpswap((jl_value_t**)pp, &expected, y)) + if (jl_atomic_cmpswap((_Atomic(jl_value_t*)*)pp, &expected, y)) break; } else { @@ -215,7 +215,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerreplace(jl_value_t *p, jl_value_t *exp result = expected; int success; while (1) { - success = jl_atomic_cmpswap((jl_value_t**)pp, &result, x); + success = jl_atomic_cmpswap((_Atomic(jl_value_t*)*)pp, &result, x); if (success || !jl_egal(result, expected)) break; } diff --git a/src/safepoint.c b/src/safepoint.c index d54c7c62bec56..dba2387fcde2e 100644 --- a/src/safepoint.c +++ b/src/safepoint.c @@ -19,7 +19,7 @@ extern "C" { // 1: at least one sigint is pending, only the sigint page is enabled. // 2: at least one sigint is pending, both safepoint pages are enabled. JL_DLLEXPORT sig_atomic_t jl_signal_pending = 0; -uint32_t jl_gc_running = 0; +_Atomic(uint32_t) jl_gc_running = 0; char *jl_safepoint_pages = NULL; // The number of safepoints enabled on the three pages. // The first page, is the SIGINT page, only used by the master thread. @@ -115,7 +115,7 @@ int jl_safepoint_start_gc(void) return 1; } // The thread should have set this already - assert(jl_current_task->ptls->gc_state == JL_GC_STATE_WAITING); + assert(jl_atomic_load_relaxed(&jl_current_task->ptls->gc_state) == JL_GC_STATE_WAITING); jl_mutex_lock_nogc(&safepoint_lock); // In case multiple threads enter the GC at the same time, only allow // one of them to actually run the collection. We can't just let the @@ -157,7 +157,7 @@ void jl_safepoint_end_gc(void) void jl_safepoint_wait_gc(void) { // The thread should have set this is already - assert(jl_current_task->ptls->gc_state != 0); + assert(jl_atomic_load_relaxed(&jl_current_task->ptls->gc_state) != 0); // Use normal volatile load in the loop for speed until GC finishes. // Then use an acquire load to make sure the GC result is visible on this thread. while (jl_atomic_load_relaxed(&jl_gc_running) || jl_atomic_load_acquire(&jl_gc_running)) { diff --git a/src/signals-unix.c b/src/signals-unix.c index bb19e2bd65d78..f7087b252469b 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -780,7 +780,22 @@ static void *signal_listener(void *arg) } jl_set_safe_restore(old_buf); - // Mark the end of this block with 0 + jl_ptls_t ptls = jl_all_tls_states[i]; + + // store threadid but add 1 as 0 is preserved to indicate end of block + bt_data_prof[bt_size_cur++].uintptr = ptls->tid + 1; + + // store task id + bt_data_prof[bt_size_cur++].jlvalue = (jl_value_t*)ptls->current_task; + + // store cpu cycle clock + bt_data_prof[bt_size_cur++].uintptr = cycleclock(); + + // store whether thread is sleeping but add 1 as 0 is preserved to indicate end of block + bt_data_prof[bt_size_cur++].uintptr = ptls->sleep_check_state + 1; + + // Mark the end of this block with two 0's + bt_data_prof[bt_size_cur++].uintptr = 0; bt_data_prof[bt_size_cur++].uintptr = 0; } } diff --git a/src/smallintset.c b/src/smallintset.c index 7598d8fd85ce4..54fdad616a758 100644 --- a/src/smallintset.c +++ b/src/smallintset.c @@ -130,14 +130,16 @@ static int smallintset_insert_(jl_array_t *a, uint_t hv, size_t val1) return 0; } -static void smallintset_rehash(jl_array_t **cache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np); +static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np); -void jl_smallintset_insert(jl_array_t **cache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data) +void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data) { - if (val + 1 > jl_max_int(*cache)) - smallintset_rehash(cache, parent, hash, data, jl_array_len(*cache), val + 1); + jl_array_t *a = jl_atomic_load_relaxed(pcache); + if (val + 1 > jl_max_int(a)) + smallintset_rehash(pcache, parent, hash, data, jl_array_len(a), val + 1); while (1) { - if (smallintset_insert_(*cache, hash(val, data), val + 1)) + a = jl_atomic_load_relaxed(pcache); + if (smallintset_insert_(a, hash(val, data), val + 1)) return; /* table full */ @@ -145,20 +147,21 @@ void jl_smallintset_insert(jl_array_t **cache, jl_value_t *parent, smallintset_h /* it's important to grow the table really fast; otherwise we waste */ /* lots of time rehashing all the keys over and over. */ size_t newsz; - size_t sz = jl_array_len(*cache); + a = jl_atomic_load_relaxed(pcache); + size_t sz = jl_array_len(a); if (sz < HT_N_INLINE) newsz = HT_N_INLINE; else if (sz >= (1 << 19) || (sz <= (1 << 8))) newsz = sz << 1; else newsz = sz << 2; - smallintset_rehash(cache, parent, hash, data, newsz, 0); + smallintset_rehash(pcache, parent, hash, data, newsz, 0); } } -static void smallintset_rehash(jl_array_t **cache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np) +static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np) { - jl_array_t *a = *cache; + jl_array_t *a = jl_atomic_load_relaxed(pcache); size_t sz = jl_array_len(a); size_t i; for (i = 0; i < sz; i += 1) { @@ -179,7 +182,7 @@ static void smallintset_rehash(jl_array_t **cache, jl_value_t *parent, smallints } JL_GC_POP(); if (i == sz) { - *cache = newa; + jl_atomic_store_release(pcache, newa); jl_gc_wb(parent, newa); return; } diff --git a/src/staticdata.c b/src/staticdata.c index 0a473684ff5dc..537abd77aa669 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -403,8 +403,8 @@ static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m) jl_serialize_value(s, (jl_value_t*)table[i]); jl_binding_t *b = (jl_binding_t*)table[i+1]; jl_serialize_value(s, b->name); - jl_serialize_value(s, b->value); - jl_serialize_value(s, b->globalref); + jl_serialize_value(s, jl_atomic_load_relaxed(&b->value)); + jl_serialize_value(s, jl_atomic_load_relaxed(&b->globalref)); jl_serialize_value(s, b->owner); } } @@ -655,8 +655,8 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t record_gvar(s, jl_get_llvm_gv(native_functions, (jl_value_t*)b), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset); write_pointerfield(s, (jl_value_t*)b->name); - write_pointerfield(s, b->value); - write_pointerfield(s, b->globalref); + write_pointerfield(s, jl_atomic_load_relaxed(&b->value)); + write_pointerfield(s, jl_atomic_load_relaxed(&b->globalref)); write_pointerfield(s, (jl_value_t*)b->owner); size_t flag_offset = offsetof(jl_binding_t, owner) + sizeof(b->owner); ios_write(s->s, (char*)b + flag_offset, sizeof(*b) - flag_offset); diff --git a/src/subtype.c b/src/subtype.c index 2e1bfb4b05a66..4c1ae89f6c0ab 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -107,7 +107,7 @@ typedef struct jl_stenv_t { // state manipulation utilities // look up a type variable in an environment -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ static jl_varbinding_t *lookup(jl_stenv_t *e, jl_tvar_t *v) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; #else static jl_varbinding_t *lookup(jl_stenv_t *e, jl_tvar_t *v) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT @@ -171,7 +171,7 @@ static void save_env(jl_stenv_t *e, jl_value_t **root, jl_savedenv_t *se) if (root) *root = (jl_value_t*)jl_alloc_svec(len * 3); se->buf = (int8_t*)(len > 8 ? malloc_s(len * 2) : &se->_space); -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ memset(se->buf, 0, len * 2); #endif int i=0, j=0; v = e->vars; diff --git a/src/support/analyzer_annotations.h b/src/support/analyzer_annotations.h index 1579584a572a9..70b5a273953f1 100644 --- a/src/support/analyzer_annotations.h +++ b/src/support/analyzer_annotations.h @@ -8,7 +8,7 @@ #endif #define JL_NONNULL _Nonnull -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ #define JL_PROPAGATES_ROOT __attribute__((annotate("julia_propagates_root"))) #define JL_NOTSAFEPOINT __attribute__((annotate("julia_not_safepoint"))) diff --git a/src/symbol.c b/src/symbol.c index fe8e975f8f525..fd5024eefeca2 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -15,7 +15,7 @@ extern "C" { #endif -static jl_sym_t *symtab = NULL; +static _Atomic(jl_sym_t*) symtab = NULL; #define MAX_SYM_LEN ((size_t)INTPTR_MAX - sizeof(jl_taggedvalue_t) - sizeof(jl_sym_t) - 1) @@ -48,9 +48,9 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT return sym; } -static jl_sym_t *symtab_lookup(jl_sym_t **ptree, const char *str, size_t len, jl_sym_t ***slot) JL_NOTSAFEPOINT +static jl_sym_t *symtab_lookup(_Atomic(jl_sym_t*) *ptree, const char *str, size_t len, _Atomic(jl_sym_t*) **slot) JL_NOTSAFEPOINT { - jl_sym_t *node = jl_atomic_load_acquire(ptree); // consume + jl_sym_t *node = jl_atomic_load_relaxed(ptree); // consume uintptr_t h = hash_symbol(str, len); // Tree nodes sorted by major key of (int(hash)) and minor key of (str). @@ -68,7 +68,7 @@ static jl_sym_t *symtab_lookup(jl_sym_t **ptree, const char *str, size_t len, jl ptree = &node->left; else ptree = &node->right; - node = jl_atomic_load_acquire(ptree); // consume + node = jl_atomic_load_relaxed(ptree); // consume } if (slot != NULL) *slot = ptree; @@ -77,14 +77,14 @@ static jl_sym_t *symtab_lookup(jl_sym_t **ptree, const char *str, size_t len, jl jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT // (or throw) { -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ // Hide the error throwing from the analyser since there isn't a way to express // "safepoint only when throwing error" currently. if (len > MAX_SYM_LEN) jl_exceptionf(jl_argumenterror_type, "Symbol name too long"); #endif assert(!memchr(str, 0, len)); - jl_sym_t **slot; + _Atomic(jl_sym_t*) *slot; jl_sym_t *node = symtab_lookup(&symtab, str, len, &slot); if (node == NULL) { JL_LOCK_NOGC(&gc_perm_lock); @@ -122,7 +122,7 @@ JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void) return symtab; } -static uint32_t gs_ctr = 0; // TODO: per-thread +static _Atomic(uint32_t) gs_ctr = 0; // TODO: per-module? uint32_t jl_get_gs_ctr(void) { return gs_ctr; } void jl_set_gs_ctr(uint32_t ctr) { gs_ctr = ctr; } diff --git a/src/task.c b/src/task.c index 82b988a588132..c1d7edede69dc 100644 --- a/src/task.c +++ b/src/task.c @@ -188,7 +188,7 @@ static void restore_stack2(jl_task_t *t, jl_ptls_t ptls, jl_task_t *lastt) #endif /* Rooted by the base module */ -static jl_function_t *task_done_hook_func JL_GLOBALLY_ROOTED = NULL; +static _Atomic(jl_function_t*) task_done_hook_func JL_GLOBALLY_ROOTED = NULL; void JL_NORETURN jl_finish_task(jl_task_t *t) { @@ -727,7 +727,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion t->next = jl_nothing; t->queue = jl_nothing; t->tls = jl_nothing; - t->_state = JL_TASK_STATE_RUNNABLE; + jl_atomic_store_relaxed(&t->_state, JL_TASK_STATE_RUNNABLE); t->start = start; t->result = jl_nothing; t->donenotify = completion_future; @@ -743,7 +743,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion t->excstack = NULL; t->started = 0; t->prio = -1; - t->tid = t->copy_stack ? ct->tid : -1; // copy_stacks are always pinned since they can't be moved + jl_atomic_store_relaxed(&t->tid, t->copy_stack ? ct->tid : -1); // copy_stacks are always pinned since they can't be moved t->ptls = NULL; t->world_age = 0; @@ -840,7 +840,7 @@ STATIC_OR_JS void NOINLINE JL_NORETURN start_task(void) CFI_NORETURN // this runs the first time we switch to a task sanitizer_finish_switch_fiber(); -#ifdef __clang_analyzer__ +#ifdef __clang_gcanalyzer__ jl_task_t *ct = jl_get_current_task(); JL_GC_PROMISE_ROOTED(ct); #else @@ -940,7 +940,7 @@ static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) char *stkbuf = (char*)jl_malloc_stack(ssize, owner); if (stkbuf == NULL) return NULL; -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ ((char**)t)[0] = stkbuf; // stash the stack pointer somewhere for start_fiber ((size_t*)t)[1] = *ssize; // stash the stack size somewhere for start_fiber #endif @@ -1289,7 +1289,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi) ct->next = jl_nothing; ct->queue = jl_nothing; ct->tls = jl_nothing; - ct->_state = JL_TASK_STATE_RUNNABLE; + jl_atomic_store_relaxed(&ct->_state, JL_TASK_STATE_RUNNABLE); ct->start = NULL; ct->result = jl_nothing; ct->donenotify = jl_nothing; @@ -1298,7 +1298,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi) ct->eh = NULL; ct->gcstack = NULL; ct->excstack = NULL; - ct->tid = ptls->tid; + jl_atomic_store_relaxed(&ct->tid, ptls->tid); ct->sticky = 1; ct->ptls = ptls; ct->world_age = 1; // OK to run Julia code on this task diff --git a/src/threading.c b/src/threading.c index ffe53c07b45ee..cac82eb4b2e84 100644 --- a/src/threading.c +++ b/src/threading.c @@ -271,7 +271,7 @@ JL_DLLEXPORT void jl_pgcstack_setkey(jl_get_pgcstack_func *f, jl_pgcstack_key_t JL_DLLEXPORT jl_gcframe_t **jl_get_pgcstack(void) JL_GLOBALLY_ROOTED { -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ return jl_get_pgcstack_cb(); #endif } @@ -287,8 +287,8 @@ void jl_pgcstack_getkey(jl_get_pgcstack_func **f, jl_pgcstack_key_t *k) #endif jl_ptls_t *jl_all_tls_states JL_GLOBALLY_ROOTED; -uint8_t jl_measure_compile_time_enabled = 0; -uint64_t jl_cumulative_compile_time = 0; +_Atomic(uint8_t) jl_measure_compile_time_enabled = 0; +_Atomic(uint64_t) jl_cumulative_compile_time = 0; // return calling thread's ID // Also update the suspended_threads list in signals-mach when changing the @@ -467,7 +467,7 @@ void jl_init_threading(void) } if (jl_n_threads <= 0) jl_n_threads = 1; -#ifndef __clang_analyzer__ +#ifndef __clang_gcanalyzer__ jl_all_tls_states = (jl_ptls_t*)calloc(jl_n_threads, sizeof(void*)); #endif } @@ -527,7 +527,7 @@ void jl_start_threads(void) uv_barrier_wait(&thread_init_done); } -unsigned volatile _threadedregion; // HACK: keep track of whether it is safe to do IO +_Atomic(unsigned) _threadedregion; // HACK: keep track of whether it is safe to do IO JL_DLLEXPORT int jl_in_threaded_region(void) { diff --git a/src/typemap.c b/src/typemap.c index 58dd2b8b13069..13e97a5ada160 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -259,28 +259,28 @@ static int is_cache_leaf(jl_value_t *ty, int tparam) return (jl_is_concrete_type(ty) && (tparam || !jl_is_kind(ty))); } -static jl_typemap_t **mtcache_hash_lookup_bp(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT +static _Atomic(jl_typemap_t*) *mtcache_hash_lookup_bp(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT { if (cache == (jl_array_t*)jl_an_empty_vec_any) return NULL; - jl_typemap_t **pml = jl_table_peek_bp(cache, ty); + _Atomic(jl_typemap_t*) *pml = jl_table_peek_bp(cache, ty); JL_GC_PROMISE_ROOTED(pml); // clang-sa doesn't trust our JL_PROPAGATES_ROOT claim return pml; } -static void mtcache_hash_insert(jl_array_t **cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) +static void mtcache_hash_insert(_Atomic(jl_array_t*) *cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) { int inserted = 0; - jl_array_t *a = *cache; + jl_array_t *a = jl_atomic_load_relaxed(cache); if (a == (jl_array_t*)jl_an_empty_vec_any) { a = jl_alloc_vec_any(16); - *cache = a; + jl_atomic_store_release(cache, a); jl_gc_wb(parent, a); } a = jl_eqtable_put(a, key, val, &inserted); assert(inserted); if (a != *cache) { - *cache = a; + jl_atomic_store_release(cache, a); jl_gc_wb(parent, a); } } @@ -299,7 +299,7 @@ static jl_typemap_t *mtcache_hash_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, j static int jl_typemap_array_visitor(jl_array_t *a, jl_typemap_visitor_fptr fptr, void *closure) { size_t i, l = jl_array_len(a); - jl_typemap_t **data = (jl_typemap_t **)jl_array_data(a); + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_data(a); for (i = 1; i < l; i += 2) { jl_value_t *d = jl_atomic_load_relaxed(&data[i]); JL_GC_PROMISE_ROOTED(d); @@ -394,7 +394,7 @@ static int jl_typemap_intersection_array_visitor(jl_array_t *a, jl_value_t *ty, { JL_GC_PUSH1(&a); size_t i, l = jl_array_len(a); - jl_typemap_t **data = (jl_typemap_t **)jl_array_data(a); + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_data(a); unsigned height = tparam & 2 ? jl_supertype_height((jl_datatype_t*)ty) : 0; for (i = 0; i < l; i += 2) { jl_value_t *t = jl_atomic_load_relaxed(&data[i]); @@ -845,7 +845,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (!ty || !jl_has_empty_intersection((jl_value_t*)jl_type_type, ty)) { // couldn't figure out unique `a0` initial point, so scan all for matches size_t i, l = jl_array_len(tname); - jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(tname); + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(tname); JL_GC_PUSH1(&tname); for (i = 1; i < l; i += 2) { jl_typemap_t *ml = jl_atomic_load_relaxed(&data[i]); @@ -884,7 +884,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( else { // doing subtype, but couldn't figure out unique `ty`, so scan all for supertypes size_t i, l = jl_array_len(name1); - jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(name1); + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(name1); JL_GC_PUSH1(&name1); for (i = 1; i < l; i += 2) { jl_typemap_t *ml = jl_atomic_load_relaxed(&data[i]); @@ -1034,7 +1034,7 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v else { // couldn't figure out unique `name` initial point, so must scan all for matches size_t i, l = jl_array_len(tname); - jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(tname); + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(tname); JL_GC_PUSH1(&tname); for (i = 1; i < l; i += 2) { jl_typemap_t *ml_or_cache = data[i]; @@ -1122,10 +1122,10 @@ static jl_typemap_level_t *jl_method_convert_list_to_cache( } static void jl_typemap_list_insert_( - jl_typemap_t *map, jl_typemap_entry_t **pml, jl_value_t *parent, + jl_typemap_t *map, _Atomic(jl_typemap_entry_t*) *pml, jl_value_t *parent, jl_typemap_entry_t *newrec) { - jl_typemap_entry_t *l = *pml; + jl_typemap_entry_t *l = jl_atomic_load_relaxed(pml); while ((jl_value_t*)l != jl_nothing) { if (newrec->isleafsig || !l->isleafsig) if (newrec->issimplesig || !l->issimplesig) @@ -1141,37 +1141,39 @@ static void jl_typemap_list_insert_( } static void jl_typemap_insert_generic( - jl_typemap_t *map, jl_typemap_t **pml, jl_value_t *parent, + jl_typemap_t *map, _Atomic(jl_typemap_t*) *pml, jl_value_t *parent, jl_typemap_entry_t *newrec, int8_t offs) { - if (jl_typeof(*pml) == (jl_value_t*)jl_typemap_level_type) { - jl_typemap_level_insert_(map, (jl_typemap_level_t*)*pml, newrec, offs); + jl_typemap_t *ml = jl_atomic_load_relaxed(pml); + if (jl_typeof(ml) == (jl_value_t*)jl_typemap_level_type) { + jl_typemap_level_insert_(map, (jl_typemap_level_t*)ml, newrec, offs); return; } - unsigned count = jl_typemap_list_count_locked((jl_typemap_entry_t*)*pml); + unsigned count = jl_typemap_list_count_locked((jl_typemap_entry_t*)ml); if (count > MAX_METHLIST_COUNT) { - *pml = (jl_typemap_t*)jl_method_convert_list_to_cache( - map, (jl_typemap_entry_t *)*pml, - offs); - jl_gc_wb(parent, *pml); - jl_typemap_level_insert_(map, (jl_typemap_level_t*)*pml, newrec, offs); + ml = (jl_typemap_t*)jl_method_convert_list_to_cache( + map, (jl_typemap_entry_t*)ml, offs); + jl_atomic_store_release(pml, ml); + jl_gc_wb(parent, ml); + jl_typemap_level_insert_(map, (jl_typemap_level_t*)ml, newrec, offs); return; } - jl_typemap_list_insert_(map, (jl_typemap_entry_t **)pml, + jl_typemap_list_insert_(map, (_Atomic(jl_typemap_entry_t*)*)pml, parent, newrec); } static void jl_typemap_array_insert_( - jl_typemap_t *map, jl_array_t **cache, jl_value_t *key, jl_typemap_entry_t *newrec, + jl_typemap_t *map, _Atomic(jl_array_t*) *pcache, jl_value_t *key, jl_typemap_entry_t *newrec, jl_value_t *parent, int8_t offs) { - jl_typemap_t **pml = mtcache_hash_lookup_bp(*cache, key); + jl_array_t *cache = jl_atomic_load_relaxed(pcache); + _Atomic(jl_typemap_t*) *pml = mtcache_hash_lookup_bp(cache, key); if (pml != NULL) - jl_typemap_insert_generic(map, pml, (jl_value_t*)*cache, newrec, offs+1); + jl_typemap_insert_generic(map, pml, (jl_value_t*)cache, newrec, offs+1); else - mtcache_hash_insert(cache, parent, key, (jl_typemap_t*)newrec); + mtcache_hash_insert(pcache, parent, key, (jl_typemap_t*)newrec); } static void jl_typemap_level_insert_( @@ -1276,7 +1278,7 @@ jl_typemap_entry_t *jl_typemap_alloc( newrec->simplesig = simpletype; newrec->func.value = newvalue; newrec->guardsigs = guardsigs; - newrec->next = (jl_typemap_entry_t*)jl_nothing; + jl_atomic_store_relaxed(&newrec->next, (jl_typemap_entry_t*)jl_nothing); newrec->min_world = min_world; newrec->max_world = max_world; newrec->va = isva; @@ -1285,10 +1287,11 @@ jl_typemap_entry_t *jl_typemap_alloc( return newrec; } -void jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent, +void jl_typemap_insert(_Atomic(jl_typemap_t *) *pcache, jl_value_t *parent, jl_typemap_entry_t *newrec, int8_t offs) { - jl_typemap_insert_generic(*cache, cache, parent, newrec, offs); + jl_typemap_t *cache = jl_atomic_load_relaxed(pcache); + jl_typemap_insert_generic(cache, pcache, parent, newrec, offs); } #ifdef __cplusplus diff --git a/test/clangsa/GCPushPop.cpp b/test/clangsa/GCPushPop.cpp index a992630291bb5..f57f9ba7711df 100644 --- a/test/clangsa/GCPushPop.cpp +++ b/test/clangsa/GCPushPop.cpp @@ -1,6 +1,6 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license -// RUN: clang --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -Xclang -verify -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker -x c++ %s +// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -Xclang -verify -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker -x c++ %s #include "julia.h" diff --git a/test/clangsa/MissingRoots.c b/test/clangsa/MissingRoots.c index 78dcc195d59ce..f0b32c54bc7b8 100644 --- a/test/clangsa/MissingRoots.c +++ b/test/clangsa/MissingRoots.c @@ -1,6 +1,6 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license -// RUN: clang --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -Xclang -verify -x c %s +// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -Xclang -verify -x c %s #include "julia.h" #include "julia_internal.h"