diff --git a/base/Base.jl b/base/Base.jl index 87e1fe94b0ad2..c729ab9ca707e 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -91,6 +91,36 @@ function replaceproperty!(x, f::Symbol, expected, desired, success_order::Symbol val = desired isa ty ? desired : convert(ty, desired) return Core.replacefield!(x, f, expected, val, success_order, fail_order) end +function setpropertyonce!(x, f::Symbol, desired, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) + @inline + ty = fieldtype(typeof(x), f) + val = desired isa ty ? desired : convert(ty, desired) + return Core.setfieldonce!(x, f, val, success_order, fail_order) +end + +function swapproperty!(x::Module, f::Symbol, v, order::Symbol=:not_atomic) + @inline + ty = Core.get_binding_type(x, f) + val = v isa ty ? v : convert(ty, v) + return Core.swapglobal!(x, f, val, order) +end +function modifyproperty!(x::Module, f::Symbol, op, v, order::Symbol=:not_atomic) + @inline + return Core.modifyglobal!(x, f, op, v, order) +end +function replaceproperty!(x::Module, f::Symbol, expected, desired, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) + @inline + ty = Core.get_binding_type(x, f) + val = desired isa ty ? desired : convert(ty, desired) + return Core.replaceglobal!(x, f, expected, val, success_order, fail_order) +end +function setpropertyonce!(x::Module, f::Symbol, desired, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) + @inline + ty = Core.get_binding_type(x, f) + val = desired isa ty ? desired : convert(ty, desired) + return Core.setglobalonce!(x, f, val, success_order, fail_order) +end + convert(::Type{Any}, Core.@nospecialize x) = x convert(::Type{T}, x::T) where {T} = x diff --git a/base/boot.jl b/base/boot.jl index 13bb6bcd7cd4b..9f34f02981cf6 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -65,7 +65,7 @@ #end # struct GenericMemoryRef{kind::Symbol, T, AS::AddrSpace} -# mem::Memory{kind, T, AS} +# mem::GenericMemory{kind, T, AS} # data::Ptr{Cvoid} # make this GenericPtr{addrspace, Cvoid} #end @@ -191,8 +191,8 @@ export Tuple, Type, UnionAll, TypeVar, Union, Nothing, Cvoid, AbstractArray, DenseArray, NamedTuple, Pair, # special objects - Function, Method, Array, Memory, MemoryRef, GenericMemory, GenericMemoryRef, - Module, Symbol, Task, UndefInitializer, undef, WeakRef, VecElement, + Function, Method, Module, Symbol, Task, UndefInitializer, undef, WeakRef, VecElement, + Array, Memory, MemoryRef, AtomicMemory, AtomicMemoryRef, GenericMemory, GenericMemoryRef, # numeric types Number, Real, Integer, Bool, Ref, Ptr, AbstractFloat, Float16, Float32, Float64, @@ -209,10 +209,10 @@ export # AST representation Expr, QuoteNode, LineNumberNode, GlobalRef, # object model functions - fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!, + fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!, setfieldonce!, nfields, throw, tuple, ===, isdefined, eval, # access to globals - getglobal, setglobal!, + getglobal, setglobal!, swapglobal!, modifyglobal!, replaceglobal!, setglobalonce!, # ifelse, sizeof # not exported, to avoid conflicting with Base # type reflection <:, typeof, isa, typeassert, @@ -516,22 +516,24 @@ const undef = UndefInitializer() (self::Type{GenericMemory{kind,T,addrspace}})(::UndefInitializer, d::NTuple{1,Int}) where {T,kind,addrspace} = self(undef, getfield(d,1)) # empty vector constructor (self::Type{GenericMemory{kind,T,addrspace}})() where {T,kind,addrspace} = self(undef, 0) -# copy constructors -const Memory{T} = GenericMemory{:not_atomic, T, CPU} -const MemoryRef{T} = GenericMemoryRef{:not_atomic, T, CPU} GenericMemoryRef(mem::GenericMemory) = memoryref(mem) GenericMemoryRef(ref::GenericMemoryRef, i::Integer) = memoryref(ref, Int(i), @_boundscheck) GenericMemoryRef(mem::GenericMemory, i::Integer) = memoryref(memoryref(mem), Int(i), @_boundscheck) -MemoryRef(mem::Memory) = memoryref(mem) -MemoryRef(ref::MemoryRef, i::Integer) = memoryref(ref, Int(i), @_boundscheck) -MemoryRef(mem::Memory, i::Integer) = memoryref(memoryref(mem), Int(i), @_boundscheck) -MemoryRef{T}(mem::Memory{T}) where {T} = memoryref(mem) -MemoryRef{T}(ref::MemoryRef{T}, i::Integer) where {T} = memoryref(ref, Int(i), @_boundscheck) -MemoryRef{T}(mem::Memory{T}, i::Integer) where {T} = memoryref(memoryref(mem), Int(i), @_boundscheck) +GenericMemoryRef{kind,<:Any,AS}(mem::GenericMemory{kind,<:Any,AS}) where {kind,AS} = memoryref(mem) +GenericMemoryRef{kind,<:Any,AS}(ref::GenericMemoryRef{kind,<:Any,AS}, i::Integer) where {kind,AS} = memoryref(ref, Int(i), @_boundscheck) +GenericMemoryRef{kind,<:Any,AS}(mem::GenericMemory{kind,<:Any,AS}, i::Integer) where {kind,AS} = memoryref(memoryref(mem), Int(i), @_boundscheck) +GenericMemoryRef{kind,T,AS}(mem::GenericMemory{kind,T,AS}) where {kind,T,AS} = memoryref(mem) +GenericMemoryRef{kind,T,AS}(ref::GenericMemoryRef{kind,T,AS}, i::Integer) where {kind,T,AS} = memoryref(ref, Int(i), @_boundscheck) +GenericMemoryRef{kind,T,AS}(mem::GenericMemory{kind,T,AS}, i::Integer) where {kind,T,AS} = memoryref(memoryref(mem), Int(i), @_boundscheck) + +const Memory{T} = GenericMemory{:not_atomic, T, CPU} +const MemoryRef{T} = GenericMemoryRef{:not_atomic, T, CPU} +const AtomicMemory{T} = GenericMemory{:atomic, T, CPU} +const AtomicMemoryRef{T} = GenericMemoryRef{:atomic, T, CPU} # construction helpers for Array -new_as_memoryref(self::Type{GenericMemoryRef{isatomic,T,addrspace}}, m::Int) where {T,isatomic,addrspace} = memoryref(fieldtype(self, :mem)(undef, m)) +new_as_memoryref(self::Type{GenericMemoryRef{kind,T,addrspace}}, m::Int) where {T,kind,addrspace} = memoryref(fieldtype(self, :mem)(undef, m)) # checked-multiply intrinsic function for dimensions _checked_mul_dims() = 1, false diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 292a39e42e77b..889a9a541246e 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1113,12 +1113,12 @@ function const_prop_function_heuristic(interp::AbstractInterpreter, @nospecializ if !still_nothrow || ismutabletype(arrty) return false end - elseif โŠ‘(๐•ƒแตข, arrty, Array) + elseif โŠ‘(๐•ƒแตข, arrty, Array) || โŠ‘(๐•ƒแตข, arrty, GenericMemory) return false end elseif istopfunction(f, :iterate) itrty = argtypes[2] - if โŠ‘(๐•ƒแตข, itrty, Array) + if โŠ‘(๐•ƒแตข, itrty, Array) || โŠ‘(๐•ƒแตข, itrty, GenericMemory) return false end end @@ -1501,7 +1501,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n # Return Bottom if this is not an iterator. # WARNING: Changes to the iteration protocol must be reflected here, # this is not just an optimization. - # TODO: this doesn't realize that Array, SimpleVector, Tuple, and NamedTuple do not use the iterate protocol + # TODO: this doesn't realize that Array, GenericMemory, SimpleVector, Tuple, and NamedTuple do not use the iterate protocol stateordonet === Bottom && return AbstractIterationResult(Any[Bottom], AbstractIterationInfo(CallMeta[CallMeta(Bottom, Any, call.effects, info)], true)) valtype = statetype = Bottom ret = Any[] @@ -2076,8 +2076,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return abstract_apply(interp, argtypes, si, sv, max_methods) elseif f === invoke return abstract_invoke(interp, arginfo, si, sv) - elseif f === modifyfield! - return abstract_modifyfield!(interp, argtypes, si, sv) + elseif f === modifyfield! || f === Core.modifyglobal! || f === Core.memoryrefmodify! || f === atomic_pointermodify + return abstract_modifyop!(interp, f, argtypes, si, sv) elseif f === Core.finalizer return abstract_finalizer(interp, argtypes, sv) elseif f === applicable diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index e6438b3f9325f..74d3987ee0257 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1286,10 +1286,18 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stat end end - if (sig.f !== Core.invoke && sig.f !== Core.finalizer && sig.f !== modifyfield!) && - is_builtin(optimizer_lattice(state.interp), sig) - # No inlining for builtins (other invoke/apply/typeassert/finalizer) - return nothing + if is_builtin(optimizer_lattice(state.interp), sig) + let f = sig.f + if (f !== Core.invoke && + f !== Core.finalizer && + f !== modifyfield! && + f !== Core.modifyglobal! && + f !== Core.memoryrefmodify! && + f !== atomic_pointermodify) + # No inlining defined for most builtins (just invoke/apply/typeassert/finalizer), so attempt an early exit for them + return nothing + end + end end # Special case inliners for regular functions @@ -1571,7 +1579,7 @@ function handle_opaque_closure_call!(todo::Vector{Pair{Int,Any}}, return nothing end -function handle_modifyfield!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyFieldInfo, state::InliningState) +function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOpInfo, state::InliningState) info = info.info info isa MethodResultPure && (info = info.info) info isa ConstCallInfo && (info = info.call) @@ -1679,8 +1687,8 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) # handle special cased builtins if isa(info, OpaqueClosureCallInfo) handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) - elseif isa(info, ModifyFieldInfo) - handle_modifyfield!_call!(ir, idx, stmt, info, state) + elseif isa(info, ModifyOpInfo) + handle_modifyop!_call!(ir, idx, stmt, info, state) elseif isa(info, InvokeCallInfo) handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) diff --git a/base/compiler/stmtinfo.jl b/base/compiler/stmtinfo.jl index e28858eea60aa..25f5bb894eaa9 100644 --- a/base/compiler/stmtinfo.jl +++ b/base/compiler/stmtinfo.jl @@ -222,13 +222,18 @@ struct FinalizerInfo <: CallInfo end """ - info::ModifyFieldInfo <: CallInfo + info::ModifyOpInfo <: CallInfo -Represents a resolved all of `modifyfield!(obj, name, op, x, [order])`. -`info.info` wraps the call information of `op(getfield(obj, name), x)`. +Represents a resolved call of one of: + - `modifyfield!(obj, name, op, x, [order])` + - `modifyglobal!(mod, var, op, x, order)` + - `memoryrefmodify!(memref, op, x, order, boundscheck)` + - `Intrinsics.atomic_pointermodify(ptr, op, x, order)` + +`info.info` wraps the call information of `op(getval(), x)`. """ -struct ModifyFieldInfo <: CallInfo - info::CallInfo # the callinfo for the `op(getfield(obj, name), x)` call +struct ModifyOpInfo <: CallInfo + info::CallInfo # the callinfo for the `op(getval(), x)` call end @specialize diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index d4485783c191c..b817c2af9c49c 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1363,40 +1363,88 @@ end return false end -@nospecs function swapfield!_tfunc(๐•ƒ::AbstractLattice, o, f, v, order) +@nospecs function swapfield!_tfunc(๐•ƒ::AbstractLattice, o, f, v, order=Symbol) + setfield!_tfunc(๐•ƒ, o, f, v) === Bottom && return Bottom return getfield_tfunc(๐•ƒ, o, f) end -@nospecs function swapfield!_tfunc(๐•ƒ::AbstractLattice, o, f, v) - return getfield_tfunc(๐•ƒ, o, f) -end -@nospecs function modifyfield!_tfunc(๐•ƒ::AbstractLattice, o, f, op, v, order) - return modifyfield!_tfunc(๐•ƒ, o, f, op, v) -end -@nospecs function modifyfield!_tfunc(๐•ƒ::AbstractLattice, o, f, op, v) +@nospecs function modifyfield!_tfunc(๐•ƒ::AbstractLattice, o, f, op, v, order=Symbol) T = _fieldtype_tfunc(๐•ƒ, o, f, isconcretetype(o)) T === Bottom && return Bottom PT = Const(Pair) return instanceof_tfunc(apply_type_tfunc(๐•ƒ, PT, T, T), true)[1] end -function abstract_modifyfield!(interp::AbstractInterpreter, argtypes::Vector{Any}, si::StmtInfo, sv::AbsIntState) +@nospecs function replacefield!_tfunc(๐•ƒ::AbstractLattice, o, f, x, v, success_order=Symbol, failure_order=Symbol) + T = _fieldtype_tfunc(๐•ƒ, o, f, isconcretetype(o)) + T === Bottom && return Bottom + PT = Const(ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T) + return instanceof_tfunc(apply_type_tfunc(๐•ƒ, PT, T), true)[1] +end +@nospecs function setfieldonce!_tfunc(๐•ƒ::AbstractLattice, o, f, v, success_order=Symbol, failure_order=Symbol) + setfield!_tfunc(๐•ƒ, o, f, v) === Bottom && return Bottom + isdefined_tfunc(๐•ƒ, o, f) === Const(true) && return Const(false) + return Bool +end + +@nospecs function abstract_modifyop!(interp::AbstractInterpreter, ff, argtypes::Vector{Any}, si::StmtInfo, sv::AbsIntState) + if ff === modifyfield! + minargs = 5 + maxargs = 6 + op_argi = 4 + v_argi = 5 + elseif ff === Core.modifyglobal! + minargs = 5 + maxargs = 6 + op_argi = 4 + v_argi = 5 + elseif ff === Core.memoryrefmodify! + minargs = 6 + maxargs = 6 + op_argi = 3 + v_argi = 4 + elseif ff === atomic_pointermodify + minargs = 5 + maxargs = 5 + op_argi = 3 + v_argi = 4 + else + @assert false "unreachable" + end + nargs = length(argtypes) if !isempty(argtypes) && isvarargtype(argtypes[nargs]) - nargs - 1 <= 6 || return CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo()) - nargs > 3 || return CallMeta(Any, Any, Effects(), NoCallInfo()) + nargs - 1 <= maxargs || return CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo()) + nargs + 1 >= op_argi || return CallMeta(Any, Any, Effects(), NoCallInfo()) else - 5 <= nargs <= 6 || return CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo()) + minargs <= nargs <= maxargs || return CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo()) end ๐•ƒแตข = typeinf_lattice(interp) - o = unwrapva(argtypes[2]) - f = unwrapva(argtypes[3]) - RT = modifyfield!_tfunc(๐•ƒแตข, o, f, Any, Any) + if ff === modifyfield! + o = unwrapva(argtypes[2]) + f = unwrapva(argtypes[3]) + RT = modifyfield!_tfunc(๐•ƒแตข, o, f, Any, Any, Symbol) + TF = getfield_tfunc(๐•ƒแตข, o, f) + elseif ff === Core.modifyglobal! + o = unwrapva(argtypes[2]) + f = unwrapva(argtypes[3]) + RT = modifyglobal!_tfunc(๐•ƒแตข, o, f, Any, Any, Symbol) + TF = getglobal_tfunc(๐•ƒแตข, o, f, Symbol) + elseif ff === Core.memoryrefmodify! + o = unwrapva(argtypes[2]) + RT = memoryrefmodify!_tfunc(๐•ƒแตข, o, Any, Any, Symbol, Bool) + TF = memoryrefget_tfunc(๐•ƒแตข, o, Symbol, Bool) + elseif ff === atomic_pointermodify + o = unwrapva(argtypes[2]) + RT = atomic_pointermodify_tfunc(๐•ƒแตข, o, Any, Any, Symbol) + TF = atomic_pointerref_tfunc(๐•ƒแตข, o, Symbol) + else + @assert false "unreachable" + end info = NoCallInfo() - if nargs >= 5 && RT !== Bottom + if nargs >= v_argi && RT !== Bottom # we may be able to refine this to a PartialStruct by analyzing `op(o.f, v)::T` # as well as compute the info for the method matches - op = unwrapva(argtypes[4]) - v = unwrapva(argtypes[5]) - TF = getfield_tfunc(๐•ƒแตข, o, f) + op = unwrapva(argtypes[op_argi]) + v = unwrapva(argtypes[v_argi]) callinfo = abstract_call(interp, ArgInfo(nothing, Any[op, TF, v]), StmtInfo(true), sv, #=max_methods=#1) TF2 = tmeet(callinfo.rt, widenconst(TF)) if TF2 === Bottom @@ -1404,31 +1452,19 @@ function abstract_modifyfield!(interp::AbstractInterpreter, argtypes::Vector{Any elseif isconcretetype(RT) && has_nontrivial_extended_info(๐•ƒแตข, TF2) # isconcrete condition required to form a PartialStruct RT = PartialStruct(RT, Any[TF, TF2]) end - info = ModifyFieldInfo(callinfo.info) + info = ModifyOpInfo(callinfo.info) end return CallMeta(RT, Any, Effects(), info) end -@nospecs function replacefield!_tfunc(๐•ƒ::AbstractLattice, o, f, x, v, success_order, failure_order) - return replacefield!_tfunc(๐•ƒ, o, f, x, v) -end -@nospecs function replacefield!_tfunc(๐•ƒ::AbstractLattice, o, f, x, v, success_order) - return replacefield!_tfunc(๐•ƒ, o, f, x, v) -end -@nospecs function replacefield!_tfunc(๐•ƒ::AbstractLattice, o, f, x, v) - T = _fieldtype_tfunc(๐•ƒ, o, f, isconcretetype(o)) - T === Bottom && return Bottom - PT = Const(ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T) - return instanceof_tfunc(apply_type_tfunc(๐•ƒ, PT, T), true)[1] -end # we could use tuple_tfunc instead of widenconst, but `o` is mutable, so that is unlikely to be beneficial add_tfunc(getfield, 2, 4, getfield_tfunc, 1) add_tfunc(setfield!, 3, 4, setfield!_tfunc, 3) - add_tfunc(swapfield!, 3, 4, swapfield!_tfunc, 3) add_tfunc(modifyfield!, 4, 5, modifyfield!_tfunc, 3) add_tfunc(replacefield!, 4, 6, replacefield!_tfunc, 3) +add_tfunc(setfieldonce!, 3, 5, setfieldonce!_tfunc, 3) @nospecs function fieldtype_nothrow(๐•ƒ::AbstractLattice, s0, name) s0 === Bottom && return true # unreachable @@ -1976,19 +2012,45 @@ function tuple_tfunc(๐•ƒ::AbstractLattice, argtypes::Vector{Any}) end @nospecs function memoryrefget_tfunc(๐•ƒ::AbstractLattice, mem, order, boundscheck) - return _memoryrefget_tfunc(๐•ƒ, mem, order, boundscheck) -end -@nospecs function _memoryrefget_tfunc(๐•ƒ::AbstractLattice, mem, order, boundscheck) memoryref_builtin_common_errorcheck(mem, order, boundscheck) || return Bottom return memoryref_elemtype(mem) end -add_tfunc(memoryrefget, 3, 3, memoryrefget_tfunc, 20) - @nospecs function memoryrefset!_tfunc(๐•ƒ::AbstractLattice, mem, item, order, boundscheck) - hasintersect(widenconst(item), _memoryrefget_tfunc(๐•ƒ, mem, order, boundscheck)) || return Bottom - return mem + hasintersect(widenconst(item), memoryrefget_tfunc(๐•ƒ, mem, order, boundscheck)) || return Bottom + return item +end +@nospecs function memoryrefswap!_tfunc(๐•ƒ::AbstractLattice, mem, v, order, boundscheck) + memoryrefset!_tfunc(๐•ƒ, mem, v, order, boundscheck) === Bottom && return Bottom + return memoryrefget_tfunc(๐•ƒ, mem, order, boundscheck) +end +@nospecs function memoryrefmodify!_tfunc(๐•ƒ::AbstractLattice, mem, op, v, order, boundscheck) + memoryrefget_tfunc(๐•ƒ, mem, order, boundscheck) === Bottom && return Bottom + T = _memoryref_elemtype(mem) + T === Bottom && return Bottom + PT = Const(Pair) + return instanceof_tfunc(apply_type_tfunc(๐•ƒ, PT, T, T), true)[1] +end +@nospecs function memoryrefreplace!_tfunc(๐•ƒ::AbstractLattice, mem, x, v, success_order, failure_order, boundscheck) + memoryrefset!_tfunc(๐•ƒ, mem, v, success_order, boundscheck) === Bottom && return Bottom + hasintersect(widenconst(failure_order), Symbol) || return Bottom + T = _memoryref_elemtype(mem) + T === Bottom && return Bottom + PT = Const(ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T) + return instanceof_tfunc(apply_type_tfunc(๐•ƒ, PT, T), true)[1] +end +@nospecs function memoryrefsetonce!_tfunc(๐•ƒ::AbstractLattice, mem, v, success_order, failure_order, boundscheck) + memoryrefset!_tfunc(๐•ƒ, mem, v, success_order, boundscheck) === Bottom && return Bottom + hasintersect(widenconst(failure_order), Symbol) || return Bottom + return Bool end -add_tfunc(memoryrefset!, 4, 4, memoryrefset!_tfunc, 20) + +add_tfunc(Core.memoryrefget, 3, 3, memoryrefget_tfunc, 20) +add_tfunc(Core.memoryrefset!, 4, 4, memoryrefset!_tfunc, 20) +add_tfunc(Core.memoryrefswap!, 4, 4, memoryrefswap!_tfunc, 20) +add_tfunc(Core.memoryrefmodify!, 5, 5, memoryrefmodify!_tfunc, 20) +add_tfunc(Core.memoryrefreplace!, 6, 6, memoryrefreplace!_tfunc, 20) +add_tfunc(Core.memoryrefsetonce!, 5, 5, memoryrefsetonce!_tfunc, 20) + @nospecs function memoryref_isassigned_tfunc(๐•ƒ::AbstractLattice, mem, order, boundscheck) return _memoryref_isassigned_tfunc(๐•ƒ, mem, order, boundscheck) @@ -2039,7 +2101,7 @@ add_tfunc(memoryrefoffset, 1, 1, memoryrefoffset_tfunc, 5) return true end -function memoryref_elemtype(@nospecialize mem) +@nospecs function memoryref_elemtype(@nospecialize mem) m = widenconst(mem) if !has_free_typevars(m) && m <: GenericMemoryRef m0 = m @@ -2055,6 +2117,23 @@ function memoryref_elemtype(@nospecialize mem) return Any end +@nospecs function _memoryref_elemtype(@nospecialize mem) + m = widenconst(mem) + if !has_free_typevars(m) && m <: GenericMemoryRef + m0 = m + if isa(m, UnionAll) + m = unwrap_unionall(m0) + end + if isa(m, DataType) + T = m.parameters[2] + valid_as_lattice(T, true) || return Bottom + has_free_typevars(T) || return Const(T) + return rewrap_unionall(Type{T}, m0) + end + end + return Type +end + @nospecs function opaque_closure_tfunc(๐•ƒ::AbstractLattice, arg, lb, ub, source, env::Vector{Any}, linfo::MethodInstance) argt, argt_exact = instanceof_tfunc(arg) lbt, lb_exact = instanceof_tfunc(lb) @@ -3059,6 +3138,8 @@ end elseif !(hasintersect(widenconst(M), Module) && hasintersect(widenconst(s), Symbol)) return Bottom end + T = get_binding_type_tfunc(๐•ƒ, M, s) + T isa Const && return T.val return Any end @nospecs function setglobal!_tfunc(๐•ƒ::AbstractLattice, M, s, v, order=Symbol) @@ -3067,8 +3148,39 @@ end end return v end -add_tfunc(getglobal, 2, 3, getglobal_tfunc, 1) -add_tfunc(setglobal!, 3, 4, setglobal!_tfunc, 3) +@nospecs function swapglobal!_tfunc(๐•ƒ::AbstractLattice, M, s, v, order=Symbol) + setglobal!_tfunc(๐•ƒ, M, s, v) === Bottom && return Bottom + return getglobal_tfunc(๐•ƒ, M, s) +end +@nospecs function modifyglobal!_tfunc(๐•ƒ::AbstractLattice, M, s, op, v, order=Symbol) + T = get_binding_type_tfunc(๐•ƒ, M, s) + T === Bottom && return Bottom + T isa Const || return Pair + T = T.val + return Pair{T, T} +end +@nospecs function replaceglobal!_tfunc(๐•ƒ::AbstractLattice, M, s, x, v, success_order=Symbol, failure_order=Symbol) + v = setglobal!_tfunc(๐•ƒ, M, s, v) + v === Bottom && return Bottom + T = get_binding_type_tfunc(๐•ƒ, M, s) + T === Bottom && return Bottom + T isa Const || return ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T + T = T.val + return ccall(:jl_apply_cmpswap_type, Any, (Any,), T) +end +@nospecs function setglobalonce!_tfunc(๐•ƒ::AbstractLattice, M, s, v, success_order=Symbol, failure_order=Symbol) + setglobal!_tfunc(๐•ƒ, M, s, v) === Bottom && return Bottom + return Bool +end + +add_tfunc(Core.getglobal, 2, 3, getglobal_tfunc, 1) +add_tfunc(Core.setglobal!, 3, 4, setglobal!_tfunc, 3) +add_tfunc(Core.swapglobal!, 3, 4, swapglobal!_tfunc, 3) +add_tfunc(Core.modifyglobal!, 4, 5, modifyglobal!_tfunc, 3) +add_tfunc(Core.replaceglobal!, 4, 6, replaceglobal!_tfunc, 3) +add_tfunc(Core.setglobalonce!, 3, 5, setglobalonce!_tfunc, 3) + + @nospecs function setglobal!_nothrow(M, s, newty, o) global_order_nothrow(o, #=loading=#false, #=storing=#true) || return false return setglobal!_nothrow(M, s, newty) diff --git a/base/deepcopy.jl b/base/deepcopy.jl index d2e675dc53bf5..bee1f7994a150 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -115,7 +115,7 @@ function _deepcopy_memory_t(@nospecialize(x::Memory), T, stackdict::IdDict) xi = deepcopy_internal(xi, stackdict)::typeof(xi) end di = Core.memoryref(dr, i, false) - di = Core.memoryrefset!(di, xi, :not_atomic, false) + Core.memoryrefset!(di, xi, :not_atomic, false) end end return dest diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 896cbf30b6520..383ae2c683182 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# module BaseDocs @@ -2235,11 +2236,14 @@ setfield! swapfield!(value, name::Symbol, x, [order::Symbol]) swapfield!(value, i::Int, x, [order::Symbol]) -These atomically perform the operations to simultaneously get and set a field: +Atomically perform the operations to simultaneously get and set a field: y = getfield(value, name) setfield!(value, name, x) return y + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. """ swapfield! @@ -2247,7 +2251,7 @@ swapfield! modifyfield!(value, name::Symbol, op, x, [order::Symbol]) -> Pair modifyfield!(value, i::Int, op, x, [order::Symbol]) -> Pair -These atomically perform the operations to get and set a field after applying +Atomically perform the operations to get and set a field after applying the function `op`. y = getfield(value, name) @@ -2257,6 +2261,9 @@ the function `op`. If supported by the hardware (for example, atomic increment), this may be optimized to the appropriate hardware instruction, otherwise it'll use a loop. + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. """ modifyfield! @@ -2266,7 +2273,7 @@ modifyfield! replacefield!(value, i::Int, expected, desired, [success_order::Symbol, [fail_order::Symbol=success_order]) -> (; old, success::Bool) -These atomically perform the operations to get and conditionally set a field to +Atomically perform the operations to get and conditionally set a field to a given value. y = getfield(value, name, fail_order) @@ -2278,9 +2285,30 @@ a given value. If supported by the hardware, this may be optimized to the appropriate hardware instruction, otherwise it'll use a loop. + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. """ replacefield! +""" + setfieldonce!(value, name::Union{Int,Symbol}, desired, + [success_order::Symbol, [fail_order::Symbol=success_order]) -> success::Bool + +Atomically perform the operations to set a field to +a given value, only if it was previously not set. + + ok = !isdefined(value, name, fail_order) + if ok + setfield!(value, name, desired, success_order) + end + return ok + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. +""" +setfieldonce! + """ getglobal(module::Module, name::Symbol, [order::Symbol=:monotonic]) @@ -2321,6 +2349,7 @@ julia> getglobal(M, :a) """ getglobal + """ setglobal!(module::Module, name::Symbol, x, [order::Symbol=:monotonic]) @@ -2362,6 +2391,9 @@ setglobal! Core.get_binding_type(module::Module, name::Symbol) Retrieve the declared type of the binding `name` from the module `module`. + +!!! compat "Julia 1.9" + This function requires Julia 1.9 or later. """ Core.get_binding_type @@ -2371,9 +2403,65 @@ Core.get_binding_type Set the declared type of the binding `name` in the module `module` to `type`. Error if the binding already has a type that is not equivalent to `type`. If the `type` argument is absent, set the binding type to `Any` if unset, but do not error. + +!!! compat "Julia 1.9" + This function requires Julia 1.9 or later. """ Core.set_binding_type! +""" + swapglobal!(module::Module, name::Symbol, x, [order::Symbol=:monotonic]) + +Atomically perform the operations to simultaneously get and set a global. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`swapproperty!`](@ref Base.swapproperty!) and [`setglobal!`](@ref). +""" +swapglobal! + +""" + modifyglobal!(module::Module, name::Symbol, op, x, [order::Symbol=:monotonic]) -> Pair + +Atomically perform the operations to get and set a global after applying +the function `op`. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`modifyproperty!`](@ref Base.modifyproperty!) and [`setglobal!`](@ref). +""" +modifyglobal! + +""" + replaceglobal!(module::Module, name::Symbol, expected, desired, + [success_order::Symbol, [fail_order::Symbol=success_order]) -> (; old, success::Bool) + +Atomically perform the operations to get and conditionally set a global to +a given value. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`replaceproperty!`](@ref Base.replaceproperty!) and [`setglobal!`](@ref). +""" +replaceglobal! + +""" + setglobalonce!(module::Module, name::Symbol, value, + [success_order::Symbol, [fail_order::Symbol=success_order]) -> success::Bool + +Atomically perform the operations to set a global to +a given value, only if it was previously not set. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`setpropertyonce!`](@ref Base.setpropertyonce!) and [`setglobal!`](@ref). +""" +setglobalonce! + """ typeof(x) @@ -3177,14 +3265,30 @@ Base.modifyproperty! replaceproperty!(x, f::Symbol, expected, desired, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) Perform a compare-and-swap operation on `x.f` from `expected` to `desired`, per -egal. The syntax `@atomic_replace! x.f expected => desired` can be used instead +egal. The syntax `@atomicreplace x.f expected => desired` can be used instead of the function call form. See also [`replacefield!`](@ref Core.replacefield!) -and [`setproperty!`](@ref Base.setproperty!). +[`setproperty!`](@ref Base.setproperty!), +[`setpropertyonce!`](@ref Base.setpropertyonce!). """ Base.replaceproperty! +""" + setpropertyonce!(x, f::Symbol, value, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) + +Perform a compare-and-swap operation on `x.f` to set it to `value` if previously unset. +The syntax `@atomiconce x.f = value` can be used instead of the function call form. + +See also [`setfieldonce!`](@ref Core.replacefield!), +[`setproperty!`](@ref Base.setproperty!), +[`replaceproperty!`](@ref Base.replaceproperty!). + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. +""" +Base.setpropertyonce! + """ StridedArray{T, N} diff --git a/base/docs/intrinsicsdocs.jl b/base/docs/intrinsicsdocs.jl index 9f6ec773ff9a8..ca06ad678bdbf 100644 --- a/base/docs/intrinsicsdocs.jl +++ b/base/docs/intrinsicsdocs.jl @@ -27,6 +27,9 @@ Core.Intrinsics Core.memoryref(::GenericMemoryRef, index::Int, [boundscheck::Bool]) Return a `GenericMemoryRef` for a `GenericMemory`. See [`MemoryRef`](@ref). + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. """ Core.memoryref @@ -34,6 +37,9 @@ Core.memoryref Core..memoryrefoffset(::GenericMemoryRef) Return the offset index that was used to construct the `MemoryRef`. See [`Core.memoryref`](@ref). + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. """ Core.memoryrefoffset @@ -42,6 +48,9 @@ Core.memoryrefoffset Return the value stored at the `MemoryRef`, throwing a `BoundsError` if the `Memory` is empty. See `ref[]`. The memory ordering specified must be compatible with the `isatomic` parameter. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. """ Core.memoryrefget @@ -50,6 +59,9 @@ Core.memoryrefget Store the value to the `MemoryRef`, throwing a `BoundsError` if the `Memory` is empty. See `ref[] = value`. The memory ordering specified must be compatible with the `isatomic` parameter. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. """ Core.memoryrefset! @@ -59,5 +71,110 @@ Core.memoryrefset! Return whether there is a value stored at the `MemoryRef`, returning false if the `Memory` is empty. See [`isassigned(::Base.RefValue)`](@ref), [`Core.memoryrefget`](@ref). The memory ordering specified must be compatible with the `isatomic` parameter. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. """ Core.memoryref_isassigned + +""" + Core.memoryrefswap!(::GenericMemoryRef, value, ordering::Symbol, boundscheck::Bool) + +Atomically perform the operations to simultaneously get and set a `MemoryRef` value. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`swapproperty!`](@ref Base.swapproperty!) and [`Core.memoryrefset!`](@ref). +""" +Core.memoryrefswap! + +""" + Core.memoryrefmodify!(::GenericMemoryRef, op, value, ordering::Symbol, boundscheck::Bool) -> Pair + +Atomically perform the operations to get and set a `MemoryRef` value after applying +the function `op`. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`modifyproperty!`](@ref Base.modifyproperty!) and [`Core.memoryrefset!`](@ref). +""" +Core.memoryrefmodify! + +""" + Core.memoryrefreplace!(::GenericMemoryRef, expected, desired, + success_order::Symbol, fail_order::Symbol=success_order, boundscheck::Bool) -> (; old, success::Bool) + +Atomically perform the operations to get and conditionally set a `MemoryRef` value. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`replaceproperty!`](@ref Base.replaceproperty!) and [`Core.memoryrefset!`](@ref). +""" +Core.memoryrefreplace! + +""" + Core.memoryrefsetonce!(::GenericMemoryRef, value, + success_order::Symbol, fail_order::Symbol=success_order, boundscheck::Bool) -> success::Bool + +Atomically perform the operations to set a `MemoryRef` to +a given value, only if it was previously not set. + +!!! compat "Julia 1.11" + This function requires Julia 1.11 or later. + +See also [`setpropertyonce!`](@ref Base.replaceproperty!) and [`Core.memoryrefset!`](@ref). +""" +Core.memoryrefsetonce! + +""" + Core.Intrinsics.atomic_pointerref(pointer::Ptr{T}, order::Symbol) --> T + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. + +See [`unsafe_load`](@ref Base.unsafe_load). +""" +Core.Intrinsics.atomic_pointerref + +""" + Core.Intrinsics.atomic_pointerset(pointer::Ptr{T}, new::T, order::Symbol) --> pointer + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. + +See [`unsafe_store!`](@ref Base.unsafe_store!). +""" +Core.Intrinsics.atomic_pointerset + +""" + Core.Intrinsics.atomic_pointerswap(pointer::Ptr{T}, new::T, order::Symbol) --> old + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. + +See [`unsafe_swap!`](@ref Base.unsafe_swap!). +""" +Core.Intrinsics.atomic_pointerswap + +""" + Core.Intrinsics.atomic_pointermodify(pointer::Ptr{T}, function::(old::T,arg::S)->T, arg::S, order::Symbol) --> old + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. + +See [`unsafe_modify!`](@ref Base.unsafe_modify!). +""" +Core.Intrinsics.atomic_pointermodify + +""" + Core.Intrinsics.atomic_pointerreplace(pointer::Ptr{T}, expected::Any, new::T, success_order::Symbol, failure_order::Symbol) --> (old, cmp) + +!!! compat "Julia 1.7" + This function requires Julia 1.7 or later. + +See [`unsafe_replace!`](@ref Base.unsafe_replace!). +""" +Core.Intrinsics.atomic_pointerreplace diff --git a/base/essentials.jl b/base/essentials.jl index 51f3ac43f3a14..8e329c4f7c4d9 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -366,9 +366,14 @@ macro _nospecializeinfer_meta() return Expr(:meta, :nospecializeinfer) end -getindex(A::GenericMemory{:not_atomic}, i::Int) = (@_noub_if_noinbounds_meta; - memoryrefget(memoryref(memoryref(A), i, @_boundscheck), :not_atomic, false)) -getindex(A::GenericMemoryRef{:not_atomic}) = memoryrefget(A, :not_atomic, @_boundscheck) +default_access_order(a::GenericMemory{:not_atomic}) = :not_atomic +default_access_order(a::GenericMemory{:atomic}) = :monotonic +default_access_order(a::GenericMemoryRef{:not_atomic}) = :not_atomic +default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic + +getindex(A::GenericMemory, i::Int) = (@_noub_if_noinbounds_meta; + memoryrefget(memoryref(memoryref(A), i, @_boundscheck), default_access_order(A), false)) +getindex(A::GenericMemoryRef) = memoryrefget(A, default_access_order(A), @_boundscheck) function iterate end @@ -889,8 +894,8 @@ function setindex!(A::Array{Any}, @nospecialize(x), i::Int) return A end setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryref(memoryref(A), i, @_boundscheck), x, :not_atomic, @_boundscheck); A) -setindex!(A::MemoryRef{T}, x) where {T} = memoryrefset!(A, convert(T, x), :not_atomic, @_boundscheck) -setindex!(A::MemoryRef{Any}, @nospecialize(x)) = memoryrefset!(A, x, :not_atomic, @_boundscheck) +setindex!(A::MemoryRef{T}, x) where {T} = (memoryrefset!(A, convert(T, x), :not_atomic, @_boundscheck); A) +setindex!(A::MemoryRef{Any}, @nospecialize(x)) = (memoryrefset!(A, x, :not_atomic, @_boundscheck); A) # SimpleVector diff --git a/base/exports.jl b/base/exports.jl index a9a753d0fda52..6b783eff9179b 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -756,6 +756,7 @@ export swapproperty!, modifyproperty!, replaceproperty!, + setpropertyonce!, fieldoffset, fieldname, fieldnames, @@ -1065,6 +1066,7 @@ export @atomic, @atomicswap, @atomicreplace, + @atomiconce, @__dot__, @enum, @label, diff --git a/base/expr.jl b/base/expr.jl index ab4314fbd6cc8..f6ca12f07a63d 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1333,3 +1333,60 @@ function make_atomicreplace(success_order, fail_order, ex, old_new) return :(replaceproperty!($ll, $lr, $old_new::Pair..., $success_order, $fail_order)) end end + +""" + @atomiconce a.b.x = value + @atomiconce :sequentially_consistent a.b.x = value + @atomiconce :sequentially_consistent :monotonic a.b.x = value + +Perform the conditional assignment of value atomically if it was previously +unset, returning the value `success::Bool`. Where `success` indicates whether +the assignment was completed. + +This operation translates to a `setpropertyonce!(a.b, :x, value)` call. + +See [Per-field atomics](@ref man-atomics) section in the manual for more details. + +# Examples +```jldoctest +julia> mutable struct AtomicOnce + @atomic x + AtomicOnce() = new() + end + +julia> a = AtomicOnce() +AtomicOnce(#undef) + +julia> @atomiconce a.x = 1 # set field x of a to 1, if unset, with sequential consistency +true + +julia> @atomic a.x # fetch field x of a, with sequential consistency +1 + +julia> @atomiconce a.x = 1 # set field x of a to 1, if unset, with sequential consistency +false +``` + +!!! compat "Julia 1.11" + This functionality requires at least Julia 1.11. +""" +macro atomiconce(success_order, fail_order, ex) + fail_order isa QuoteNode || (fail_order = esc(fail_order)) + success_order isa QuoteNode || (success_order = esc(success_order)) + return make_atomiconce(success_order, fail_order, ex) +end +macro atomiconce(order, ex) + order isa QuoteNode || (order = esc(order)) + return make_atomiconce(order, order, ex) +end +macro atomiconce(ex) + return make_atomiconce(QuoteNode(:sequentially_consistent), QuoteNode(:sequentially_consistent), ex) +end +function make_atomiconce(success_order, fail_order, ex) + @nospecialize + is_expr(ex, :(=), 2) || error("@atomiconce expression missing assignment") + l, val = ex.args[1], esc(ex.args[2]) + is_expr(l, :., 2) || error("@atomiconce expression missing field access") + ll, lr = esc(l.args[1]), esc(l.args[2]) + return :(setpropertyonce!($ll, $lr, $val, $success_order, $fail_order)) +end diff --git a/base/genericmemory.jl b/base/genericmemory.jl index 13f54216581df..b5f519a0f854d 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -3,18 +3,36 @@ ## genericmemory.jl: Managed Memory """ - GenericMemory{kind::Symbol, T, addrspace::Int} <: AbstractVector{T} + GenericMemory{kind::Symbol, T, addrspace=Core.CPU} <: AbstractVector{T} One-dimensional dense array with elements of type `T`. + +!!! compat "Julia 1.11" + This type requires Julia 1.11 or later. """ GenericMemory + """ Memory{T} == GenericMemory{:not_atomic, T, Core.CPU} One-dimensional dense array with elements of type `T`. + +!!! compat "Julia 1.11" + This type requires Julia 1.11 or later. """ Memory +""" + AtomicMemory{T} == GenericMemory{:atomic, T, Core.CPU} + +One-dimensional dense array with elements of type `T`, where each element is +independently atomic when accessed, and cannot be set non-atomically. + +!!! compat "Julia 1.11" + This type requires Julia 1.11 or later. +""" +AtomicMemory + ## Basic functions ## using Core: memoryrefoffset, memoryref_isassigned # import more functions which were not essential @@ -48,6 +66,7 @@ function _unsetindex!(A::MemoryRef{T}) where T elseif arrayelem != isunion if !datatype_pointerfree(T::DataType) for j = 1:Core.sizeof(Ptr{Cvoid}):elsz + # XXX: this violates memory ordering, since it writes more than one C_NULL to each Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic) end end @@ -56,17 +75,17 @@ function _unsetindex!(A::MemoryRef{T}) where T return A end -elsize(@nospecialize _::Type{A}) where {T,A<:GenericMemory{<:Any,T}} = aligned_sizeof(T) +elsize(@nospecialize _::Type{A}) where {T,A<:GenericMemory{<:Any,T}} = aligned_sizeof(T) # XXX: probably supposed to be the stride? sizeof(a::GenericMemory) = Core.sizeof(a) # multi arg case will be overwritten later. This is needed for bootstrapping -function isassigned(a::Memory, i::Int) +function isassigned(a::GenericMemory, i::Int) @inline @boundscheck (i - 1)%UInt < length(a)%UInt || return false - return @inbounds memoryref_isassigned(GenericMemoryRef(a, i), :not_atomic, false) + return @inbounds memoryref_isassigned(GenericMemoryRef(a, i), default_access_order(a), false) end -isassigned(a::GenericMemoryRef) = memoryref_isassigned(a, :not_atomic, @_boundscheck) +isassigned(a::GenericMemoryRef) = memoryref_isassigned(a, default_access_order(a), @_boundscheck) ## copy ## function unsafe_copyto!(dest::MemoryRef{T}, src::MemoryRef{T}, n) where {T} @@ -128,11 +147,16 @@ end ## Constructors ## -similar(a::Memory{T}) where {T} = Memory{T}(undef, length(a)) -similar(a::Memory{T}, S::Type) where {T} = Memory{S}(undef, length(a)) -similar(a::Memory{T}, m::Int) where {T} = Memory{T}(undef, m) -similar(a::Memory, T::Type, dims::Dims{1}) = Memory{T}(undef, dims[1]) -similar(a::Memory{T}, dims::Dims{1}) where {T} = Memory{T}(undef, dims[1]) +similar(a::GenericMemory) = + typeof(a)(undef, length(a)) +similar(a::GenericMemory{kind,<:Any,AS}, T::Type) where {kind,AS} = + GenericMemory{kind,T,AS}(undef, length(a)) +similar(a::GenericMemory, m::Int) = + typeof(a)(undef, m) +similar(a::GenericMemory{kind,<:Any,AS}, T::Type, dims::Dims{1}) where {kind,AS} = + GenericMemory{kind,T,AS}(undef, dims[1]) +similar(a::GenericMemory, dims::Dims{1}) = + typeof(a)(undef, dims[1]) function fill!(a::Union{Memory{UInt8}, Memory{Int8}}, x::Integer) t = @_gc_preserve_begin a @@ -145,10 +169,9 @@ end ## Conversions ## -convert(::Type{T}, a::AbstractArray) where {T<:GenericMemory} = a isa T ? a : T(a)::T +convert(::Type{T}, a::AbstractArray) where {T<:Memory} = a isa T ? a : T(a)::T promote_rule(a::Type{Memory{T}}, b::Type{Memory{S}}) where {T,S} = el_same(promote_type(T,S), a, b) -promote_rule(a::Type{GenericMemory{:atomic,T,Core.CPU}}, b::Type{GenericMemory{:atomic,S,Core.CPU}}) where {T,S} = el_same(promote_type(T,S), a, b) ## Constructors ## diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 0d1692b0ed581..64c635ed3043b 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -1,4 +1,4 @@ -# Essentials +#replacefield Essentials ## Introduction @@ -148,6 +148,7 @@ Base.setproperty! Base.replaceproperty! Base.swapproperty! Base.modifyproperty! +Base.setpropertyonce! Base.propertynames Base.hasproperty Core.getfield @@ -155,9 +156,8 @@ Core.setfield! Core.modifyfield! Core.replacefield! Core.swapfield! +Core.setfieldonce! Core.isdefined -Core.getglobal -Core.setglobal! Base.@isdefined Base.convert Base.promote @@ -461,8 +461,15 @@ Base.nameof(::Function) Base.functionloc(::Any, ::Any) Base.functionloc(::Method) Base.@locals +Core.getglobal +Core.setglobal! +Core.modifyglobal! +Core.swapglobal! +Core.setglobalonce! +Core.replaceglobal! ``` + ## Documentation (See also the [documentation](@ref man-documentation) chapter.) ```@docs diff --git a/doc/src/base/multi-threading.md b/doc/src/base/multi-threading.md index 45a60b14d541a..9e3bc49acf6dc 100644 --- a/doc/src/base/multi-threading.md +++ b/doc/src/base/multi-threading.md @@ -25,19 +25,13 @@ atomic Base.@atomic Base.@atomicswap Base.@atomicreplace +Base.@atomiconce +Base.AtomicMemory ``` -!!! note - - The following APIs are fairly primitive, and will likely be exposed through an `unsafe_*`-like wrapper. - -``` -Core.Intrinsics.atomic_pointerref(pointer::Ptr{T}, order::Symbol) --> T -Core.Intrinsics.atomic_pointerset(pointer::Ptr{T}, new::T, order::Symbol) --> pointer -Core.Intrinsics.atomic_pointerswap(pointer::Ptr{T}, new::T, order::Symbol) --> old -Core.Intrinsics.atomic_pointermodify(pointer::Ptr{T}, function::(old::T,arg::S)->T, arg::S, order::Symbol) --> old -Core.Intrinsics.atomic_pointerreplace(pointer::Ptr{T}, expected::Any, new::T, success_order::Symbol, failure_order::Symbol) --> (old, cmp) -``` +There are also optional memory ordering parameters for the `unsafe` set of functions, that +select the C/C++-compatible versions of these atomic operations, if that parameter is specified to +[`unsafe_load`](@ref), [`unsafe_store!`](@ref), [`unsafe_swap!`](@ref), [`unsafe_replace!`](@ref), and [`unsafe_modify!`](@ref). !!! warning diff --git a/doc/src/devdocs/builtins.md b/doc/src/devdocs/builtins.md index 02c64f4839fd7..5ce7fae16f488 100644 --- a/doc/src/devdocs/builtins.md +++ b/doc/src/devdocs/builtins.md @@ -12,6 +12,15 @@ Core.memoryrefoffset Core.memoryrefget Core.memoryrefset! Core.memoryref_isassigned +Core.memoryrefswap! +Core.memoryrefmodify! +Core.memoryrefreplace! +Core.memoryrefsetonce! +Core.Intrinsics.atomic_pointerref +Core.Intrinsics.atomic_pointerset +Core.Intrinsics.atomic_pointerswap +Core.Intrinsics.atomic_pointermodify +Core.Intrinsics.atomic_pointerreplace Core.get_binding_type Core.set_binding_type! Core.IntrinsicFunction diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index 71e49e65b19a8..085c6b835c87c 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -385,8 +385,9 @@ julia> acc[] #### [Per-field atomics](@id man-atomics) We can also use atomics on a more granular level using the [`@atomic`](@ref -Base.@atomic), [`@atomicswap`](@ref Base.@atomicswap), and -[`@atomicreplace`](@ref Base.@atomicreplace) macros. +Base.@atomic), [`@atomicswap`](@ref Base.@atomicswap), +[`@atomicreplace`](@ref Base.@atomicreplace) macros, and +[`@atomiconce`](@ref Base.@atomiconce) macros. Specific details of the memory model and other details of the design are written in the [Julia Atomics diff --git a/src/builtin_proto.h b/src/builtin_proto.h index a009b535ac951..b0536bef24e27 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -21,48 +21,57 @@ extern "C" { JL_DLLEXPORT extern jl_fptr_args_t jl_f_##name##_addr #endif -DECLARE_BUILTIN(applicable); DECLARE_BUILTIN(_apply_iterate); DECLARE_BUILTIN(_apply_pure); -DECLARE_BUILTIN(apply_type); -DECLARE_BUILTIN(memoryref); -DECLARE_BUILTIN(memoryrefoffset); -DECLARE_BUILTIN(memoryrefget); -DECLARE_BUILTIN(memoryrefset); -DECLARE_BUILTIN(memoryref_isassigned); DECLARE_BUILTIN(_call_in_world); DECLARE_BUILTIN(_call_in_world_total); DECLARE_BUILTIN(_call_latest); -DECLARE_BUILTIN(replacefield); +DECLARE_BUILTIN(_compute_sparams); DECLARE_BUILTIN(_expr); +DECLARE_BUILTIN(_svec_ref); +DECLARE_BUILTIN(_typebody); +DECLARE_BUILTIN(_typevar); +DECLARE_BUILTIN(applicable); +DECLARE_BUILTIN(apply_type); +DECLARE_BUILTIN(compilerbarrier); +DECLARE_BUILTIN(current_scope); +DECLARE_BUILTIN(donotdelete); DECLARE_BUILTIN(fieldtype); +DECLARE_BUILTIN(finalizer); DECLARE_BUILTIN(getfield); +DECLARE_BUILTIN(getglobal); DECLARE_BUILTIN(ifelse); DECLARE_BUILTIN(invoke); DECLARE_BUILTIN(is); DECLARE_BUILTIN(isa); DECLARE_BUILTIN(isdefined); DECLARE_BUILTIN(issubtype); +DECLARE_BUILTIN(memoryref); +DECLARE_BUILTIN(memoryref_isassigned); +DECLARE_BUILTIN(memoryrefget); +DECLARE_BUILTIN(memoryrefmodify); +DECLARE_BUILTIN(memoryrefoffset); +DECLARE_BUILTIN(memoryrefreplace); +DECLARE_BUILTIN(memoryrefset); +DECLARE_BUILTIN(memoryrefsetonce); +DECLARE_BUILTIN(memoryrefswap); DECLARE_BUILTIN(modifyfield); +DECLARE_BUILTIN(modifyglobal); DECLARE_BUILTIN(nfields); +DECLARE_BUILTIN(replacefield); +DECLARE_BUILTIN(replaceglobal); DECLARE_BUILTIN(setfield); +DECLARE_BUILTIN(setfieldonce); +DECLARE_BUILTIN(setglobal); +DECLARE_BUILTIN(setglobalonce); DECLARE_BUILTIN(sizeof); DECLARE_BUILTIN(svec); DECLARE_BUILTIN(swapfield); +DECLARE_BUILTIN(swapglobal); DECLARE_BUILTIN(throw); DECLARE_BUILTIN(tuple); DECLARE_BUILTIN(typeassert); -DECLARE_BUILTIN(_typebody); DECLARE_BUILTIN(typeof); -DECLARE_BUILTIN(_typevar); -DECLARE_BUILTIN(donotdelete); -DECLARE_BUILTIN(compilerbarrier); -DECLARE_BUILTIN(getglobal); -DECLARE_BUILTIN(setglobal); -DECLARE_BUILTIN(finalizer); -DECLARE_BUILTIN(_compute_sparams); -DECLARE_BUILTIN(_svec_ref); -DECLARE_BUILTIN(current_scope); JL_CALLABLE(jl_f__structtype); JL_CALLABLE(jl_f__abstracttype); diff --git a/src/builtins.c b/src/builtins.c index 3c4e8215fc572..412ccaf8bab04 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -626,7 +626,7 @@ STATIC_INLINE void _grow_to(jl_value_t **root, jl_value_t ***oldargs, jl_svec_t static jl_value_t *jl_arrayref(jl_array_t *a, size_t i) { - return jl_memoryrefget(jl_memoryrefindex(a->ref, i)); + return jl_memoryrefget(jl_memoryrefindex(a->ref, i), 0); } static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *iterate) @@ -1035,9 +1035,11 @@ JL_CALLABLE(jl_f_getfield) jl_atomic_error("getfield: non-atomic field cannot be accessed atomically"); if (isatomic && order == jl_memory_order_notatomic) jl_atomic_error("getfield: atomic field cannot be accessed non-atomically"); - v = jl_get_nth_field_checked(v, idx); - if (order >= jl_memory_order_acq_rel || order == jl_memory_order_acquire) - jl_fence(); // `v` already had at least consume ordering + if (order >= jl_memory_order_seq_cst) + jl_fence(); + v = jl_get_nth_field_checked(v, idx); // `v` already had at least consume ordering + if (order >= jl_memory_order_acquire) + jl_fence(); return v; } @@ -1059,7 +1061,7 @@ JL_CALLABLE(jl_f_setfield) jl_value_t *ft = jl_field_type_concrete(st, idx); if (!jl_isa(args[2], ft)) jl_type_error("setfield!", ft, args[2]); - if (order >= jl_memory_order_acq_rel || order == jl_memory_order_release) + if (order >= jl_memory_order_release) jl_fence(); // `st->[idx]` will have at least relaxed ordering set_nth_field(st, v, idx, args[2], isatomic); return args[2]; @@ -1133,6 +1135,35 @@ JL_CALLABLE(jl_f_replacefield) return v; } +JL_CALLABLE(jl_f_setfieldonce) +{ + enum jl_memory_order success_order = jl_memory_order_notatomic; + JL_NARGS(setfieldonce!, 3, 5); + if (nargs >= 4) { + JL_TYPECHK(setfieldonce!, symbol, args[3]); + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + } + enum jl_memory_order failure_order = success_order; + if (nargs == 5) { + JL_TYPECHK(setfieldonce!, symbol, args[4]); + failure_order = jl_get_atomic_order_checked((jl_sym_t*)args[4], 1, 0); + } + if (failure_order > success_order) + jl_atomic_error("invalid atomic ordering"); + // TODO: filter more invalid ordering combinations? + jl_value_t *v = args[0]; + jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); + size_t idx = get_checked_fieldindex("setfieldonce!", st, v, args[1], 1); + int isatomic = !!jl_field_isatomic(st, idx); + if (isatomic == (success_order == jl_memory_order_notatomic)) + jl_atomic_error(isatomic ? "setfieldonce!: atomic field cannot be written non-atomically" + : "setfieldonce!: non-atomic field cannot be written atomically"); + if (isatomic == (failure_order == jl_memory_order_notatomic)) + jl_atomic_error(isatomic ? "setfieldonce!: atomic field cannot be accessed non-atomically" + : "setfieldonce!: non-atomic field cannot be accessed atomically"); + int success = set_nth_fieldonce(st, v, idx, args[2], isatomic); // always seq_cst, if isatomic needed at all + return success ? jl_true : jl_false; +} static jl_value_t *get_fieldtype(jl_value_t *t, jl_value_t *f, int dothrow) { @@ -1241,7 +1272,12 @@ JL_CALLABLE(jl_f_isdefined) JL_TYPECHK(isdefined, symbol, args[1]); m = (jl_module_t*)args[0]; s = (jl_sym_t*)args[1]; - return jl_boundp(m, s) ? jl_true : jl_false; // is seq_cst already + if (order == jl_memory_order_unspecified) + order = jl_memory_order_unordered; + if (order < jl_memory_order_unordered) + jl_atomic_error("isdefined: module binding cannot be accessed non-atomically"); + int bound = jl_boundp(m, s); // seq_cst always + return bound ? jl_true : jl_false; } jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(args[0]); assert(jl_is_datatype(vt)); @@ -1268,15 +1304,11 @@ JL_CALLABLE(jl_f_isdefined) jl_atomic_error("isdefined: non-atomic field cannot be accessed atomically"); if (isatomic && order == jl_memory_order_notatomic) jl_atomic_error("isdefined: atomic field cannot be accessed non-atomically"); - int v = jl_field_isdefined(args[0], idx); - if (v == 2) { - if (order > jl_memory_order_notatomic) - jl_fence(); // isbits case has no ordering already - } - else { - if (order >= jl_memory_order_acq_rel || order == jl_memory_order_acquire) - jl_fence(); // `v` already gave at least consume ordering - } + if (order >= jl_memory_order_seq_cst) + jl_fence(); + int v = jl_field_isdefined(args[0], idx); // relaxed ordering + if (order >= jl_memory_order_acquire) + jl_fence(); return v ? jl_true : jl_false; } @@ -1297,8 +1329,11 @@ JL_CALLABLE(jl_f_getglobal) JL_TYPECHK(getglobal, symbol, (jl_value_t*)sym); if (order == jl_memory_order_notatomic) jl_atomic_error("getglobal: module binding cannot be read non-atomically"); - jl_value_t *v = jl_eval_global_var(mod, sym); - // is seq_cst already, no fence needed + else if (order >= jl_memory_order_seq_cst) + jl_fence(); + jl_value_t *v = jl_eval_global_var(mod, sym); // relaxed load + if (order >= jl_memory_order_acquire) + jl_fence(); return v; } @@ -1316,9 +1351,12 @@ JL_CALLABLE(jl_f_setglobal) JL_TYPECHK(setglobal!, symbol, (jl_value_t*)var); if (order == jl_memory_order_notatomic) jl_atomic_error("setglobal!: module binding cannot be written non-atomically"); - // is seq_cst already, no fence needed + else if (order >= jl_memory_order_seq_cst) + jl_fence(); jl_binding_t *b = jl_get_binding_wr(mod, var); - jl_checked_assignment(b, mod, var, args[2]); + jl_checked_assignment(b, mod, var, args[2]); // release store + if (order >= jl_memory_order_seq_cst) + jl_fence(); return args[2]; } @@ -1365,6 +1403,104 @@ JL_CALLABLE(jl_f_set_binding_type) return jl_nothing; } +JL_CALLABLE(jl_f_swapglobal) +{ + enum jl_memory_order order = jl_memory_order_release; + JL_NARGS(swapglobal!, 3, 4); + if (nargs == 4) { + JL_TYPECHK(swapglobal!, symbol, args[3]); + order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + } + jl_module_t *mod = (jl_module_t*)args[0]; + jl_sym_t *var = (jl_sym_t*)args[1]; + JL_TYPECHK(swapglobal!, module, (jl_value_t*)mod); + JL_TYPECHK(swapglobal!, symbol, (jl_value_t*)var); + if (order == jl_memory_order_notatomic) + jl_atomic_error("swapglobal!: module binding cannot be written non-atomically"); + // is seq_cst already, no fence needed + jl_binding_t *b = jl_get_binding_wr(mod, var); + return jl_checked_swap(b, mod, var, args[2]); +} + +JL_CALLABLE(jl_f_modifyglobal) +{ + enum jl_memory_order order = jl_memory_order_release; + JL_NARGS(modifyglobal!, 4, 5); + if (nargs == 5) { + JL_TYPECHK(modifyglobal!, symbol, args[4]); + order = jl_get_atomic_order_checked((jl_sym_t*)args[4], 1, 1); + } + jl_module_t *mod = (jl_module_t*)args[0]; + jl_sym_t *var = (jl_sym_t*)args[1]; + JL_TYPECHK(modifyglobal!, module, (jl_value_t*)mod); + JL_TYPECHK(modifyglobal!, symbol, (jl_value_t*)var); + if (order == jl_memory_order_notatomic) + jl_atomic_error("modifyglobal!: module binding cannot be written non-atomically"); + jl_binding_t *b = jl_get_binding_wr(mod, var); + // is seq_cst already, no fence needed + return jl_checked_modify(b, mod, var, args[2], args[3]); +} + +JL_CALLABLE(jl_f_replaceglobal) +{ + enum jl_memory_order success_order = jl_memory_order_release; + JL_NARGS(replaceglobal!, 4, 6); + if (nargs >= 5) { + JL_TYPECHK(replaceglobal!, symbol, args[4]); + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[4], 1, 1); + } + enum jl_memory_order failure_order = success_order; + if (nargs == 6) { + JL_TYPECHK(replaceglobal!, symbol, args[5]); + failure_order = jl_get_atomic_order_checked((jl_sym_t*)args[5], 1, 0); + } + if (failure_order > success_order) + jl_atomic_error("invalid atomic ordering"); + // TODO: filter more invalid ordering combinations? + jl_module_t *mod = (jl_module_t*)args[0]; + jl_sym_t *var = (jl_sym_t*)args[1]; + JL_TYPECHK(replaceglobal!, module, (jl_value_t*)mod); + JL_TYPECHK(replaceglobal!, symbol, (jl_value_t*)var); + if (success_order == jl_memory_order_notatomic) + jl_atomic_error("replaceglobal!: module binding cannot be written non-atomically"); + if (failure_order == jl_memory_order_notatomic) + jl_atomic_error("replaceglobal!: module binding cannot be accessed non-atomically"); + jl_binding_t *b = jl_get_binding_wr(mod, var); + // is seq_cst already, no fence needed + return jl_checked_replace(b, mod, var, args[2], args[3]); +} + +JL_CALLABLE(jl_f_setglobalonce) +{ + enum jl_memory_order success_order = jl_memory_order_release; + JL_NARGS(setglobalonce!, 3, 5); + if (nargs >= 4) { + JL_TYPECHK(setglobalonce!, symbol, args[3]); + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + } + enum jl_memory_order failure_order = success_order; + if (nargs == 5) { + JL_TYPECHK(setglobalonce!, symbol, args[4]); + failure_order = jl_get_atomic_order_checked((jl_sym_t*)args[4], 1, 0); + } + if (failure_order > success_order) + jl_atomic_error("invalid atomic ordering"); + // TODO: filter more invalid ordering combinations? + jl_module_t *mod = (jl_module_t*)args[0]; + jl_sym_t *var = (jl_sym_t*)args[1]; + JL_TYPECHK(setglobalonce!, module, (jl_value_t*)mod); + JL_TYPECHK(setglobalonce!, symbol, (jl_value_t*)var); + if (success_order == jl_memory_order_notatomic) + jl_atomic_error("setglobalonce!: module binding cannot be written non-atomically"); + if (failure_order == jl_memory_order_notatomic) + jl_atomic_error("setglobalonce!: module binding cannot be accessed non-atomically"); + jl_binding_t *b = jl_get_binding_wr(mod, var); + // is seq_cst already, no fence needed + jl_value_t *old = jl_checked_assignonce(b, mod, var, args[2]); + return old == NULL ? jl_true : jl_false; +} + + // apply_type ----------------------------------------------------------------- @@ -1590,51 +1726,197 @@ JL_CALLABLE(jl_f_memoryrefoffset) JL_CALLABLE(jl_f_memoryrefget) { + enum jl_memory_order order = jl_memory_order_notatomic; JL_NARGS(memoryrefget, 3, 3); JL_TYPECHK(memoryrefget, genericmemoryref, args[0]); JL_TYPECHK(memoryrefget, symbol, args[1]); JL_TYPECHK(memoryrefget, bool, args[2]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); - if (isatomic == jl_false) - if (args[1] != (jl_value_t*)jl_not_atomic_sym) - jl_atomic_error("memoryrefget!: non-atomic memory cannot be accessed atomically"); + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[1] != kind) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[1], 1, 0); + jl_atomic_error("memoryrefget: non-atomic memory cannot be accessed atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[1], 1, 0); + if (order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefget: atomic memory cannot be accessed non-atomically"); + } if (m.mem->length == 0) jl_bounds_error_int((jl_value_t*)m.mem, 1); - return jl_memoryrefget(m); + return jl_memoryrefget(m, kind == (jl_value_t*)jl_atomic_sym); } JL_CALLABLE(jl_f_memoryrefset) { + enum jl_memory_order order = jl_memory_order_notatomic; JL_NARGS(memoryrefset!, 4, 4); JL_TYPECHK(memoryrefset!, genericmemoryref, args[0]); JL_TYPECHK(memoryrefset!, symbol, args[2]); JL_TYPECHK(memoryrefset!, bool, args[3]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); - if (isatomic == jl_false) - if (args[2] != (jl_value_t*)jl_not_atomic_sym) + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[2] != kind) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 0, 1); jl_atomic_error("memoryrefset!: non-atomic memory cannot be written atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 0, 1); + if (order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefset!: atomic memory cannot be written non-atomically"); + } if (m.mem->length == 0) jl_bounds_error_int((jl_value_t*)m.mem, 1); - jl_memoryrefset(m, args[1]); - return args[0]; + jl_memoryrefset(m, args[1], kind == (jl_value_t*)jl_atomic_sym); + return args[1]; } JL_CALLABLE(jl_f_memoryref_isassigned) { + enum jl_memory_order order = jl_memory_order_notatomic; JL_NARGS(memoryref_isassigned, 3, 3); JL_TYPECHK(memoryref_isassigned, genericmemoryref, args[0]); JL_TYPECHK(memoryref_isassigned, symbol, args[1]); JL_TYPECHK(memoryref_isassigned, bool, args[2]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); - if (isatomic == jl_false) - if (args[1] != (jl_value_t*)jl_not_atomic_sym) - jl_atomic_error("memoryref_isassigned!: non-atomic memory cannot be accessed atomically"); + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[1] != kind) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[1], 1, 0); + jl_atomic_error("memoryref_isassigned: non-atomic memory cannot be accessed atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[1], 1, 0); + if (order == jl_memory_order_notatomic) + jl_atomic_error("memoryref_isassigned: atomic memory cannot be accessed non-atomically"); + } if (m.mem->length == 0) + // TODO(jwn): decide on the fences required for ordering here return jl_false; - return jl_memoryref_isassigned(m); + return jl_memoryref_isassigned(m, kind == (jl_value_t*)jl_atomic_sym); +} + +JL_CALLABLE(jl_f_memoryrefswap) +{ + enum jl_memory_order order = jl_memory_order_notatomic; + JL_NARGS(memoryrefswap!, 4, 4); + JL_TYPECHK(memoryrefswap!, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefswap!, symbol, args[2]); + JL_TYPECHK(memoryrefswap!, bool, args[3]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[2] != kind) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 1, 1); + jl_atomic_error("memoryrefswap!: non-atomic memory cannot be written atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 1, 1); + if (order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefswap!: atomic memory cannot be written non-atomically"); + } + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + return jl_memoryrefswap(m, args[1], kind == (jl_value_t*)jl_atomic_sym); +} + +JL_CALLABLE(jl_f_memoryrefmodify) +{ + enum jl_memory_order order = jl_memory_order_notatomic; + JL_NARGS(memoryrefmodify!, 5, 5); + JL_TYPECHK(memoryrefmodify!, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefmodify!, symbol, args[3]); + JL_TYPECHK(memoryrefmodify!, bool, args[4]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[3] != kind) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + jl_atomic_error("memoryrefmodify!: non-atomic memory cannot be written atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + if (order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefmodify!: atomic memory cannot be written non-atomically"); + } + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + return jl_memoryrefmodify(m, args[1], args[2], kind == (jl_value_t*)jl_atomic_sym); +} + +JL_CALLABLE(jl_f_memoryrefreplace) +{ + enum jl_memory_order success_order = jl_memory_order_notatomic; + enum jl_memory_order failure_order = jl_memory_order_notatomic; + JL_NARGS(memoryrefreplace!, 6, 6); + JL_TYPECHK(memoryrefreplace!, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefreplace!, symbol, args[3]); + JL_TYPECHK(memoryrefreplace!, symbol, args[4]); + JL_TYPECHK(memoryrefreplace!, bool, args[5]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[4] != kind) + jl_atomic_error("invalid atomic ordering"); // because either it is invalid, or failure_order > success_order + if (args[3] != kind) { + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + jl_atomic_error("memoryrefreplace!: non-atomic memory cannot be written atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 1); + failure_order = jl_get_atomic_order_checked((jl_sym_t*)args[4], 1, 0); + if (failure_order > success_order) + jl_atomic_error("invalid atomic ordering"); // because either it is invalid, or failure_order > success_order + if (success_order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefreplace!: atomic memory cannot be written non-atomically"); + if (failure_order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefreplace!: atomic memory cannot be accessed non-atomically"); + } + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + return jl_memoryrefreplace(m, args[1], args[2], kind == (jl_value_t*)jl_atomic_sym); +} + +JL_CALLABLE(jl_f_memoryrefsetonce) +{ + enum jl_memory_order success_order = jl_memory_order_notatomic; + enum jl_memory_order failure_order = jl_memory_order_notatomic; + JL_NARGS(memoryrefsetonce!, 5, 5); + JL_TYPECHK(memoryrefsetonce!, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefsetonce!, symbol, args[2]); + JL_TYPECHK(memoryrefsetonce!, symbol, args[3]); + JL_TYPECHK(memoryrefsetonce!, bool, args[4]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *kind = jl_tparam0(jl_typetagof(m.mem)); + if (kind == (jl_value_t*)jl_not_atomic_sym) { + if (args[3] != kind) + jl_atomic_error("invalid atomic ordering"); // because either it is invalid, or failure_order > success_order + if (args[2] != kind) { + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 1, 1); + jl_atomic_error("memoryrefsetonce!: non-atomic memory cannot be written atomically"); + } + } + else if (kind == (jl_value_t*)jl_atomic_sym) { + success_order = jl_get_atomic_order_checked((jl_sym_t*)args[2], 1, 1); + failure_order = jl_get_atomic_order_checked((jl_sym_t*)args[3], 1, 0); + if (failure_order > success_order) + jl_atomic_error("invalid atomic ordering"); // because either it is invalid, or failure_order > success_order + if (success_order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefsetonce!: atomic memory cannot be written non-atomically"); + if (failure_order == jl_memory_order_notatomic) + jl_atomic_error("memoryrefsetonce!: atomic memory cannot be accessed non-atomically"); + } + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + return jl_memoryrefsetonce(m, args[1], kind == (jl_value_t*)jl_atomic_sym); } // type definition ------------------------------------------------------------ @@ -2119,6 +2401,7 @@ void jl_init_primitives(void) JL_GC_DISABLED // field access jl_builtin_getfield = add_builtin_func("getfield", jl_f_getfield); jl_builtin_setfield = add_builtin_func("setfield!", jl_f_setfield); + jl_builtin_setfieldonce = add_builtin_func("setfieldonce!", jl_f_setfieldonce); jl_builtin_swapfield = add_builtin_func("swapfield!", jl_f_swapfield); jl_builtin_modifyfield = add_builtin_func("modifyfield!", jl_f_modifyfield); jl_builtin_replacefield = add_builtin_func("replacefield!", jl_f_replacefield); @@ -2131,6 +2414,10 @@ void jl_init_primitives(void) JL_GC_DISABLED jl_builtin_setglobal = add_builtin_func("setglobal!", jl_f_setglobal); add_builtin_func("get_binding_type", jl_f_get_binding_type); add_builtin_func("set_binding_type!", jl_f_set_binding_type); + jl_builtin_swapglobal = add_builtin_func("swapglobal!", jl_f_swapglobal); + jl_builtin_replaceglobal = add_builtin_func("replaceglobal!", jl_f_replaceglobal); + jl_builtin_modifyglobal = add_builtin_func("modifyglobal!", jl_f_modifyglobal); + jl_builtin_setglobalonce = add_builtin_func("setglobalonce!", jl_f_setglobalonce); // memory primitives jl_builtin_memoryref = add_builtin_func("memoryref", jl_f_memoryref); @@ -2138,6 +2425,10 @@ void jl_init_primitives(void) JL_GC_DISABLED jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); jl_builtin_memoryrefset = add_builtin_func("memoryrefset!", jl_f_memoryrefset); jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); + jl_builtin_memoryrefswap = add_builtin_func("memoryrefswap!", jl_f_memoryrefswap); + jl_builtin_memoryrefreplace = add_builtin_func("memoryrefreplace!", jl_f_memoryrefreplace); + jl_builtin_memoryrefmodify = add_builtin_func("memoryrefmodify!", jl_f_memoryrefmodify); + jl_builtin_memoryrefsetonce = add_builtin_func("memoryrefsetonce!", jl_f_memoryrefsetonce); // method table utils jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); diff --git a/src/ccall.cpp b/src/ccall.cpp index 92ede74402791..b71d86623065f 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -669,7 +669,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va // --- code generator for cglobal --- -static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, const jl_cgval_t *argv, size_t nargs); +static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, ArrayRef argv, size_t nargs); static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) { @@ -684,7 +684,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg rt = static_eval(ctx, args[2]); if (rt == NULL) { JL_GC_POP(); - jl_cgval_t argv[2] = {jl_cgval_t(), jl_cgval_t()}; + jl_cgval_t argv[2]; argv[0] = emit_expr(ctx, args[1]); argv[1] = emit_expr(ctx, args[2]); return emit_runtime_call(ctx, JL_I::cglobal, argv, nargs); @@ -1999,7 +1999,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } else if (symarg.jl_ptr != NULL) { ++LiteralCCalls; - null_pointer_check(ctx, symarg.jl_ptr); + null_pointer_check(ctx, symarg.jl_ptr, nullptr); Type *funcptype = PointerType::get(functype, 0); llvmf = emit_inttoptr(ctx, symarg.jl_ptr, funcptype); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 2a53cf0528694..007e4c5936a4f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1381,16 +1381,31 @@ static void raise_exception_unless(jl_codectx_t &ctx, Value *cond, Value *exc) raise_exception(ctx, exc, passBB); } +static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name, jl_value_t *scope) +{ + ++EmittedUndefVarErrors; + BasicBlock *err = BasicBlock::Create(ctx.builder.getContext(), "err", ctx.f); + BasicBlock *ifok = BasicBlock::Create(ctx.builder.getContext(), "ok"); + ctx.builder.CreateCondBr(ok, ifok, err); + ctx.builder.SetInsertPoint(err); + ctx.builder.CreateCall(prepare_call(jlundefvarerror_func), { + mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)name)), + mark_callee_rooted(ctx, literal_pointer_val(ctx, scope))}); + ctx.builder.CreateUnreachable(); + ifok->insertInto(ctx.f); + ctx.builder.SetInsertPoint(ifok); +} + static Value *null_pointer_cmp(jl_codectx_t &ctx, Value *v) { ++EmittedNullchecks; - return ctx.builder.CreateICmpNE(v, Constant::getNullValue(v->getType())); + return ctx.builder.CreateIsNotNull(v); } // If `nullcheck` is not NULL and a pointer NULL check is necessary // store the pointer to be checked in `*nullcheck` instead of checking it -static void null_pointer_check(jl_codectx_t &ctx, Value *v, Value **nullcheck = nullptr) +static void null_pointer_check(jl_codectx_t &ctx, Value *v, Value **nullcheck) { if (nullcheck) { *nullcheck = v; @@ -1400,6 +1415,16 @@ static void null_pointer_check(jl_codectx_t &ctx, Value *v, Value **nullcheck = literal_pointer_val(ctx, jl_undefref_exception)); } + +static void null_load_check(jl_codectx_t &ctx, Value *v, jl_module_t *scope, jl_sym_t *name) +{ + Value *notnull = null_pointer_cmp(ctx, v); + if (name && scope) + undef_var_error_ifnot(ctx, notnull, name, (jl_value_t*)scope); + else + raise_exception_unless(ctx, notnull, literal_pointer_val(ctx, jl_undefref_exception)); +} + template static Value *emit_guarded_test(jl_codectx_t &ctx, Value *ifnot, Value *defval, Func &&func) { @@ -1887,16 +1912,16 @@ Value *extract_first_ptr(jl_codectx_t &ctx, Value *V) static void emit_lockstate_value(jl_codectx_t &ctx, Value *strct, bool newstate) { ++EmittedLockstates; - Value *v = mark_callee_rooted(ctx, strct); - ctx.builder.CreateCall(prepare_call(newstate ? jllockvalue_func : jlunlockvalue_func), v); -} -static void emit_lockstate_value(jl_codectx_t &ctx, const jl_cgval_t &strct, bool newstate) -{ - assert(strct.isboxed); - emit_lockstate_value(ctx, boxed(ctx, strct), newstate); + if (strct->getType()->getPointerAddressSpace() == AddressSpace::Loaded) { + Value *v = emit_bitcast(ctx, strct, PointerType::get(ctx.types().T_jlvalue, AddressSpace::Loaded)); + ctx.builder.CreateCall(prepare_call(newstate ? jllockfield_func : jlunlockfield_func), v); + } + else { + Value *v = mark_callee_rooted(ctx, strct); + ctx.builder.CreateCall(prepare_call(newstate ? jllockvalue_func : jlunlockvalue_func), v); + } } - // If `nullcheck` is not NULL and a pointer NULL check is necessary // store the pointer to be checked in `*nullcheck` instead of checking it static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, jl_value_t *jltype, @@ -1906,8 +1931,11 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j { // TODO: we should use unordered loads for anything with CountTrackedPointers(elty).count > 0 (if not otherwise locked) Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jltype); - if (type_is_ghost(elty)) + if (type_is_ghost(elty)) { + if (isStrongerThanMonotonic(Order)) + ctx.builder.CreateFence(Order); return ghostValue(ctx, jltype); + } unsigned nb = isboxed ? sizeof(void*) : jl_datatype_size(jltype); // note that nb == jl_Module->getDataLayout().getTypeAllocSize(elty) or getTypeStoreSize, depending on whether it is a struct or primitive type AllocaInst *intcast = NULL; @@ -2007,12 +2035,13 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j } static jl_cgval_t typed_store(jl_codectx_t &ctx, - Value *ptr, Value *idx_0based, jl_cgval_t rhs, jl_cgval_t cmp, + Value *ptr, jl_cgval_t rhs, jl_cgval_t cmp, jl_value_t *jltype, MDNode *tbaa, MDNode *aliasscope, Value *parent, // for the write barrier, NULL if no barrier needed bool isboxed, AtomicOrdering Order, AtomicOrdering FailOrder, unsigned alignment, - bool needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, - bool maybe_null_if_boxed, const jl_cgval_t *modifyop, const Twine &fname) + Value *needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, bool issetfieldonce, + bool maybe_null_if_boxed, const jl_cgval_t *modifyop, const Twine &fname, + jl_module_t *mod, jl_sym_t *var) { auto newval = [&](const jl_cgval_t &lhs) { const jl_cgval_t argv[3] = { cmp, lhs, rhs }; @@ -2028,9 +2057,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ret = update_julia_type(ctx, ret, jltype); return ret; }; - assert(!needlock || parent != nullptr); Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jltype); - if (type_is_ghost(elty)) { + if (type_is_ghost(elty) || + (issetfieldonce && !maybe_null_if_boxed) || + (issetfieldonce && !isboxed && !jl_type_hasptr(jltype))) { if (isStrongerThanMonotonic(Order)) ctx.builder.CreateFence(Order); if (issetfield) { @@ -2046,13 +2076,21 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, else if (isswapfield) { return ghostValue(ctx, jltype); } - else { // modifyfield + else if (ismodifyfield) { jl_cgval_t oldval = ghostValue(ctx, jltype); const jl_cgval_t argv[2] = { oldval, newval(oldval) }; jl_datatype_t *rettyp = jl_apply_modify_type(jltype); return emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } + else { // issetfieldonce + return mark_julia_const(ctx, jl_false); + } } + // if FailOrder was inherited from Order, may need to remove Load-only effects now + if (FailOrder == AtomicOrdering::AcquireRelease) + FailOrder = AtomicOrdering::Acquire; + if (FailOrder == AtomicOrdering::Release) + FailOrder = AtomicOrdering::Monotonic; unsigned nb = isboxed ? sizeof(void*) : jl_datatype_size(jltype); AllocaInst *intcast = nullptr; if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) { @@ -2069,7 +2107,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb2); } Value *r = nullptr; - if (issetfield || isswapfield || isreplacefield) { + if (issetfield || isswapfield || isreplacefield || issetfieldonce) { if (isboxed) r = boxed(ctx, rhs); else if (aliasscope || Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { @@ -2081,8 +2119,6 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); if (ptr->getType() != ptrty) ptr = ctx.builder.CreateBitCast(ptr, ptrty); - if (idx_0based) - ptr = ctx.builder.CreateInBoundsGEP(elty, ptr, idx_0based); if (isboxed) alignment = sizeof(void*); else if (!alignment) @@ -2092,12 +2128,13 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Value *Success = nullptr; BasicBlock *DoneBB = nullptr; if (needlock) - emit_lockstate_value(ctx, parent, true); + emit_lockstate_value(ctx, needlock, true); jl_cgval_t oldval = rhs; + // TODO: we should do Release ordering for anything with CountTrackedPointers(elty).count > 0, instead of just isboxed if (issetfield || (Order == AtomicOrdering::NotAtomic && isswapfield)) { if (isswapfield) { auto *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); - setName(ctx.emission_context, load, "swapfield_load"); + setName(ctx.emission_context, load, "swap_load"); if (isboxed) load->setOrdering(AtomicOrdering::Unordered); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -2118,17 +2155,19 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, emit_unbox_store(ctx, rhs, ptr, tbaa, alignment); } } - else if (isswapfield && isStrongerThanMonotonic(Order)) { + else if (isswapfield) { + if (Order == AtomicOrdering::Unordered) + Order = AtomicOrdering::Monotonic; assert(Order != AtomicOrdering::NotAtomic && r); auto *store = ctx.builder.CreateAtomicRMW(AtomicRMWInst::Xchg, ptr, r, Align(alignment), Order); - setName(ctx.emission_context, store, "swapfield_atomicrmw"); + setName(ctx.emission_context, store, "swap_atomicrmw"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); ai.decorateInst(store); instr = store; } else { - // replacefield, modifyfield, or swapfield (isboxed && atomic) + // replacefield, modifyfield, swapfield, setfieldonce (isboxed && atomic) DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); bool needloop; PHINode *Succ = nullptr, *Current = nullptr; @@ -2146,7 +2185,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.CreateCondBr(SameType, BB, SkipBB); ctx.builder.SetInsertPoint(SkipBB); LoadInst *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); - setName(ctx.emission_context, load, "atomic_replacefield_initial"); + setName(ctx.emission_context, load, "atomic_replace_initial"); load->setOrdering(FailOrder == AtomicOrdering::NotAtomic && isboxed ? AtomicOrdering::Monotonic : FailOrder); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); @@ -2174,6 +2213,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, needloop = true; } } + else if (issetfieldonce) { + needloop = !isboxed && Order != AtomicOrdering::NotAtomic && nb > sizeof(void*); + if (Order != AtomicOrdering::NotAtomic) + Compare = Constant::getNullValue(elty); + } else { // swap or modify LoadInst *Current = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); Current->setOrdering(Order == AtomicOrdering::NotAtomic && !isboxed ? Order : AtomicOrdering::Monotonic); @@ -2196,7 +2240,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } if (ismodifyfield) { if (needlock) - emit_lockstate_value(ctx, parent, false); + emit_lockstate_value(ctx, needlock, false); Value *realCompare = Compare; if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); @@ -2208,7 +2252,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (maybe_null_if_boxed) { Value *first_ptr = isboxed ? Compare : extract_first_ptr(ctx, Compare); if (first_ptr) - null_pointer_check(ctx, first_ptr, nullptr); + null_load_check(ctx, first_ptr, mod, var); } if (intcast) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); @@ -2224,12 +2268,12 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, r = ctx.builder.CreateZExt(r, elty); } if (needlock) - emit_lockstate_value(ctx, parent, true); + emit_lockstate_value(ctx, needlock, true); cmp = oldval; } Value *Done; if (Order == AtomicOrdering::NotAtomic) { - // modifyfield or replacefield + // modifyfield or replacefield or setfieldonce assert(elty == realelty && !intcast); auto *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -2241,9 +2285,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (maybe_null_if_boxed && !ismodifyfield) first_ptr = isboxed ? load : extract_first_ptr(ctx, load); oldval = mark_julia_type(ctx, load, isboxed, jltype); - Success = emit_nullcheck_guard(ctx, first_ptr, [&] { - return emit_f_is(ctx, oldval, cmp); - }); + assert(!issetfieldonce || first_ptr != nullptr); + if (issetfieldonce) + Success = ctx.builder.CreateIsNull(first_ptr); + else + Success = emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); if (needloop && ismodifyfield) CmpPhi->addIncoming(load, ctx.builder.GetInsertBlock()); assert(Succ == nullptr); @@ -2263,13 +2309,13 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.CreateBr(DoneBB); instr = load; } - else { + else { // something atomic assert(r); if (Order == AtomicOrdering::Unordered) Order = AtomicOrdering::Monotonic; if (Order == AtomicOrdering::Monotonic && isboxed) Order = AtomicOrdering::Release; - if (!isreplacefield) + if (!isreplacefield && !issetfieldonce) FailOrder = AtomicOrdering::Monotonic; else if (FailOrder == AtomicOrdering::Unordered) FailOrder = AtomicOrdering::Monotonic; @@ -2280,7 +2326,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, instr = ctx.builder.Insert(ExtractValueInst::Create(store, 0)); Success = ctx.builder.Insert(ExtractValueInst::Create(store, 1)); Done = Success; - if (isreplacefield && needloop) { + if ((isreplacefield || issetfieldonce) && needloop) { Value *realinstr = instr; if (realelty != elty) realinstr = ctx.builder.CreateTrunc(realinstr, realelty); @@ -2293,15 +2339,22 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, else { oldval = mark_julia_type(ctx, realinstr, isboxed, jltype); } - Done = emit_guarded_test(ctx, ctx.builder.CreateNot(Success), false, [&] { - Value *first_ptr = nullptr; - if (maybe_null_if_boxed) - first_ptr = isboxed ? realinstr : extract_first_ptr(ctx, realinstr); - return emit_nullcheck_guard(ctx, first_ptr, [&] { - return emit_f_is(ctx, oldval, cmp); + if (issetfieldonce) { + assert(!isboxed && maybe_null_if_boxed); + Value *first_ptr = extract_first_ptr(ctx, realinstr); + assert(first_ptr != nullptr); + Done = ctx.builder.CreateIsNotNull(first_ptr); + } + else { + // Done = !(!Success && (first_ptr != NULL && oldval == cmp)) + Done = emit_guarded_test(ctx, ctx.builder.CreateNot(Success), false, [&] { + Value *first_ptr = nullptr; + if (maybe_null_if_boxed) + first_ptr = isboxed ? realinstr : extract_first_ptr(ctx, realinstr); + return emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); }); - }); - Done = ctx.builder.CreateNot(Done); + Done = ctx.builder.CreateNot(Done); + } } if (needloop) ctx.builder.CreateCondBr(Done, DoneBB, BB); @@ -2320,9 +2373,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (DoneBB) ctx.builder.SetInsertPoint(DoneBB); if (needlock) - emit_lockstate_value(ctx, parent, false); + emit_lockstate_value(ctx, needlock, false); if (parent != NULL) { - if (isreplacefield) { + if (isreplacefield || issetfieldonce) { // TODO: avoid this branch if we aren't making a write barrier BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "xchg_wb", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg_wb", ctx.f); @@ -2335,7 +2388,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, else if (!type_is_permalloc(rhs.typ)) emit_write_barrier(ctx, parent, r); } - if (isreplacefield) { + if (isreplacefield || issetfieldonce) { ctx.builder.CreateBr(DoneBB); ctx.builder.SetInsertPoint(DoneBB); } @@ -2345,6 +2398,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, jl_datatype_t *rettyp = jl_apply_modify_type(jltype); oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } + else if (issetfieldonce) { + oldval = mark_julia_type(ctx, Success, false, jl_bool_type); + } else if (!issetfield) { // swapfield or replacefield if (realelty != elty) instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty)); @@ -2357,7 +2413,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, instr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); Value *first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr); if (first_ptr) - null_pointer_check(ctx, first_ptr, nullptr); + null_load_check(ctx, first_ptr, mod, var); if (intcast && !first_ptr) instr = nullptr; } @@ -2472,7 +2528,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, setName(ctx.emission_context, fld, "getfield"); jl_value_t *jft = issame ? jl_svecref(types, 0) : (jl_value_t*)jl_any_type; if (isboxed && maybe_null) - null_pointer_check(ctx, fld); + null_pointer_check(ctx, fld, nullptr); *ret = mark_julia_type(ctx, fld, isboxed, jft); return true; } @@ -2511,7 +2567,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, ai.decorateInst(fld); maybe_mark_load_dereferenceable(fld, maybe_null, minimum_field_size, minimum_align); if (maybe_null) - null_pointer_check(ctx, fld); + null_pointer_check(ctx, fld, nullptr); *ret = mark_julia_type(ctx, fld, true, jl_any_type); return true; } @@ -2601,7 +2657,6 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st }; jl_value_t *jfty = jl_field_type(jt, idx); bool isatomic = jl_field_isatomic(jt, idx); - bool needlock = isatomic && !jl_field_isptr(jt, idx) && jl_datatype_size(jfty) > MAX_ATOMIC_SIZE; if (!isatomic && order != jl_memory_order_notatomic && order != jl_memory_order_unspecified) { emit_atomic_error(ctx, "getfield: non-atomic field cannot be accessed atomically"); return jl_cgval_t(); // unreachable @@ -2619,6 +2674,11 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st } if (type_is_ghost(julia_type_to_llvm(ctx, jfty))) return ghostValue(ctx, jfty); + Value *needlock = nullptr; + if (isatomic && !jl_field_isptr(jt, idx) && jl_datatype_size(jfty) > MAX_ATOMIC_SIZE) { + assert(strct.isboxed); + needlock = boxed(ctx, strct); + } size_t nfields = jl_datatype_nfields(jt); bool maybe_null = idx >= nfields - (unsigned)jt->name->n_uninitialized; size_t byte_offset = jl_field_offset(jt, idx); @@ -2694,7 +2754,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st } unsigned align = jl_field_align(jt, idx); if (needlock) - emit_lockstate_value(ctx, strct, true); + emit_lockstate_value(ctx, needlock, true); jl_cgval_t ret = typed_load(ctx, addr, NULL, jfty, tbaa, nullptr, false, needlock ? AtomicOrdering::NotAtomic : get_llvm_atomic_order(order), maybe_null, align, nullcheck); @@ -2702,7 +2762,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st setNameWithField(ctx.emission_context, ret.V, get_objname, jt, idx, Twine()); } if (needlock) - emit_lockstate_value(ctx, strct, false); + emit_lockstate_value(ctx, needlock, false); return ret; } else if (isa(strct.V)) { @@ -3563,11 +3623,97 @@ static void emit_write_multibarrier(jl_codectx_t &ctx, Value *parent, Value *agg emit_write_barrier(ctx, parent, ptrs); } +static jl_cgval_t union_store(jl_codectx_t &ctx, + Value *ptr, Value *ptindex, jl_cgval_t rhs, jl_cgval_t cmp, + jl_value_t *jltype, MDNode *tbaa, MDNode *aliasscope, MDNode *tbaa_tindex, + AtomicOrdering Order, AtomicOrdering FailOrder, + Value *needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, bool issetfieldonce, + const jl_cgval_t *modifyop, const Twine &fname) +{ + assert(Order == AtomicOrdering::NotAtomic); + if (issetfieldonce) + return mark_julia_const(ctx, jl_false); + size_t fsz = 0, al = 0; + int union_max = jl_islayout_inline(jltype, &fsz, &al); + assert(union_max > 0); + // compute tindex from rhs + jl_cgval_t rhs_union = convert_julia_type(ctx, rhs, jltype); + if (rhs_union.typ == jl_bottom_type) + return jl_cgval_t(); + if (needlock) + emit_lockstate_value(ctx, needlock, true); + BasicBlock *ModifyBB = NULL; + if (ismodifyfield) { + ModifyBB = BasicBlock::Create(ctx.builder.getContext(), "modify_xchg", ctx.f); + ctx.builder.CreateBr(ModifyBB); + ctx.builder.SetInsertPoint(ModifyBB); + } + jl_cgval_t oldval = rhs; + if (!issetfield) + oldval = emit_unionload(ctx, ptr, ptindex, jltype, fsz, al, tbaa, true, union_max, tbaa_tindex); + Value *Success = NULL; + BasicBlock *DoneBB = NULL; + if (isreplacefield || ismodifyfield) { + if (ismodifyfield) { + if (needlock) + emit_lockstate_value(ctx, needlock, false); + const jl_cgval_t argv[3] = { cmp, oldval, rhs }; + if (modifyop) { + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + } + else { + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); + rhs = mark_julia_type(ctx, callval, true, jl_any_type); + } + emit_typecheck(ctx, rhs, jltype, fname); + rhs = update_julia_type(ctx, rhs, jltype); + rhs_union = convert_julia_type(ctx, rhs, jltype); + if (rhs_union.typ == jl_bottom_type) + return jl_cgval_t(); + if (needlock) + emit_lockstate_value(ctx, needlock, true); + cmp = oldval; + oldval = emit_unionload(ctx, ptr, ptindex, jltype, fsz, al, tbaa, true, union_max, tbaa_tindex); + } + BasicBlock *XchgBB = BasicBlock::Create(ctx.builder.getContext(), "xchg", ctx.f); + DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); + Success = emit_f_is(ctx, oldval, cmp); + ctx.builder.CreateCondBr(Success, XchgBB, ismodifyfield ? ModifyBB : DoneBB); + ctx.builder.SetInsertPoint(XchgBB); + } + Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jltype); + tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); + // copy data + if (!rhs.isghost) { + emit_unionmove(ctx, ptr, tbaa, rhs, nullptr); + } + if (isreplacefield || ismodifyfield) { + ctx.builder.CreateBr(DoneBB); + ctx.builder.SetInsertPoint(DoneBB); + } + if (needlock) + emit_lockstate_value(ctx, needlock, false); + if (isreplacefield) { + Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); + jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; + jl_datatype_t *rettyp = jl_apply_cmpswap_type(jltype); + oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); + } + else if (ismodifyfield) { + jl_cgval_t argv[2] = {oldval, rhs}; + jl_datatype_t *rettyp = jl_apply_modify_type(jltype); + oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); + } + return oldval; +} + static jl_cgval_t emit_setfield(jl_codectx_t &ctx, jl_datatype_t *sty, const jl_cgval_t &strct, size_t idx0, jl_cgval_t rhs, jl_cgval_t cmp, bool wb, AtomicOrdering Order, AtomicOrdering FailOrder, - bool needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, + Value *needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, bool issetfieldonce, const jl_cgval_t *modifyop, const Twine &fname) { auto get_objname = [&]() { @@ -3582,111 +3728,33 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, addr = ctx.builder.CreateInBoundsGEP( getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, addr, getInt8PtrTy(ctx.builder.getContext())), - ConstantInt::get(ctx.types().T_size, byte_offset)); // TODO: use emit_struct_gep + ConstantInt::get(ctx.types().T_size, byte_offset)); setNameWithField(ctx.emission_context, addr, get_objname, sty, idx0, Twine("_ptr")); } jl_value_t *jfty = jl_field_type(sty, idx0); - if (!jl_field_isptr(sty, idx0) && jl_is_uniontype(jfty)) { - size_t fsz = 0, al = 0; - int union_max = jl_islayout_inline(jfty, &fsz, &al); - bool isptr = (union_max == 0); - assert(!isptr && fsz < jl_field_size(sty, idx0)); (void)isptr; + bool isboxed = jl_field_isptr(sty, idx0); + if (!isboxed && jl_is_uniontype(jfty)) { size_t fsz1 = jl_field_size(sty, idx0) - 1; - // compute tindex from rhs - jl_cgval_t rhs_union = convert_julia_type(ctx, rhs, jfty); - if (rhs_union.typ == jl_bottom_type) - return jl_cgval_t(); Value *ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, addr, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, fsz1)); setNameWithField(ctx.emission_context, ptindex, get_objname, sty, idx0, Twine(".tindex_ptr")); - if (needlock) - emit_lockstate_value(ctx, strct, true); - BasicBlock *ModifyBB = NULL; - if (ismodifyfield) { - ModifyBB = BasicBlock::Create(ctx.builder.getContext(), "modify_xchg", ctx.f); - ctx.builder.CreateBr(ModifyBB); - ctx.builder.SetInsertPoint(ModifyBB); - } - jl_cgval_t oldval = rhs; - if (!issetfield) - oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); - Value *Success = NULL; - BasicBlock *DoneBB = NULL; - if (isreplacefield || ismodifyfield) { - if (ismodifyfield) { - if (needlock) - emit_lockstate_value(ctx, strct, false); - const jl_cgval_t argv[3] = { cmp, oldval, rhs }; - if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); - } - else { - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); - rhs = mark_julia_type(ctx, callval, true, jl_any_type); - } - emit_typecheck(ctx, rhs, jfty, fname); - rhs = update_julia_type(ctx, rhs, jfty); - rhs_union = convert_julia_type(ctx, rhs, jfty); - if (rhs_union.typ == jl_bottom_type) - return jl_cgval_t(); - if (needlock) - emit_lockstate_value(ctx, strct, true); - cmp = oldval; - oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); - } - BasicBlock *XchgBB = BasicBlock::Create(ctx.builder.getContext(), "xchg", ctx.f); - DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); - Success = emit_f_is(ctx, oldval, cmp); - ctx.builder.CreateCondBr(Success, XchgBB, ismodifyfield ? ModifyBB : DoneBB); - ctx.builder.SetInsertPoint(XchgBB); - } - Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jfty); - tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); - ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); - // copy data - if (!rhs.isghost) { - emit_unionmove(ctx, addr, tbaa, rhs, nullptr); - } - if (isreplacefield || ismodifyfield) { - ctx.builder.CreateBr(DoneBB); - ctx.builder.SetInsertPoint(DoneBB); - } - if (needlock) - emit_lockstate_value(ctx, strct, false); - if (isreplacefield) { - Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); - jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; - jl_datatype_t *rettyp = jl_apply_cmpswap_type(jfty); - oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); - if (oldval.V) { - setNameWithField(ctx.emission_context, oldval.V, get_objname, sty, idx0, Twine()); - } - } - else if (ismodifyfield) { - jl_cgval_t argv[2] = {oldval, rhs}; - jl_datatype_t *rettyp = jl_apply_modify_type(jfty); - oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); - if (oldval.V) { - setNameWithField(ctx.emission_context, oldval.V, get_objname, sty, idx0, Twine()); - } - } - return oldval; - } - else { - unsigned align = jl_field_align(sty, idx0); - bool isboxed = jl_field_isptr(sty, idx0); - size_t nfields = jl_datatype_nfields(sty); - bool maybe_null = idx0 >= nfields - (unsigned)sty->name->n_uninitialized; - return typed_store(ctx, addr, NULL, rhs, cmp, jfty, tbaa, nullptr, - wb ? boxed(ctx, strct) : nullptr, - isboxed, Order, FailOrder, align, - needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, maybe_null, modifyop, fname); + return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, nullptr, ctx.tbaa().tbaa_unionselbyte, + Order, FailOrder, + needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, issetfieldonce, + modifyop, fname); } + unsigned align = jl_field_align(sty, idx0); + size_t nfields = jl_datatype_nfields(sty); + bool maybe_null = idx0 >= nfields - (unsigned)sty->name->n_uninitialized; + return typed_store(ctx, addr, rhs, cmp, jfty, tbaa, nullptr, + wb ? boxed(ctx, strct) : nullptr, + isboxed, Order, FailOrder, align, + needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, issetfieldonce, + maybe_null, modifyop, fname, nullptr, nullptr); } -static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, const jl_cgval_t *argv, bool is_promotable) +static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable) { ++EmittedNewStructs; assert(jl_is_datatype(ty)); @@ -3930,7 +3998,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg rhs = update_julia_type(ctx, rhs, ft); if (rhs.typ == jl_bottom_type) return jl_cgval_t(); - emit_setfield(ctx, sty, strctinfo, i, rhs, jl_cgval_t(), need_wb, AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, false, true, false, false, false, nullptr, ""); + emit_setfield(ctx, sty, strctinfo, i, rhs, jl_cgval_t(), need_wb, AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, nullptr, true, false, false, false, false, nullptr, "new"); } return strctinfo; } diff --git a/src/codegen.cpp b/src/codegen.cpp index c91e70b5e508e..6de2669ac0f7d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -879,6 +879,42 @@ static const auto jlcheckassign_func = new JuliaFunction<>{ {T_pjlvalue, T_pjlvalue, T_pjlvalue, PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, nullptr, }; +static const auto jlcheckreplace_func = new JuliaFunction<>{ + XSTR(jl_checked_replace), + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); + return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pjlvalue, T_prjlvalue, T_prjlvalue}, false); }, + nullptr, +}; +static const auto jlcheckmodify_func = new JuliaFunction<>{ + XSTR(jl_checked_modify), + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); + return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pjlvalue, T_prjlvalue, T_prjlvalue}, false); }, + nullptr, +}; +static const auto jlcheckswap_func = new JuliaFunction<>{ + XSTR(jl_checked_swap), + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); + return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pjlvalue, PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, + nullptr, +}; +static const auto jlcheckassignonce_func = new JuliaFunction<>{ + XSTR(jl_checked_assignonce), + [](LLVMContext &C) { + auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); + auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); + return FunctionType::get(T_prjlvalue, + {T_pjlvalue, T_pjlvalue, T_pjlvalue, PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, + nullptr, +}; static const auto jldeclareconst_func = new JuliaFunction<>{ XSTR(jl_declare_constant), [](LLVMContext &C) { @@ -992,6 +1028,24 @@ static const auto jlunlockvalue_func = new JuliaFunction<>{ AttributeSet(), {Attributes(C, {Attribute::NoCapture})}); }, }; +static const auto jllockfield_func = new JuliaFunction<>{ + XSTR(jl_lock_field), + [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), + {PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::Loaded)}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + AttributeSet(), + {Attributes(C, {Attribute::NoCapture})}); }, +}; +static const auto jlunlockfield_func = new JuliaFunction<>{ + XSTR(jl_unlock_field), + [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), + {PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::Loaded)}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + AttributeSet(), + AttributeSet(), + {Attributes(C, {Attribute::NoCapture})}); }, +}; static const auto jlenter_func = new JuliaFunction<>{ XSTR(jl_enter_handler), [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), @@ -1525,6 +1579,10 @@ static const auto &builtin_func_map() { { jl_f_memoryref_addr, new JuliaFunction<>{XSTR(jl_f_memoryref), get_func_sig, get_func_attrs} }, { jl_f_memoryrefoffset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefoffset), get_func_sig, get_func_attrs} }, { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefswap_addr, new JuliaFunction<>{XSTR(jl_f_memoryswapset), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefreplace_addr, new JuliaFunction<>{XSTR(jl_f_memoryreplaceset), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefmodify_addr, new JuliaFunction<>{XSTR(jl_f_memorymodifyset), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefsetonce_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefsetonce), get_func_sig, get_func_attrs} }, { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, { jl_f_donotdelete_addr, new JuliaFunction<>{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} }, @@ -1999,13 +2057,13 @@ static Value *get_current_ptls(jl_codectx_t &ctx); static Value *get_last_age_field(jl_codectx_t &ctx); static void CreateTrap(IRBuilder<> &irbuilder, bool create_new_block = true); static CallInst *emit_jlcall(jl_codectx_t &ctx, FunctionCallee theFptr, Value *theF, - const jl_cgval_t *args, size_t nargs, JuliaFunction<> *trampoline); + ArrayRef args, size_t nargs, JuliaFunction<> *trampoline); static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value *theF, - const jl_cgval_t *args, size_t nargs, JuliaFunction<> *trampoline); + ArrayRef args, size_t nargs, JuliaFunction<> *trampoline); static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); -static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, const jl_cgval_t *argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const jl_cgval_t *argv, size_t nargs, jl_value_t *rt); +static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -2766,7 +2824,7 @@ static void cg_bdw(jl_codectx_t &ctx, jl_sym_t *var, jl_binding_t *b) } } -static jl_value_t *static_apply_type(jl_codectx_t &ctx, const jl_cgval_t *args, size_t nargs) +static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef args, size_t nargs) { assert(nargs > 1); SmallVector v(nargs); @@ -3104,27 +3162,80 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * return emit_checked_var(ctx, bp, name, (jl_value_t*)mod, false, ctx.tbaa().tbaa_binding); } -static bool emit_globalset(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *sym, const jl_cgval_t &rval_info, AtomicOrdering Order) +static jl_cgval_t emit_globalop(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *sym, jl_cgval_t rval, const jl_cgval_t &cmp, + AtomicOrdering Order, AtomicOrdering FailOrder, + bool issetglobal, bool isreplaceglobal, bool isswapglobal, bool ismodifyglobal, bool issetglobalonce, + const jl_cgval_t *modifyop) { jl_binding_t *bnd = NULL; Value *bp = global_binding_pointer(ctx, mod, sym, &bnd, true); if (bp == NULL) - return false; - Value *rval = boxed(ctx, rval_info); + return jl_cgval_t(); if (bnd && !bnd->constp) { jl_value_t *ty = jl_atomic_load_relaxed(&bnd->ty); - if (ty && jl_subtype(rval_info.typ, ty)) { // TODO: use typeassert here instead - StoreInst *v = ctx.builder.CreateAlignedStore(rval, julia_binding_pvalue(ctx, bp), Align(sizeof(void*))); - v->setOrdering(Order); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); - ai.decorateInst(v); - emit_write_barrier(ctx, bp, rval); - return true; - } - } - ctx.builder.CreateCall(prepare_call(jlcheckassign_func), - { bp, literal_pointer_val(ctx, (jl_value_t*)mod), literal_pointer_val(ctx, (jl_value_t*)sym), mark_callee_rooted(ctx, rval) }); - return true; + if (ty != nullptr) { + const std::string fname = issetglobal ? "setglobal!" : isreplaceglobal ? "replaceglobal!" : isswapglobal ? "swapglobal!" : ismodifyglobal ? "modifyglobal!" : "setglobalonce!"; + if (!ismodifyglobal) { + // TODO: use typeassert in jl_check_binding_wr too + emit_typecheck(ctx, rval, ty, "typeassert"); + rval = update_julia_type(ctx, rval, ty); + if (rval.typ == jl_bottom_type) + return jl_cgval_t(); + } + bool isboxed = true; + bool maybe_null = jl_atomic_load_relaxed(&bnd->value) == NULL; + return typed_store(ctx, + julia_binding_pvalue(ctx, bp), + rval, cmp, ty, + ctx.tbaa().tbaa_binding, + nullptr, + bp, + isboxed, + Order, + FailOrder, + 0, + nullptr, + issetglobal, + isreplaceglobal, + isswapglobal, + ismodifyglobal, + issetglobalonce, + maybe_null, + modifyop, + fname, + mod, + sym); + + } + } + Value *m = literal_pointer_val(ctx, (jl_value_t*)mod); + Value *s = literal_pointer_val(ctx, (jl_value_t*)sym); + if (issetglobal) { + ctx.builder.CreateCall(prepare_call(jlcheckassign_func), + { bp, m, s, mark_callee_rooted(ctx, boxed(ctx, rval)) }); + return rval; + } + else if (isreplaceglobal) { + Value *r = ctx.builder.CreateCall(prepare_call(jlcheckreplace_func), + { bp, m, s, boxed(ctx, cmp), boxed(ctx, rval) }); + return mark_julia_type(ctx, r, true, jl_any_type); + } + else if (isswapglobal) { + Value *r = ctx.builder.CreateCall(prepare_call(jlcheckswap_func), + { bp, m, s, mark_callee_rooted(ctx, boxed(ctx, rval)) }); + return mark_julia_type(ctx, r, true, jl_any_type); + } + else if (ismodifyglobal) { + Value *r = ctx.builder.CreateCall(prepare_call(jlcheckmodify_func), + { bp, m, s, boxed(ctx, cmp), boxed(ctx, rval) }); + return mark_julia_type(ctx, r, true, jl_any_type); + } + else if (issetglobalonce) { + Value *r = ctx.builder.CreateCall(prepare_call(jlcheckassignonce_func), + { bp, m, s, mark_callee_rooted(ctx, boxed(ctx, rval)) }); + return mark_julia_type(ctx, r, true, jl_bool_type); + } + abort(); // unreachable } static Value *emit_box_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, @@ -3405,37 +3516,69 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva } static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, - const jl_cgval_t *argv, size_t nargs, const jl_cgval_t *modifyop) + ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { + bool issetglobal = f == jl_builtin_setglobal; + bool isreplaceglobal = f == jl_builtin_replaceglobal; + bool isswapglobal = f == jl_builtin_swapglobal; + bool ismodifyglobal = f == jl_builtin_modifyglobal; + bool issetglobalonce = f == jl_builtin_setglobalonce; + const jl_cgval_t undefval; const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; - const jl_cgval_t &val = argv[3]; - enum jl_memory_order order = jl_memory_order_unspecified; - assert(f == jl_builtin_setglobal && modifyop == nullptr && "unimplemented"); - - if (nargs == 4) { - const jl_cgval_t &arg4 = argv[4]; - if (arg4.constant && jl_is_symbol(arg4.constant)) - order = jl_get_atomic_order((jl_sym_t*)arg4.constant, false, true); - else + jl_cgval_t val = argv[isreplaceglobal || ismodifyglobal ? 4 : 3]; + const jl_cgval_t &cmp = isreplaceglobal || ismodifyglobal ? argv[3] : undefval; + enum jl_memory_order order = jl_memory_order_release; + const std::string fname = issetglobal ? "setglobal!" : isreplaceglobal ? "replaceglobal!" : isswapglobal ? "swapglobal!" : ismodifyglobal ? "modifyglobal!" : "setglobalonce!"; + if (nargs >= (isreplaceglobal || ismodifyglobal ? 5 : 4)) { + const jl_cgval_t &ord = argv[isreplaceglobal || ismodifyglobal ? 5 : 4]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) return false; + order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetglobal, true); + } + enum jl_memory_order fail_order = order; + if ((isreplaceglobal || issetglobalonce) && nargs == (isreplaceglobal ? 6 : 5)) { + const jl_cgval_t &ord = argv[isreplaceglobal ? 6 : 5]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + } + if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { + emit_atomic_error(ctx, "invalid atomic ordering"); + *ret = jl_cgval_t(); // unreachable + return true; } - else - order = jl_memory_order_release; - if (order == jl_memory_order_invalid || order == jl_memory_order_notatomic) { - emit_atomic_error(ctx, order == jl_memory_order_invalid ? "invalid atomic ordering" : "setglobal!: module binding cannot be written non-atomically"); + if (order == jl_memory_order_notatomic) { + emit_atomic_error(ctx, + issetglobal ? "setglobal!: module binding cannot be written non-atomically" : + isreplaceglobal ? "replaceglobal!: module binding cannot be written non-atomically" : + isswapglobal ? "swapglobal!: module binding cannot be written non-atomically" : + ismodifyglobal ? "modifyglobal!: module binding cannot be written non-atomically" : + "setglobalonce!: module binding cannot be written non-atomically"); + *ret = jl_cgval_t(); // unreachable + return true; + } + else if (fail_order == jl_memory_order_notatomic) { + emit_atomic_error(ctx, + isreplaceglobal ? "replaceglobal!: module binding cannot be accessed non-atomically" : + "setglobalonce!: module binding cannot be accessed non-atomically"); *ret = jl_cgval_t(); // unreachable return true; } if (sym.constant && jl_is_symbol(sym.constant)) { - jl_sym_t *name = (jl_sym_t*)sym.constant; if (mod.constant && jl_is_module(mod.constant)) { - if (emit_globalset(ctx, (jl_module_t*)mod.constant, name, val, get_llvm_atomic_order(order))) - *ret = val; - else - *ret = jl_cgval_t(); // unreachable + *ret = emit_globalop(ctx, (jl_module_t*)mod.constant, (jl_sym_t*)sym.constant, val, cmp, + get_llvm_atomic_order(order), get_llvm_atomic_order(fail_order), + issetglobal, + isreplaceglobal, + isswapglobal, + ismodifyglobal, + issetglobalonce, + modifyop); return true; } } @@ -3444,20 +3587,21 @@ static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, - const jl_cgval_t *argv, size_t nargs, const jl_cgval_t *modifyop) + ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { ++EmittedOpfields; bool issetfield = f == jl_builtin_setfield; bool isreplacefield = f == jl_builtin_replacefield; bool isswapfield = f == jl_builtin_swapfield; bool ismodifyfield = f == jl_builtin_modifyfield; + bool issetfieldonce = f == jl_builtin_setfieldonce; const jl_cgval_t undefval; const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; jl_cgval_t val = argv[isreplacefield || ismodifyfield ? 4 : 3]; const jl_cgval_t &cmp = isreplacefield || ismodifyfield ? argv[3] : undefval; enum jl_memory_order order = jl_memory_order_notatomic; - const std::string fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : isswapfield ? "swapfield!" : "modifyfield!"; + const std::string fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : isswapfield ? "swapfield!" : ismodifyfield ? "modifyfield!" : "setfieldonce!"; if (nargs >= (isreplacefield || ismodifyfield ? 5 : 4)) { const jl_cgval_t &ord = argv[isreplacefield || ismodifyfield ? 5 : 4]; emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); @@ -3466,8 +3610,8 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); } enum jl_memory_order fail_order = order; - if (isreplacefield && nargs == 6) { - const jl_cgval_t &ord = argv[6]; + if ((isreplacefield || issetfieldonce) && nargs == (isreplacefield ? 6 : 5)) { + const jl_cgval_t &ord = argv[isreplacefield ? 6 : 5]; emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; @@ -3515,13 +3659,19 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, isswapfield ? (isatomic ? "swapfield!: atomic field cannot be written non-atomically" : "swapfield!: non-atomic field cannot be written atomically") : + ismodifyfield ? (isatomic ? "modifyfield!: atomic field cannot be written non-atomically" - : "modifyfield!: non-atomic field cannot be written atomically")); + : "modifyfield!: non-atomic field cannot be written atomically") : + (isatomic ? "setfieldonce!: atomic field cannot be written non-atomically" + : "setfieldonce!: non-atomic field cannot be written atomically")); } else if (isatomic == (fail_order == jl_memory_order_notatomic)) { emit_atomic_error(ctx, + isreplacefield ? (isatomic ? "replacefield!: atomic field cannot be accessed non-atomically" - : "replacefield!: non-atomic field cannot be accessed atomically")); + : "replacefield!: non-atomic field cannot be accessed atomically") : + (isatomic ? "setfieldonce!: atomic field cannot be accessed non-atomically" + : "setfieldonce!: non-atomic field cannot be accessed atomically")); } else if (!uty->name->mutabl) { std::string msg = fname + ": immutable struct of type " @@ -3538,6 +3688,7 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_error(ctx, msg); } else { + assert(obj.isboxed); *ret = emit_setfield(ctx, uty, obj, idx, val, cmp, true, (needlock || order <= jl_memory_order_notatomic) ? AtomicOrdering::NotAtomic @@ -3545,7 +3696,8 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, (needlock || fail_order <= jl_memory_order_notatomic) ? AtomicOrdering::NotAtomic : get_llvm_atomic_order(fail_order), - needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, + needlock ? boxed(ctx, obj) : nullptr, + issetfield, isreplacefield, isswapfield, ismodifyfield, issetfieldonce, modifyop, fname); } return true; @@ -3555,6 +3707,185 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } +static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, + ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) +{ + bool issetmemory = f == jl_builtin_memoryrefset; + bool isreplacememory = f == jl_builtin_memoryrefreplace; + bool isswapmemory = f == jl_builtin_memoryrefswap; + bool ismodifymemory = f == jl_builtin_memoryrefmodify; + bool issetmemoryonce = f == jl_builtin_memoryrefsetonce; + + const jl_cgval_t undefval; + const jl_cgval_t &ref = argv[1]; + jl_cgval_t val = argv[isreplacememory || ismodifymemory ? 3 : 2]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (!jl_is_genericmemoryref_type(mty_dt) || !jl_is_concrete_type(mty_dt)) + return false; + + jl_value_t *kind = jl_tparam0(mty_dt); + jl_value_t *ety = jl_tparam1(mty_dt); + jl_value_t *addrspace = jl_tparam2(mty_dt); (void)addrspace; // TODO + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + if (kind != (jl_value_t*)jl_not_atomic_sym && kind != (jl_value_t*)jl_atomic_sym) + return false; + + const jl_cgval_t &cmp = isreplacememory || ismodifymemory ? argv[2] : undefval; + enum jl_memory_order order = jl_memory_order_notatomic; + const std::string fname = issetmemory ? "memoryrefset!" : isreplacememory ? "memoryrefreplace!" : isswapmemory ? "memoryrefswap!" : ismodifymemory ? "memoryrefmodify!" : "memoryrefsetonce!"; + { + const jl_cgval_t &ord = argv[isreplacememory || ismodifymemory ? 4 : 3]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetmemory, true); + } + enum jl_memory_order fail_order = order; + if (isreplacememory || issetmemoryonce) { + const jl_cgval_t &ord = argv[isreplacememory ? 5 : 4]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + } + if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { + emit_atomic_error(ctx, "invalid atomic ordering"); + *ret = jl_cgval_t(); // unreachable + return true; + } + + jl_value_t *boundscheck = argv[nargs].constant; + emit_typecheck(ctx, argv[nargs], (jl_value_t*)jl_bool_type, fname); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isboxed = layout->flags.arrayelem_isboxed; + bool isunion = layout->flags.arrayelem_isunion; + bool isatomic = kind == (jl_value_t*)jl_atomic_sym; + bool needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + size_t elsz = layout->size; + size_t al = layout->alignment; + if (isatomic == (order == jl_memory_order_notatomic)) { + emit_atomic_error(ctx, + issetmemory ? + (isatomic ? "memoryrefset!: atomic memory cannot be written non-atomically" + : "memoryrefset!: non-atomic memory cannot be written atomically") : + isreplacememory ? + (isatomic ? "memoryrefreplace!: atomic memory cannot be written non-atomically" + : "memoryrefreplace!: non-atomic memory cannot be written atomically") : + isswapmemory ? + (isatomic ? "memoryrefswap!: atomic memory cannot be written non-atomically" + : "memoryrefswap!: non-atomic memory cannot be written atomically") : + ismodifymemory ? + (isatomic ? "memoryrefmodify!: atomic memory cannot be written non-atomically" + : "memoryrefmodify!: non-atomic memory cannot be written atomically") : + (isatomic ? "memoryrefsetonce!: atomic memory cannot be written non-atomically" + : "memoryrefsetonce!: non-atomic memory cannot be written atomically")); + *ret = jl_cgval_t(); + return true; + } + else if (isatomic == (fail_order == jl_memory_order_notatomic)) { + emit_atomic_error(ctx, + isreplacememory ? + (isatomic ? "memoryrefreplace!: atomic memory cannot be accessed non-atomically" + : "memoryrefreplace!: non-atomic memory cannot be accessed atomically") : + (isatomic ? "memoryrefsetonce!: atomic memory cannot be accessed non-atomically" + : "memoryrefsetonce!: non-atomic memory cannot be accessed atomically")); + *ret = jl_cgval_t(); + return true; + } + Value *mem = emit_memoryref_mem(ctx, ref, layout); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + if (bounds_check_enabled(ctx, boundscheck)) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "load"); + ctx.builder.CreateCondBr(ctx.builder.CreateIsNull(mlen), failBB, endBB); + failBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(ctx, mem), ConstantInt::get(ctx.types().T_size, 1) }); + ctx.builder.CreateUnreachable(); + endBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(endBB); + } + if (!ismodifymemory) { + emit_typecheck(ctx, val, ety, fname); + val = update_julia_type(ctx, val, ety); + if (val.typ == jl_bottom_type) + return true; + } + AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) + ? AtomicOrdering::NotAtomic + : get_llvm_atomic_order(order); + AtomicOrdering FailOrder = (needlock || fail_order <= jl_memory_order_notatomic) + ? AtomicOrdering::NotAtomic + : get_llvm_atomic_order(fail_order); + if (isunion) { + assert(!isatomic && !needlock); + Value *V = emit_memoryref_FCA(ctx, ref, layout); + Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); + Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); + Value *data = emit_genericmemoryptr(ctx, mem, layout, AddressSpace::Loaded); + Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); + data = emit_bitcast(ctx, data, AT->getPointerTo()); + // compute tindex from val + Value *ptindex; + if (elsz == 0) { + ptindex = data; + } + else { + data = emit_bitcast(ctx, data, AT->getPointerTo()); + // isbits union selector bytes are stored after mem->length + ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); + data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); + } + ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); + ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); + *ret = union_store(ctx, data, ptindex, val, cmp, ety, + ctx.tbaa().tbaa_arraybuf, nullptr, ctx.tbaa().tbaa_arrayselbyte, + Order, FailOrder, + nullptr, issetmemory, isreplacememory, isswapmemory, ismodifymemory, issetmemoryonce, + modifyop, fname); + } + else { + Value *ptr = (layout->size == 0 ? nullptr : emit_memoryref_ptr(ctx, ref, layout)); + Value *lock = nullptr; + bool maybenull = true; + if (needlock) { + assert(ptr); + lock = ptr; + // ptr += sizeof(lock); + ptr = emit_bitcast(ctx, ptr, getInt8PtrTy(ctx.builder.getContext())); + ptr = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), ptr, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); + } + Value *data_owner = NULL; // owner object against which the write barrier must check + if (isboxed || layout->first_ptr >= 0) { // if elements are just bits, don't need a write barrier + Value *V = emit_memoryref_FCA(ctx, ref, layout); + data_owner = emit_genericmemoryowner(ctx, CreateSimplifiedExtractValue(ctx, V, 1)); + } + *ret = typed_store(ctx, + ptr, + val, cmp, ety, + isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, + ctx.noalias().aliasscope.current, + data_owner, + isboxed, + Order, + FailOrder, + al, + lock, + issetmemory, + isreplacememory, + isswapmemory, + ismodifymemory, + issetmemoryonce, + maybenull, + modifyop, + fname, + nullptr, + nullptr); + } + return true; +} + static jl_llvm_functions_t emit_function( orc::ThreadSafeModule &TSM, @@ -3566,7 +3897,7 @@ static jl_llvm_functions_t static void emit_hasnofield_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *type, jl_cgval_t name); static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, - const jl_cgval_t *argv, size_t nargs, jl_value_t *rt, + ArrayRef argv, size_t nargs, jl_value_t *rt, jl_expr_t *ex, bool is_promotable) // returns true if the call has been handled { @@ -3652,7 +3983,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } if (jl_is_tuple_type(rt) && jl_is_concrete_type(rt) && nargs == jl_datatype_nfields(rt)) { - *ret = emit_new_struct(ctx, rt, nargs, &argv[1], is_promotable); + *ret = emit_new_struct(ctx, rt, nargs, argv.drop_front(), is_promotable); return true; } } @@ -3704,13 +4035,40 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { - jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO + jl_value_t *kind = jl_tparam0(mty_dt); jl_value_t *ety = jl_tparam1(mty_dt); jl_value_t *addrspace = jl_tparam2(mty_dt); (void)addrspace; // TODO mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); - jl_value_t *order = argv[2].constant; - if (order != (jl_value_t*)jl_not_atomic_sym) + if (kind != (jl_value_t*)jl_not_atomic_sym && kind != (jl_value_t*)jl_atomic_sym) return false; + enum jl_memory_order order = jl_memory_order_unspecified; + const std::string fname = "memoryrefget"; + { + const jl_cgval_t &ord = argv[2]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + } + if (order == jl_memory_order_invalid) { + emit_atomic_error(ctx, "invalid atomic ordering"); + *ret = jl_cgval_t(); // unreachable + return true; + } + bool isatomic = kind == (jl_value_t*)jl_atomic_sym; + if (!isatomic && order != jl_memory_order_notatomic && order != jl_memory_order_unspecified) { + emit_atomic_error(ctx, "memoryrefget: non-atomic memory cannot be accessed atomically"); + *ret = jl_cgval_t(); // unreachable + return true; + } + if (isatomic && order == jl_memory_order_notatomic) { + emit_atomic_error(ctx, "memoryrefget: atomic memory cannot be accessed non-atomically"); + *ret = jl_cgval_t(); // unreachable + return true; + } + if (order == jl_memory_order_unspecified) { + order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; + } jl_value_t *boundscheck = argv[3].constant; emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; @@ -3732,11 +4090,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, bool isunion = layout->flags.arrayelem_isunion; size_t elsz = layout->size; size_t al = layout->alignment; + bool needlock = isatomic && !isboxed && elsz > MAX_ATOMIC_SIZE; + AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) + ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) + : get_llvm_atomic_order(order); + bool maybenull = true; if (!isboxed && !isunion && elsz == 0) { assert(jl_is_datatype(ety) && jl_is_datatype_singleton((jl_datatype_t*)ety)); *ret = ghostValue(ctx, ety); + if (isStrongerThanMonotonic(Order)) + ctx.builder.CreateFence(Order); } else if (isunion) { + assert(!isatomic && !needlock); Value *V = emit_memoryref_FCA(ctx, ref, layout); Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); @@ -3760,139 +4126,85 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, *ret = emit_unionload(ctx, data, ptindex, ety, elsz_c, al, ctx.tbaa().tbaa_arraybuf, true, union_max, ctx.tbaa().tbaa_arrayselbyte); } else { - *ret = typed_load(ctx, - emit_memoryref_ptr(ctx, ref, layout), - nullptr, ety, + Value *ptr = (layout->size == 0 ? nullptr : emit_memoryref_ptr(ctx, ref, layout)); + Value *lock = nullptr; + if (needlock) { + assert(ptr); + lock = ptr; + // ptr += sizeof(lock); + ptr = emit_bitcast(ctx, ptr, getInt8PtrTy(ctx.builder.getContext())); + ptr = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), ptr, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); + emit_lockstate_value(ctx, lock, true); + } + *ret = typed_load(ctx, ptr, nullptr, ety, isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, ctx.noalias().aliasscope.current, - isboxed, - AtomicOrdering::NotAtomic); + isboxed, Order, maybenull, al); + if (needlock) { + emit_lockstate_value(ctx, lock, false); + } } return true; } } - else if (f == jl_builtin_memoryrefset && nargs == 4) { + else if ((f == jl_builtin_memoryrefset && nargs == 4) || + (f == jl_builtin_memoryrefswap && nargs == 4) || + (f == jl_builtin_memoryrefreplace && nargs == 6) || + (f == jl_builtin_memoryrefmodify && nargs == 5) || + (f == jl_builtin_memoryrefsetonce && nargs == 5)) { + return emit_f_opmemory(ctx, ret, f, argv, nargs, nullptr); + } + + + else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { const jl_cgval_t &ref = argv[1]; - jl_cgval_t val = argv[2]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { - jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO - jl_value_t *ety = jl_tparam1(mty_dt); - jl_value_t *addrspace = jl_tparam2(mty_dt); (void)addrspace; // TODO + jl_value_t *kind = jl_tparam0(mty_dt); mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); - jl_value_t *order = argv[3].constant; - if (order != (jl_value_t*)jl_not_atomic_sym) + if (kind != (jl_value_t*)jl_not_atomic_sym && kind != (jl_value_t*)jl_atomic_sym) return false; - jl_value_t *boundscheck = argv[4].constant; - emit_typecheck(ctx, argv[4], (jl_value_t*)jl_bool_type, "memoryset"); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; - Value *mem = emit_memoryref_mem(ctx, ref, layout); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); - if (bounds_check_enabled(ctx, boundscheck)) { - BasicBlock *failBB, *endBB; - failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); - endBB = BasicBlock::Create(ctx.builder.getContext(), "load"); - ctx.builder.CreateCondBr(ctx.builder.CreateIsNull(mlen), failBB, endBB); - failBB->insertInto(ctx.f); - ctx.builder.SetInsertPoint(failBB); - ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(ctx, mem), ConstantInt::get(ctx.types().T_size, 1) }); - ctx.builder.CreateUnreachable(); - endBB->insertInto(ctx.f); - ctx.builder.SetInsertPoint(endBB); + enum jl_memory_order order = jl_memory_order_unspecified; + const std::string fname = "memoryref_isassigned"; + { + const jl_cgval_t &ord = argv[2]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); } - emit_typecheck(ctx, val, ety, "memoryset"); - val = update_julia_type(ctx, val, ety); - if (val.typ == jl_bottom_type) + if (order == jl_memory_order_invalid) { + emit_atomic_error(ctx, "invalid atomic ordering"); + *ret = jl_cgval_t(); // unreachable return true; - bool isboxed = layout->flags.arrayelem_isboxed; - bool isunion = layout->flags.arrayelem_isunion; - size_t elsz = layout->size; - size_t al = layout->alignment; - if (isboxed) - ety = (jl_value_t*)jl_any_type; - if (!isboxed && !isunion && elsz == 0) { - assert(jl_is_datatype(ety) && jl_datatype_size(ety) == 0); - // no-op } - else { - Value *V = emit_memoryref_FCA(ctx, ref, layout); - Value *data_owner = NULL; // owner object against which the write barrier must check - if (isboxed || layout->first_ptr >= 0) { // if elements are just bits, don't need a write barrier - data_owner = emit_genericmemoryowner(ctx, CreateSimplifiedExtractValue(ctx, V, 1)); - } - if (isunion) { - Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); - Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); - Value *data = emit_genericmemoryptr(ctx, mem, layout, AddressSpace::Loaded); - Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); - data = emit_bitcast(ctx, data, AT->getPointerTo()); - // compute tindex from val - jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety); - Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety); - tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); - Value *ptindex; - if (elsz == 0) { - ptindex = data; - } - else { - data = emit_bitcast(ctx, data, AT->getPointerTo()); - // isbits union selector bytes are stored after mem->length - ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); - data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); - } - ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); - ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayselbyte); - ai.decorateInst(ctx.builder.CreateStore(tindex, ptindex)); - if (elsz > 0 && (!jl_is_datatype(val.typ) || jl_datatype_size(val.typ) > 0)) { - // copy data (if any) - emit_unionmove(ctx, data, ctx.tbaa().tbaa_arraybuf, val, nullptr); - } - } - else { - typed_store(ctx, - emit_memoryref_ptr(ctx, ref, layout), - nullptr, val, jl_cgval_t(), ety, - isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, - ctx.noalias().aliasscope.current, - data_owner, - isboxed, - isboxed ? AtomicOrdering::Release : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - /*FailOrder*/AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - 0, - false, - true, - false, - false, - false, - false, - nullptr, - ""); - } + bool isatomic = kind == (jl_value_t*)jl_atomic_sym; + if (!isatomic && order != jl_memory_order_notatomic && order != jl_memory_order_unspecified) { + emit_atomic_error(ctx, "memoryref_isassigned: non-atomic memory cannot be accessed atomically"); + *ret = jl_cgval_t(); // unreachable + return true; + } + if (isatomic && order == jl_memory_order_notatomic) { + emit_atomic_error(ctx, "memoryref_isassigned: atomic memory cannot be accessed non-atomically"); + *ret = jl_cgval_t(); // unreachable + return true; + } + if (order == jl_memory_order_unspecified) { + order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } - *ret = ref; - return true; - } - } - - else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { - const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); - if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { - jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO - mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); - jl_value_t *order = argv[2].constant; - if (order != (jl_value_t*)jl_not_atomic_sym) - return false; jl_value_t *boundscheck = argv[3].constant; - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memory_isassigned"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, fname); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; bool isboxed = layout->flags.arrayelem_isboxed; if (isboxed || layout->first_ptr >= 0) { + bool needlock = isatomic && !isboxed && layout->size > MAX_ATOMIC_SIZE; + AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) + ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) + : get_llvm_atomic_order(order); PHINode *result = nullptr; if (oob) { BasicBlock *passBB, *endBB, *fromBB; @@ -3909,6 +4221,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ctx.builder.SetInsertPoint(passBB); } Value *elem = emit_memoryref_ptr(ctx, ref, layout); + if (needlock) { + // n.b. no actual lock acquire needed, as the check itself only needs to load a single pointer and check for null + // elem += sizeof(lock); + elem = emit_bitcast(ctx, elem, getInt8PtrTy(ctx.builder.getContext())); + elem = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), elem, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); + } elem = emit_bitcast(ctx, elem, ctx.types().T_pprjlvalue); if (!isboxed) elem = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, elem, layout->first_ptr); @@ -3917,10 +4235,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); LoadInst *fldv = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, elem, ctx.types().alignof_ptr); - //fldv->setOrdering(AtomicOrdering::Unordered); + fldv->setOrdering(Order); ai.decorateInst(fldv); Value *isdef = ctx.builder.CreateIsNotNull(fldv); - setName(ctx.emission_context, isdef, "memoryref_isassigned"); + setName(ctx.emission_context, isdef, fname); if (oob) { assert(result); result->addIncoming(isdef, ctx.builder.CreateBr(result->getParent())->getParent()); @@ -4134,14 +4452,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if (f == jl_builtin_setglobal && (nargs == 3 || nargs == 4)) { + else if ((f == jl_builtin_setglobal && (nargs == 3 || nargs == 4)) || + (f == jl_builtin_swapglobal && (nargs == 3 || nargs == 4)) || + (f == jl_builtin_replaceglobal && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == jl_builtin_modifyglobal && (nargs == 4 || nargs == 5)) || + (f == jl_builtin_setglobalonce && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opglobal(ctx, ret, f, argv, nargs, nullptr); } else if ((f == jl_builtin_setfield && (nargs == 3 || nargs == 4)) || (f == jl_builtin_swapfield && (nargs == 3 || nargs == 4)) || (f == jl_builtin_replacefield && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5))) { + (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5)) || + (f == jl_builtin_setfieldonce && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opfield(ctx, ret, f, argv, nargs, nullptr); } @@ -4413,7 +4736,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // Returns ctx.types().T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, FunctionCallee theFptr, Value *theF, - const jl_cgval_t *argv, size_t nargs, JuliaFunction<> *trampoline) + ArrayRef argv, size_t nargs, JuliaFunction<> *trampoline) { ++EmittedJLCalls; Function *TheTrampoline = prepare_call(trampoline); @@ -4434,13 +4757,13 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, FunctionCallee theFptr, Value *t // Returns ctx.types().T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value *theF, - const jl_cgval_t *argv, size_t nargs, JuliaFunction<> *trampoline) + ArrayRef argv, size_t nargs, JuliaFunction<> *trampoline) { return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, trampoline); } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, llvm::Value *callee, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - const jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { ++EmittedSpecfunCalls; // emit specialized call site @@ -4589,7 +4912,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - const jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; return emit_call_specfun_other(ctx, is_opaque_closure, mi->specTypes, jlretty, NULL, @@ -4597,7 +4920,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - const jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) { Value *theFptr; if (fromexternal) { @@ -4638,10 +4961,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv.data(), nargs, rt); + return emit_invoke(ctx, lival, argv, nargs, rt); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const jl_cgval_t *argv, size_t nargs, jl_value_t *rt) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) { ++EmittedInvokes; bool handled = false; @@ -4760,27 +5083,45 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ return jl_cgval_t(); } const jl_cgval_t &f = argv[0]; - jl_cgval_t ret; - if (f.constant && f.constant == jl_builtin_modifyfield) { - if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv.data(), nargs - 1, &lival)) - return ret; - auto it = builtin_func_map().find(jl_f_modifyfield_addr); - assert(it != builtin_func_map().end()); - Value *oldnew = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), &argv[1], nargs - 1, julia_call); - return mark_julia_type(ctx, oldnew, true, rt); - } - if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { - JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); - if (fi == JL_I::atomic_pointermodify && jl_intrinsic_nargs((int)fi) == nargs - 1) - return emit_atomic_pointerop(ctx, fi, argv.data(), nargs - 1, &lival); + if (f.constant) { + jl_cgval_t ret; + auto it = builtin_func_map().end(); + if (f.constant == jl_builtin_modifyfield) { + if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv, nargs - 1, &lival)) + return ret; + it = builtin_func_map().find(jl_f_modifyfield_addr); + assert(it != builtin_func_map().end()); + } + else if (f.constant == jl_builtin_modifyglobal) { + if (emit_f_opglobal(ctx, &ret, jl_builtin_modifyglobal, argv, nargs - 1, &lival)) + return ret; + it = builtin_func_map().find(jl_f_modifyglobal_addr); + assert(it != builtin_func_map().end()); + } + else if (f.constant == jl_builtin_memoryrefmodify) { + if (emit_f_opmemory(ctx, &ret, jl_builtin_memoryrefmodify, argv, nargs - 1, &lival)) + return ret; + it = builtin_func_map().find(jl_f_memoryrefmodify_addr); + assert(it != builtin_func_map().end()); + } + else if (jl_typetagis(f.constant, jl_intrinsic_type)) { + JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); + if (fi == JL_I::atomic_pointermodify && jl_intrinsic_nargs((int)fi) == nargs - 1) + return emit_atomic_pointerop(ctx, fi, ArrayRef(argv).drop_front(), nargs - 1, &lival); + } + + if (it != builtin_func_map().end()) { + Value *oldnew = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); + return mark_julia_type(ctx, oldnew, true, rt); + } } // emit function and arguments - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv.data(), nargs, julia_call); + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call); return mark_julia_type(ctx, callval, true, rt); } -static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, jl_value_t *sigtype, jl_cgval_t *argv, size_t nargs) +static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, jl_value_t *sigtype, MutableArrayRef argv /*n.b. this mutation is unusual */, size_t nargs) { jl_datatype_t *oc_argt = (jl_datatype_t *)jl_tparam0(oc_type); jl_value_t *oc_rett = jl_tparam1(oc_type); @@ -4827,8 +5168,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo size_t n_generic_args = nargs; - SmallVector generic_argv(n_generic_args); - jl_cgval_t *argv = generic_argv.data(); + SmallVector argv(n_generic_args); argv[0] = f; for (size_t i = 1; i < nargs; ++i) { @@ -4849,7 +5189,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo // special case for some known builtin not handled by emit_builtin_call auto it = builtin_func_map().find(jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(f.constant))); if (it != builtin_func_map().end()) { - Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), &argv[1], nargs - 1, julia_call); + Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); setName(ctx.emission_context, ret, it->second->name + "_ret"); return mark_julia_type(ctx, ret, true, rt); } @@ -4868,7 +5208,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo F = boxed(ctx, f); cc = julia_call; } - Value *ret = emit_jlcall(ctx, fptr, F, &argv[1], nargs - 1, cc); + Value *ret = emit_jlcall(ctx, fptr, F, ArrayRef(argv).drop_front(), nargs - 1, cc); setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); } @@ -4889,27 +5229,12 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo } // emit function and arguments - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, generic_argv.data(), n_generic_args, julia_call); + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, n_generic_args, julia_call); return mark_julia_type(ctx, callval, true, rt); } // --- accessing and assigning variables --- -static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name, jl_value_t *scope) -{ - ++EmittedUndefVarErrors; - BasicBlock *err = BasicBlock::Create(ctx.builder.getContext(), "err", ctx.f); - BasicBlock *ifok = BasicBlock::Create(ctx.builder.getContext(), "ok"); - ctx.builder.CreateCondBr(ok, ifok, err); - ctx.builder.SetInsertPoint(err); - ctx.builder.CreateCall(prepare_call(jlundefvarerror_func), { - mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)name)), - mark_callee_rooted(ctx, literal_pointer_val(ctx, scope))}); - ctx.builder.CreateUnreachable(); - ifok->insertInto(ctx.f); - ctx.builder.SetInsertPoint(ifok); -} - static void emit_hasnofield_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *type, jl_cgval_t name) { ++EmittedUndefVarErrors; @@ -5483,7 +5808,8 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r, ssi mod = jl_globalref_mod(l); sym = jl_globalref_name(l); } - emit_globalset(ctx, mod, sym, rval_info, AtomicOrdering::Release); + emit_globalop(ctx, mod, sym, rval_info, jl_cgval_t(), AtomicOrdering::Release, AtomicOrdering::NotAtomic, + true, false, false, false, false, nullptr); // Global variable. Does not need debug info because the debugger knows about // its memory location. } @@ -5950,12 +6276,12 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ jl_is_datatype(jl_tparam0(ty)) && jl_is_concrete_type(jl_tparam0(ty))) { assert(nargs <= jl_datatype_nfields(jl_tparam0(ty)) + 1); - jl_cgval_t res = emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, argv.data() + 1, is_promotable); + jl_cgval_t res = emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, ArrayRef(argv).drop_front(), is_promotable); if (is_promotable && res.promotion_point && res.promotion_ssa==-1) res.promotion_ssa = ssaidx_0based; return res; } - Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv.data(), nargs, julia_call); + Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv, nargs, julia_call); // temporarily mark as `Any`, expecting `emit_ssaval_assign` to update // it to the inferred type. return mark_julia_type(ctx, val, true, (jl_value_t*)jl_any_type); @@ -6022,7 +6348,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ fptr = mark_julia_type(ctx, Constant::getNullValue(ctx.types().T_size), false, jl_voidpointer_type); // TODO: Inline the env at the end of the opaque closure and generate a descriptor for GC - jl_cgval_t env = emit_new_struct(ctx, env_t, nargs-4, &argv.data()[4]); + jl_cgval_t env = emit_new_struct(ctx, env_t, nargs-4, ArrayRef(argv).drop_front(4)); jl_cgval_t closure_fields[5] = { env, @@ -6043,7 +6369,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } return mark_julia_type(ctx, - emit_jlcall(ctx, jl_new_opaque_closure_jlcall_func, Constant::getNullValue(ctx.types().T_prjlvalue), argv.data(), nargs, julia_call), + emit_jlcall(ctx, jl_new_opaque_closure_jlcall_func, Constant::getNullValue(ctx.types().T_prjlvalue), argv, nargs, julia_call), true, jl_any_type); } else if (head == jl_exc_sym) { @@ -6293,7 +6619,7 @@ static void emit_cfunc_invalidate( } } assert(AI == gf_thunk->arg_end()); - Value *gf_ret = emit_jlcall(ctx, target, nullptr, myargs.data(), nargs, julia_call); + Value *gf_ret = emit_jlcall(ctx, target, nullptr, myargs, nargs, julia_call); jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type); if (cc != jl_returninfo_t::Boxed) { emit_typecheck(ctx, gf_retbox, rettype, "cfunction"); @@ -6703,11 +7029,11 @@ static Function* gen_cfun_wrapper( // for jlcall, we need to pass the function object even if it is a ghost. Value *theF = boxed(ctx, inputargs[0]); assert(theF); - ret_jlcall = emit_jlcall(ctx, theFptr, theF, &inputargs[1], nargs, julia_call); + ret_jlcall = emit_jlcall(ctx, theFptr, theF, ArrayRef(inputargs).drop_front(), nargs, julia_call); ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_generic); } - Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs.data(), nargs + 1, julia_call); + Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); if (age_ok) { ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_after); @@ -8275,12 +8601,12 @@ static jl_llvm_functions_t vargs[i - nreq] = get_specsig_arg(argType, llvmArgType, isboxed); } if (jl_is_concrete_type(vi.value.typ)) { - jl_cgval_t tuple = emit_new_struct(ctx, vi.value.typ, ctx.nvargs, vargs.data()); + jl_cgval_t tuple = emit_new_struct(ctx, vi.value.typ, ctx.nvargs, vargs); emit_varinfo_assign(ctx, vi, tuple); } else { restTuple = emit_jlcall(ctx, jltuple_func, Constant::getNullValue(ctx.types().T_prjlvalue), - vargs.data(), ctx.nvargs, julia_call); + vargs, ctx.nvargs, julia_call); jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, vi.value.typ); emit_varinfo_assign(ctx, vi, tuple); } @@ -9518,6 +9844,10 @@ static void init_jit_functions(void) add_named_global(except_enter_func, (void*)NULL); add_named_global(julia_call, (void*)NULL); add_named_global(julia_call2, (void*)NULL); + add_named_global(jllockvalue_func, &jl_lock_value); + add_named_global(jlunlockvalue_func, &jl_unlock_value); + add_named_global(jllockfield_func, &jl_lock_field); + add_named_global(jlunlockfield_func, &jl_unlock_field); #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) diff --git a/src/datatype.c b/src/datatype.c index 7a04bfd7f2046..f9f7dec3d1b13 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -488,7 +488,7 @@ static int is_type_identityfree(jl_value_t *t) // make a copy of the layout of st, but with nfields=0 void jl_get_genericmemory_layout(jl_datatype_t *st) { - jl_value_t *isatomic = jl_tparam0(st); + jl_value_t *kind = jl_tparam0(st); jl_value_t *eltype = jl_tparam1(st); jl_value_t *addrspace = jl_tparam2(st); if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) { @@ -499,18 +499,25 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) return; } - size_t elsz = 0, al = 0; - int isunboxed = jl_islayout_inline(eltype, &elsz, &al); + size_t elsz = 0, al = 1; + int isunboxed = jl_islayout_inline(eltype, &elsz, &al) && (kind != (jl_value_t*)jl_atomic_sym || jl_is_datatype(eltype)); int isunion = isunboxed && jl_is_uniontype(eltype); - int haspadding = 1; // we may want to eventually actually compute this + int haspadding = 1; // we may want to eventually actually compute this more precisely int nfields = 0; // aka jl_is_layout_opaque int npointers = 1; int zi; uint32_t first_ptr = -1; uint32_t *pointers = &first_ptr; + int needlock = 0; if (isunboxed) { elsz = LLT_ALIGN(elsz, al); + if (kind == (jl_value_t*)jl_atomic_sym) { + if (elsz > MAX_ATOMIC_SIZE) + needlock = 1; + else if (elsz > 0) + al = elsz = next_power_of_two(elsz); + } if (isunion) { zi = 1; } @@ -532,6 +539,13 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) } } } + if (needlock) { + assert(al <= JL_SMALL_BYTE_ALIGNMENT); + size_t offset = LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); + elsz += offset; + haspadding = 1; + zi = 1; + } } else { elsz = sizeof(void*); @@ -554,13 +568,15 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) //st->ismutationfree = 0; //st->isidentityfree = 0; - if (isatomic == (jl_value_t*)jl_not_atomic_sym && jl_is_addrspacecore(addrspace) && jl_unbox_uint8(addrspace) == 0) { - jl_genericmemory_t *zeroinst = (jl_genericmemory_t*)jl_gc_permobj(LLT_ALIGN(sizeof(jl_genericmemory_t), JL_SMALL_BYTE_ALIGNMENT) + (elsz ? elsz : isunion), st); - zeroinst->length = 0; - zeroinst->ptr = (char*)zeroinst + JL_SMALL_BYTE_ALIGNMENT; - memset(zeroinst->ptr, 0, elsz ? elsz : isunion); - assert(!st->instance); - st->instance = (jl_value_t*)zeroinst; + if (jl_is_addrspacecore(addrspace) && jl_unbox_uint8(addrspace) == 0) { + if (kind == (jl_value_t*)jl_not_atomic_sym || kind == (jl_value_t*)jl_atomic_sym) { + jl_genericmemory_t *zeroinst = (jl_genericmemory_t*)jl_gc_permobj(LLT_ALIGN(sizeof(jl_genericmemory_t), JL_SMALL_BYTE_ALIGNMENT) + (elsz ? elsz : isunion), st); + zeroinst->length = 0; + zeroinst->ptr = (char*)zeroinst + JL_SMALL_BYTE_ALIGNMENT; + memset(zeroinst->ptr, 0, elsz ? elsz : isunion); + assert(!st->instance); + st->instance = (jl_value_t*)zeroinst; + } } } @@ -1060,6 +1076,7 @@ JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, const void *data) assert(!bt->smalltag); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, nb, bt); + // TODO: make this a memmove_refs if relevant memcpy(jl_assume_aligned(v, sizeof(void*)), data, nb); return v; } @@ -1214,13 +1231,10 @@ JL_DLLEXPORT int jl_atomic_bool_cmpswap_bits(char *dst, const jl_value_t *expect return success; } -JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t *rettyp, char *dst, const jl_value_t *expected, const jl_value_t *src, int nb) +JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre-allocated output */, char *dst, const jl_value_t *expected, const jl_value_t *src, int nb) { // dst must have the required alignment for an atomic of the given size // n.b.: this does not spuriously fail if there are padding bits - jl_task_t *ct = jl_current_task; - int isptr = jl_field_isptr(rettyp, 0); - jl_value_t *y = jl_gc_alloc(ct->ptls, isptr ? nb : jl_datatype_size(rettyp), isptr ? dt : rettyp); int success; jl_datatype_t *et = (jl_datatype_t*)jl_typeof(expected); if (nb == 0) { @@ -1307,16 +1321,54 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t else { abort(); } - if (isptr) { - JL_GC_PUSH1(&y); - jl_value_t *z = jl_gc_alloc(ct->ptls, jl_datatype_size(rettyp), rettyp); - *(jl_value_t**)z = y; - JL_GC_POP(); - y = z; - nb = sizeof(jl_value_t*); + return success; +} + +JL_DLLEXPORT int jl_atomic_storeonce_bits(jl_datatype_t *dt, char *dst, const jl_value_t *src, int nb) +{ + // dst must have the required alignment for an atomic of the given size + // n.b.: this does not spuriously fail + // n.b.: hasptr == 1 therefore nb >= sizeof(void*), because ((jl_datatype_t*)ty)->layout->has_ptr >= 0 + int success; +#ifdef _P64 + if (nb <= 4) { + uint32_t y32 = 0; + uint32_t z32 = zext_read32(src, nb); + success = jl_atomic_cmpswap((_Atomic(uint32_t)*)dst, &y32, z32); } - *((uint8_t*)y + nb) = success ? 1 : 0; - return y; +#if MAX_POINTERATOMIC_SIZE >= 8 + else if (nb <= 8) { + uint64_t y64 = 0; + uint64_t z64 = zext_read64(src, nb); + while (1) { + success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, &y64, z64); + if (success || undefref_check(dt, (jl_value_t*)&y64) != NULL) + break; + } + } +#endif +#else + if (nb <= 8) { + uint64_t y64 = 0; + uint64_t z64 = zext_read64(src, nb); + success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, &y64, z64); + } +#endif +#if MAX_POINTERATOMIC_SIZE >= 16 + else if (nb <= 16) { + jl_uint128_t y128 = 0; + jl_uint128_t z128 = zext_read128(src, nb); + while (1) { + success = jl_atomic_cmpswap((_Atomic(jl_uint128_t)*)dst, &y128, z128); + if (success || undefref_check(dt, (jl_value_t*)&y128) != NULL) + break; + } + } +#endif + else { + abort(); + } + return success; } #define PERMBOXN_FUNC(nb) \ @@ -1607,14 +1659,51 @@ JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) // field access --------------------------------------------------------------- -JL_DLLEXPORT void jl_lock_value(jl_value_t *v) JL_NOTSAFEPOINT +// TODO(jwn): these lock/unlock pairs must be full seq-cst fences +JL_DLLEXPORT void jl_lock_value(jl_mutex_t *v) JL_NOTSAFEPOINT +{ + JL_LOCK_NOGC(v); +} + +JL_DLLEXPORT void jl_unlock_value(jl_mutex_t *v) JL_NOTSAFEPOINT +{ + JL_UNLOCK_NOGC(v); +} + +JL_DLLEXPORT void jl_lock_field(jl_mutex_t *v) JL_NOTSAFEPOINT { - JL_LOCK_NOGC((jl_mutex_t*)v); + JL_LOCK_NOGC(v); +} + +JL_DLLEXPORT void jl_unlock_field(jl_mutex_t *v) JL_NOTSAFEPOINT +{ + JL_UNLOCK_NOGC(v); +} + +static inline char *lock(char *p, jl_value_t *parent, int needlock, enum atomic_kind isatomic) JL_NOTSAFEPOINT +{ + if (needlock) { + if (isatomic == isatomic_object) { + jl_lock_value((jl_mutex_t*)parent); + } + else { + jl_lock_field((jl_mutex_t*)p); + return p + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); + } + } + return p; } -JL_DLLEXPORT void jl_unlock_value(jl_value_t *v) JL_NOTSAFEPOINT +static inline void unlock(char *p, jl_value_t *parent, int needlock, enum atomic_kind isatomic) JL_NOTSAFEPOINT { - JL_UNLOCK_NOGC((jl_mutex_t*)v); + if (needlock) { + if (isatomic == isatomic_object) { + jl_unlock_value((jl_mutex_t*)parent); + } + else { + jl_unlock_field((jl_mutex_t*)p); + } + } } JL_DLLEXPORT int jl_field_index(jl_datatype_t *t, jl_sym_t *fld, int err) @@ -1672,11 +1761,12 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i) else if (needlock) { jl_task_t *ct = jl_current_task; r = jl_gc_alloc(ct->ptls, fsz, ty); - jl_lock_value(v); + jl_lock_value((jl_mutex_t*)v); memcpy((char*)r, (char*)v + offs, fsz); - jl_unlock_value(v); + jl_unlock_value((jl_mutex_t*)v); } else { + // TODO: a finalizer here could make the isunion case not quite right r = jl_new_bits(ty, (char*)v + offs); } return undefref_check((jl_datatype_t*)ty, r); @@ -1699,30 +1789,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) return r; } -static inline void memassign_safe(int hasptr, jl_value_t *parent, char *dst, const jl_value_t *src, size_t nb) JL_NOTSAFEPOINT -{ - if (hasptr) { - // assert that although dst might have some undefined bits, the src heap box should be okay with that - assert(LLT_ALIGN(nb, sizeof(void*)) == LLT_ALIGN(jl_datatype_size(jl_typeof(src)), sizeof(void*))); - size_t nptr = nb / sizeof(void*); - memmove_refs((_Atomic(void*)*)dst, (_Atomic(void*)*)src, nptr); - jl_gc_multi_wb(parent, src); - src = (jl_value_t*)((char*)src + nptr * sizeof(void*)); - dst = dst + nptr * sizeof(void*); - nb -= nptr * sizeof(void*); - } - else { - // src must be a heap box. - assert(nb == jl_datatype_size(jl_typeof(src))); - if (nb >= 16) { - memcpy(dst, jl_assume_aligned(src, 16), nb); - return; - } - } - memcpy(dst, jl_assume_aligned(src, sizeof(void*)), nb); -} - -void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic) JL_NOTSAFEPOINT +inline void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic) JL_NOTSAFEPOINT { size_t offs = jl_field_offset(st, i); if (rhs == NULL) { // TODO: this should be invalid, but it happens frequently in ircode.c @@ -1751,24 +1818,75 @@ void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, hasptr = 0; } else { - hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0; + hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0; } size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy + assert(!isatomic || jl_typeis(rhs, ty)); int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); if (isatomic && !needlock) { jl_atomic_store_bits((char*)v + offs, rhs, fsz); - if (hasptr) - jl_gc_multi_wb(v, rhs); // rhs is immutable } else if (needlock) { - jl_lock_value(v); + jl_lock_value((jl_mutex_t*)v); memcpy((char*)v + offs, (char*)rhs, fsz); - jl_unlock_value(v); + jl_unlock_value((jl_mutex_t*)v); + } + else { + memassign_safe(hasptr, (char*)v + offs, rhs, fsz); + } + if (hasptr) + jl_gc_multi_wb(v, rhs); // rhs is immutable + } +} + +inline jl_value_t *swap_bits(jl_value_t *ty, char *v, uint8_t *psel, jl_value_t *parent, jl_value_t *rhs, enum atomic_kind isatomic) +{ + jl_value_t *rty = jl_typeof(rhs); + int hasptr; + int isunion = psel != NULL; + if (isunion) { + assert(!isatomic); + hasptr = 0; + } + else { + hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0; + } + size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy + int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); + assert(!isatomic || jl_typeis(rhs, ty)); + jl_value_t *r; + if (isatomic && !needlock) { + r = jl_atomic_swap_bits(rty, v, rhs, fsz); + } + else { + if (needlock) { + jl_task_t *ct = jl_current_task; + r = jl_gc_alloc(ct->ptls, fsz, ty); + char *px = lock(v, parent, needlock, isatomic); + memcpy((char*)r, px, fsz); + memcpy(px, (char*)rhs, fsz); + unlock(v, parent, needlock, isatomic); } else { - memassign_safe(hasptr, v, (char*)v + offs, rhs, fsz); + r = jl_new_bits(isunion ? jl_nth_union_component(ty, *psel) : ty, v); + if (isunion) { + unsigned nth = 0; + if (!jl_find_union_component(ty, rty, &nth)) + assert(0 && "invalid field assignment to isbits union"); + *psel = nth; + if (jl_is_datatype_singleton((jl_datatype_t*)rty)) + return r; + } + memassign_safe(hasptr, v, rhs, fsz); } } + if (!isunion) + r = undefref_check((jl_datatype_t*)ty, r); + if (hasptr) + jl_gc_multi_wb(parent, rhs); // rhs is immutable + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + return r; } jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic) @@ -1778,139 +1896,139 @@ jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_ jl_type_error("swapfield!", ty, rhs); size_t offs = jl_field_offset(st, i); jl_value_t *r; + char *p = (char*)v + offs; if (jl_field_isptr(st, i)) { if (isatomic) - r = jl_atomic_exchange((_Atomic(jl_value_t*)*)((char*)v + offs), rhs); + r = jl_atomic_exchange((_Atomic(jl_value_t*)*)p, rhs); else - r = jl_atomic_exchange_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs), rhs); + r = jl_atomic_exchange_release((_Atomic(jl_value_t*)*)p, rhs); jl_gc_wb(v, rhs); + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + return r; } else { - jl_value_t *rty = jl_typeof(rhs); - int hasptr; - int isunion = jl_is_uniontype(ty); - if (isunion) { - assert(!isatomic); - r = jl_get_nth_field(v, i); - size_t fsz = jl_field_size(st, i); - uint8_t *psel = &((uint8_t*)v)[offs + fsz - 1]; - unsigned nth = 0; - if (!jl_find_union_component(ty, rty, &nth)) - assert(0 && "invalid field assignment to isbits union"); - *psel = nth; - if (jl_is_datatype_singleton((jl_datatype_t*)rty)) - return r; - hasptr = 0; - } - else { - hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0; - r = NULL; - } - size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy - int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); - if (isatomic && !needlock) { - r = jl_atomic_swap_bits(rty, (char*)v + offs, rhs, fsz); - if (hasptr) - jl_gc_multi_wb(v, rhs); // rhs is immutable + uint8_t *psel = jl_is_uniontype(ty) ? (uint8_t*)&p[jl_field_size(st, i) - 1] : NULL; + return swap_bits(ty, p, psel, v, rhs, isatomic ? isatomic_object : isatomic_none); + } +} + +inline jl_value_t *modify_value(jl_value_t *ty, _Atomic(jl_value_t*) *p, jl_value_t *parent, jl_value_t *op, jl_value_t *rhs, int isatomic, jl_module_t *mod, jl_sym_t *name) +{ + jl_value_t *r = isatomic ? jl_atomic_load(p) : jl_atomic_load_relaxed(p); + if (__unlikely(r == NULL)) { + if (mod && name) + jl_undefined_var_error(name, (jl_value_t*)mod); + jl_throw(jl_undefref_exception); + } + jl_value_t **args; + JL_GC_PUSHARGS(args, 2); + args[0] = r; + while (1) { + args[1] = rhs; + jl_value_t *y = jl_apply_generic(op, args, 2); + args[1] = y; + if (!jl_isa(y, ty)) { + if (mod && name) + jl_errorf("cannot assign an incompatible value to the global %s.%s.", jl_symbol_name(mod->name), jl_symbol_name(name)); + jl_type_error(jl_is_genericmemory(parent) ? "memoryrefmodify!" : "modifyfield!", ty, y); } - else { - if (needlock) { - jl_task_t *ct = jl_current_task; - r = jl_gc_alloc(ct->ptls, fsz, ty); - jl_lock_value(v); - memcpy((char*)r, (char*)v + offs, fsz); - memcpy((char*)v + offs, (char*)rhs, fsz); - jl_unlock_value(v); - } - else { - if (!isunion) - r = jl_new_bits(ty, (char*)v + offs); - memassign_safe(hasptr, v, (char*)v + offs, rhs, fsz); - } - if (needlock || !isunion) - r = undefref_check((jl_datatype_t*)ty, r); + if (isatomic ? jl_atomic_cmpswap(p, &r, y) : jl_atomic_cmpswap_release(p, &r, y)) { + jl_gc_wb(parent, y); + break; } + args[0] = r; + jl_gc_safepoint(); } - if (__unlikely(r == NULL)) - jl_throw(jl_undefref_exception); - return r; + // args[0] == r (old) + // args[1] == y (new) + jl_datatype_t *rettyp = jl_apply_modify_type(ty); + JL_GC_PROMISE_ROOTED(rettyp); // (JL_ALWAYS_LEAFTYPE) + args[0] = jl_new_struct(rettyp, args[0], args[1]); + JL_GC_POP(); + return args[0]; } -jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *op, jl_value_t *rhs, int isatomic) +inline jl_value_t *modify_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_t *parent, jl_value_t *op, jl_value_t *rhs, enum atomic_kind isatomic) { - size_t offs = jl_field_offset(st, i); - jl_value_t *ty = jl_field_type_concrete(st, i); - jl_value_t *r = jl_get_nth_field_checked(v, i); - if (isatomic && jl_field_isptr(st, i)) - jl_fence(); // load was previously only relaxed + int hasptr; + int isunion = psel != NULL; + if (isunion) { + assert(!isatomic); + hasptr = 0; + } + else { + hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0; + } jl_value_t **args; JL_GC_PUSHARGS(args, 2); - args[0] = r; while (1) { + jl_value_t *r; + jl_value_t *rty = isunion ? jl_nth_union_component(ty, *psel) : ty; + size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the initial copy + int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); + if (isatomic && !needlock) { + r = jl_atomic_new_bits(rty, p); + } + else if (needlock) { + jl_task_t *ct = jl_current_task; + r = jl_gc_alloc(ct->ptls, fsz, rty); + char *px = lock(p, parent, needlock, isatomic); + memcpy((char*)r, px, fsz); + unlock(p, parent, needlock, isatomic); + } + else { + r = jl_new_bits(rty, p); + } + r = undefref_check((jl_datatype_t*)rty, r); + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + args[0] = r; args[1] = rhs; jl_value_t *y = jl_apply_generic(op, args, 2); args[1] = y; - if (!jl_isa(y, ty)) - jl_type_error("modifyfield!", ty, y); - if (jl_field_isptr(st, i)) { - _Atomic(jl_value_t*) *p = (_Atomic(jl_value_t*)*)((char*)v + offs); - if (isatomic ? jl_atomic_cmpswap(p, &r, y) : jl_atomic_cmpswap_relaxed(p, &r, y)) + if (!jl_isa(y, ty)) { + jl_type_error(jl_is_genericmemory(parent) ? "memoryrefmodify!" : "modifyfield!", ty, y); + } + jl_value_t *yty = jl_typeof(y); + if (isatomic && !needlock) { + assert(yty == rty); + if (jl_atomic_bool_cmpswap_bits(p, r, y, fsz)) { + if (hasptr) + jl_gc_multi_wb(parent, y); // y is immutable break; + } } else { - jl_value_t *yty = jl_typeof(y); - jl_value_t *rty = jl_typeof(r); - int hasptr; - int isunion = jl_is_uniontype(ty); - if (isunion) { - assert(!isatomic); - hasptr = 0; - } - else { - hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0; - } - size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy - int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); - if (isatomic && !needlock) { - if (jl_atomic_bool_cmpswap_bits((char*)v + offs, r, y, fsz)) { - if (hasptr) - jl_gc_multi_wb(v, y); // y is immutable - break; - } - r = jl_atomic_new_bits(ty, (char*)v + offs); - } - else { - if (needlock) - jl_lock_value(v); - int success = memcmp((char*)v + offs, r, fsz) == 0; - if (success) { - if (isunion) { - size_t fsz_i = jl_field_size(st, i); - uint8_t *psel = &((uint8_t*)v)[offs + fsz_i - 1]; - success = (jl_typeof(r) == jl_nth_union_component(ty, *psel)); - if (success) { - unsigned nth = 0; - if (!jl_find_union_component(ty, yty, &nth)) - assert(0 && "invalid field assignment to isbits union"); - *psel = nth; - if (jl_is_datatype_singleton((jl_datatype_t*)yty)) - break; - } - fsz = jl_datatype_size((jl_datatype_t*)yty); // need to shrink-wrap the final copy - } - else { - assert(yty == ty && rty == ty); + char *px = lock(p, parent, needlock, isatomic); + int success = memcmp(px, (char*)r, fsz) == 0; + if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + success = jl_egal__bits((jl_value_t*)px, r, (jl_datatype_t*)rty); + if (success) { + if (isunion) { + success = (rty == jl_nth_union_component(ty, *psel)); + if (success) { + unsigned nth = 0; + if (!jl_find_union_component(ty, yty, &nth)) + assert(0 && "invalid field assignment to isbits union"); + *psel = nth; + if (jl_is_datatype_singleton((jl_datatype_t*)yty)) + break; } - memassign_safe(hasptr, v, (char*)v + offs, y, fsz); + fsz = jl_datatype_size((jl_datatype_t*)yty); // need to shrink-wrap the final copy } - if (needlock) - jl_unlock_value(v); - if (success) - break; - r = jl_get_nth_field(v, i); + else { + assert(yty == ty && rty == ty); + } + memassign_safe(hasptr, px, y, fsz); + } + unlock(p, parent, needlock, isatomic); + if (success) { + if (hasptr) + jl_gc_multi_wb(parent, y); // y is immutable + break; } } - args[0] = r; jl_gc_safepoint(); } // args[0] == r (old) @@ -1922,91 +2040,105 @@ jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_valu return args[0]; } -jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *expected, jl_value_t *rhs, int isatomic) +jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *op, jl_value_t *rhs, int isatomic) { - jl_value_t *ty = jl_field_type_concrete(st, i); - if (!jl_isa(rhs, ty)) - jl_type_error("replacefield!", ty, rhs); size_t offs = jl_field_offset(st, i); - jl_value_t *r = expected; + jl_value_t *ty = jl_field_type_concrete(st, i); + char *p = (char*)v + offs; + if (jl_field_isptr(st, i)) { + return modify_value(ty, (_Atomic(jl_value_t*)*)p, v, op, rhs, isatomic, NULL, NULL); + } + else { + uint8_t *psel = jl_is_uniontype(ty) ? (uint8_t*)&p[jl_field_size(st, i) - 1] : NULL; + return modify_bits(ty, p, psel, v, op, rhs, isatomic ? isatomic_object : isatomic_none); + } +} + +inline jl_value_t *replace_value(jl_value_t *ty, _Atomic(jl_value_t*) *p, jl_value_t *parent, jl_value_t *expected, jl_value_t *rhs, int isatomic, jl_module_t *mod, jl_sym_t *name) +{ jl_datatype_t *rettyp = jl_apply_cmpswap_type(ty); JL_GC_PROMISE_ROOTED(rettyp); // (JL_ALWAYS_LEAFTYPE) - if (jl_field_isptr(st, i)) { - _Atomic(jl_value_t*) *p = (_Atomic(jl_value_t*)*)((char*)v + offs); - int success; - while (1) { - success = isatomic ? jl_atomic_cmpswap(p, &r, rhs) : jl_atomic_cmpswap_relaxed(p, &r, rhs); - if (success) - jl_gc_wb(v, rhs); - if (__unlikely(r == NULL)) - jl_throw(jl_undefref_exception); - if (success || !jl_egal(r, expected)) - break; + jl_value_t *r = expected; + int success; + while (1) { + success = isatomic ? jl_atomic_cmpswap(p, &r, rhs) : jl_atomic_cmpswap_release(p, &r, rhs); + if (success) + jl_gc_wb(parent, rhs); + if (__unlikely(r == NULL)) { + if (mod && name) + jl_undefined_var_error(name, (jl_value_t*)mod); + jl_throw(jl_undefref_exception); } - JL_GC_PUSH1(&r); - r = jl_new_struct(rettyp, r, success ? jl_true : jl_false); - JL_GC_POP(); + if (success || !jl_egal(r, expected)) + break; + } + JL_GC_PUSH1(&r); + r = jl_new_struct(rettyp, r, success ? jl_true : jl_false); + JL_GC_POP(); + return r; +} + +inline jl_value_t *replace_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_t *parent, jl_value_t *expected, jl_value_t *rhs, enum atomic_kind isatomic) +{ + jl_datatype_t *rettyp = jl_apply_cmpswap_type(ty); + JL_GC_PROMISE_ROOTED(rettyp); // (JL_ALWAYS_LEAFTYPE) + int hasptr; + int isunion = psel != NULL; + size_t fsz = jl_field_size(rettyp, 0); + int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); + assert(jl_field_offset(rettyp, 1) == fsz); + jl_value_t *rty = ty; + if (isunion) { + assert(!isatomic); + hasptr = 0; + isatomic = isatomic_none; // this makes GCC happy } else { - int hasptr; - int isunion = jl_is_uniontype(ty); - int needlock; - jl_value_t *rty = ty; - size_t fsz = jl_field_size(st, i); - if (isunion) { - assert(!isatomic); - hasptr = 0; - needlock = 0; - isatomic = 0; // this makes GCC happy - } - else { - hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0; - fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy - needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); - } - if (isatomic && !needlock) { - r = jl_atomic_cmpswap_bits((jl_datatype_t*)ty, rettyp, (char*)v + offs, r, rhs, fsz); - int success = *((uint8_t*)r + fsz); - if (success && hasptr) - jl_gc_multi_wb(v, rhs); // rhs is immutable + hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0; + assert(jl_typeis(rhs, ty)); + } + int success; + jl_task_t *ct = jl_current_task; + assert(!jl_field_isptr(rettyp, 0)); + jl_value_t *r = jl_gc_alloc(ct->ptls, jl_datatype_size(rettyp), rettyp); + if (isatomic && !needlock) { + size_t rsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the compare + success = jl_atomic_cmpswap_bits((jl_datatype_t*)rty, r, p, expected, rhs, rsz); + *((uint8_t*)r + fsz) = success ? 1 : 0; + } + else { + char *px = lock(p, parent, needlock, isatomic); + if (isunion) + rty = jl_nth_union_component(rty, *psel); + size_t rsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the compare + memcpy((char*)r, px, rsz); // copy field // TODO: make this a memmove_refs if relevant + if (isunion) + *((uint8_t*)r + fsz - 1) = *psel; // copy union bits + success = (rty == jl_typeof(expected)); + if (success) { + success = memcmp((char*)r, (char*)expected, rsz) == 0; + if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + success = jl_egal__bits(r, expected, (jl_datatype_t*)rty); } - else { - jl_task_t *ct = jl_current_task; - uint8_t *psel = NULL; + *((uint8_t*)r + fsz) = success ? 1 : 0; + if (success) { + jl_value_t *rty = jl_typeof(rhs); if (isunion) { - psel = &((uint8_t*)v)[offs + fsz - 1]; - rty = jl_nth_union_component(rty, *psel); - } - assert(!jl_field_isptr(rettyp, 0)); - r = jl_gc_alloc(ct->ptls, jl_datatype_size(rettyp), (jl_value_t*)rettyp); - int success = (rty == jl_typeof(expected)); - if (needlock) - jl_lock_value(v); - memcpy((char*)r, (char*)v + offs, fsz); // copy field, including union bits - if (success) { - size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy - if (((jl_datatype_t*)rty)->layout->flags.haspadding) - success = jl_egal__bits(r, expected, (jl_datatype_t*)rty); - else - success = memcmp((char*)r, (char*)expected, fsz) == 0; + rsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy + unsigned nth = 0; + if (!jl_find_union_component(ty, rty, &nth)) + assert(0 && "invalid field assignment to isbits union"); + *psel = nth; + if (jl_is_datatype_singleton((jl_datatype_t*)rty)) + return r; } - *((uint8_t*)r + fsz) = success ? 1 : 0; - if (success) { - jl_value_t *rty = jl_typeof(rhs); - size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy - if (isunion) { - unsigned nth = 0; - if (!jl_find_union_component(ty, rty, &nth)) - assert(0 && "invalid field assignment to isbits union"); - *psel = nth; - if (jl_is_datatype_singleton((jl_datatype_t*)rty)) - return r; - } - memassign_safe(hasptr, v, (char*)v + offs, rhs, fsz); - } - if (needlock) - jl_unlock_value(v); + memassign_safe(hasptr, px, rhs, rsz); } + unlock(p, parent, needlock, isatomic); + } + if (success && hasptr) + jl_gc_multi_wb(parent, rhs); // rhs is immutable + if (!isunion) { r = undefref_check((jl_datatype_t*)rty, r); if (__unlikely(r == NULL)) jl_throw(jl_undefref_exception); @@ -2014,6 +2146,74 @@ jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val return r; } +jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *expected, jl_value_t *rhs, int isatomic) +{ + jl_value_t *ty = jl_field_type_concrete(st, i); + if (!jl_isa(rhs, ty)) + jl_type_error("replacefield!", ty, rhs); + size_t offs = jl_field_offset(st, i); + char *p = (char*)v + offs; + if (jl_field_isptr(st, i)) { + return replace_value(ty, (_Atomic(jl_value_t*)*)p, v, expected, rhs, isatomic, NULL, NULL); + } + else { + size_t fsz = jl_field_size(st, i); + int isunion = jl_is_uniontype(ty); + uint8_t *psel = isunion ? (uint8_t*)&p[fsz - 1] : NULL; + return replace_bits(ty, p, psel, v, expected, rhs, isatomic ? isatomic_object : isatomic_none); + } +} + +inline int setonce_bits(jl_datatype_t *rty, char *p, jl_value_t *parent, jl_value_t *rhs, enum atomic_kind isatomic) +{ + size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy + assert(rty->layout->first_ptr >= 0); + int hasptr = 1; + int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); + int success; + if (isatomic && !needlock) { + success = jl_atomic_storeonce_bits(rty, p, rhs, fsz); + } + else { + char *px = lock(p, parent, needlock, isatomic); + success = undefref_check(rty, (jl_value_t*)px) != NULL; + if (success) + memassign_safe(hasptr, px, rhs, fsz); + unlock(p, parent, needlock, isatomic); + } + if (success) + jl_gc_multi_wb(parent, rhs); // rhs is immutable + return success; +} + +int set_nth_fieldonce(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic) +{ + jl_value_t *ty = jl_field_type_concrete(st, i); + if (!jl_isa(rhs, ty)) + jl_type_error("setfieldonce!", ty, rhs); + size_t offs = jl_field_offset(st, i); + int success; + char *p = (char*)v + offs; + if (jl_field_isptr(st, i)) { + _Atomic(jl_value_t*) *px = (_Atomic(jl_value_t*)*)p; + jl_value_t *r = NULL; + success = isatomic ? jl_atomic_cmpswap(px, &r, rhs) : jl_atomic_cmpswap_release(px, &r, rhs); + if (success) + jl_gc_wb(v, rhs); + } + else { + int isunion = jl_is_uniontype(ty); + if (isunion) + return 0; + int hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0; + if (!hasptr) + return 0; + assert(ty == jl_typeof(rhs)); + success = setonce_bits((jl_datatype_t*)ty, p, v, rhs, isatomic ? isatomic_object : isatomic_none); + } + return success; +} + JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i) JL_NOTSAFEPOINT { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); diff --git a/src/genericmemory.c b/src/genericmemory.c index 4543c869e6c74..0bd4db30fd690 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -16,30 +16,6 @@ extern "C" { #endif -static inline void genericmemoryassign_safe(int hasptr, jl_value_t *parent, char *dst, const jl_value_t *src) JL_NOTSAFEPOINT -{ - size_t nb = jl_datatype_size(jl_typeof(src)); // make sure to shrink-wrap this copy - if (hasptr) { - size_t nptr = nb / sizeof(void*); - memmove_refs((_Atomic(void*)*)dst, (_Atomic(void*)*)src, nptr); - jl_gc_multi_wb(parent, src); - } - else { - // genericmemory can assume more alignment than a field would normally have - switch (nb) { - case 0: break; - case 1: *(uint8_t*)dst = *(uint8_t*)src; break; - case 2: *(uint16_t*)dst = *(uint16_t*)src; break; - case 4: *(uint32_t*)dst = *(uint32_t*)src; break; - case 8: *(uint64_t*)dst = *(uint64_t*)src; break; - case 16: - memcpy(jl_assume_aligned(dst, 16), jl_assume_aligned(src, 16), 16); - break; - default: memcpy(dst, src, nb); - } - } -} - // genericmemory constructors --------------------------------------------------------- JL_DLLEXPORT char *jl_genericmemory_typetagdata(jl_genericmemory_t *m) JL_NOTSAFEPOINT { @@ -100,8 +76,9 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory(jl_value_t *mtype, size_ jl_genericmemory_t *m = (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; if (m == NULL) { - if (jl_tparam0((jl_datatype_t*)mtype) != (jl_value_t*)jl_not_atomic_sym) - jl_error("GenericMemory kind must be :not_atomic"); + jl_value_t *kind = jl_tparam0((jl_datatype_t*)mtype); + if (kind != (jl_value_t*)jl_not_atomic_sym && kind != (jl_value_t*)jl_atomic_sym) + jl_error("GenericMemory kind must be :not_atomic or :atomic"); jl_value_t *addrspace = jl_tparam2((jl_datatype_t*)mtype); if (!jl_is_addrspacecore(addrspace) || jl_unbox_uint8(addrspace) != 0) jl_error("GenericMemory addrspace must be Core.CPU"); @@ -109,7 +86,6 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory(jl_value_t *mtype, size_ jl_type_error_rt("GenericMemory", "element type", (jl_value_t*)jl_type_type, jl_tparam1(mtype)); abort(); // this is checked already by jl_get_genericmemory_layout } - assert(jl_tparam0((jl_datatype_t*)mtype) == (jl_value_t*)jl_not_atomic_sym); assert(((jl_datatype_t*)mtype)->has_concrete_subtype && layout != NULL); if (nel == 0) // zero-sized allocation optimization fast path return m; @@ -143,8 +119,9 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void jl_genericmemory_t *m = (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; if (m == NULL) { - if (jl_tparam0((jl_datatype_t*)mtype) != (jl_value_t*)jl_not_atomic_sym) - jl_error("GenericMemory kind must be :not_atomic"); + jl_value_t *kind = jl_tparam0((jl_datatype_t*)mtype); + if (kind != (jl_value_t*)jl_not_atomic_sym && kind != (jl_value_t*)jl_atomic_sym) + jl_error("GenericMemory kind must be :not_atomic or :atomic"); jl_value_t *addrspace = jl_tparam2((jl_datatype_t*)mtype); if (!jl_is_addrspacecore(addrspace) || jl_unbox_uint8(addrspace) != 0) jl_error("GenericMemory addrspace must be Core.CPU"); @@ -152,7 +129,6 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void jl_type_error_rt("GenericMemory", "element type", (jl_value_t*)jl_type_type, jl_tparam1(mtype)); abort(); } - assert(jl_tparam0((jl_datatype_t*)mtype) == (jl_value_t*)jl_not_atomic_sym); assert(((jl_datatype_t*)mtype)->has_concrete_subtype && layout != NULL); //if (nel == 0) {// zero-sized allocation optimization fast path // if (own_buffer) @@ -350,97 +326,14 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda // genericmemory primitives ----------------------------------------------------------- -JL_DLLEXPORT jl_value_t *jl_ptrmemref(jl_genericmemory_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT -{ - assert(i < m->length); - assert(((jl_datatype_t*)jl_typetagof(m))->layout->flags.arrayelem_isboxed); - jl_value_t *elt = jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)m->ptr) + i); - if (elt == NULL) - jl_throw(jl_undefref_exception); - return elt; -} - -JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *m, size_t i) -{ - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; - if (layout->flags.arrayelem_isboxed) - return jl_ptrmemref(m, i); - assert(i < m->length); - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m)); (void)isatomic; // TODO - jl_value_t *eltype = jl_tparam1(jl_typetagof(m)); - if (layout->flags.arrayelem_isunion) { - // isbits union selector bytes are always stored directly after the last memory element - uint8_t sel = jl_genericmemory_typetagdata(m)[i]; - eltype = jl_nth_union_component(eltype, sel); - if (jl_is_datatype_singleton((jl_datatype_t*)eltype)) - return ((jl_datatype_t*)eltype)->instance; - } - jl_value_t *r = undefref_check((jl_datatype_t*)eltype, jl_new_bits(eltype, &((char*)m->ptr)[i * layout->size])); - if (__unlikely(r == NULL)) - jl_throw(jl_undefref_exception); - return r; -} - -JL_DLLEXPORT int jl_genericmemory_isassigned(jl_genericmemory_t *m, size_t i) -{ - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; - if (layout->flags.arrayelem_isboxed) { - return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)m->ptr) + i) != NULL; - } - else if (layout->first_ptr >= 0) { - jl_value_t **elem = (jl_value_t**)((char*)m->ptr + i * layout->size); - return elem[layout->first_ptr] != NULL; - } - return 1; -} - -JL_DLLEXPORT void jl_genericmemoryset(jl_genericmemory_t *m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i) +JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *mem, size_t i) { - assert(i < m->length); - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m)); (void)isatomic; // TODO - jl_value_t *eltype = jl_tparam1(jl_typetagof(m)); - if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { - JL_GC_PUSH1(&rhs); - if (!jl_isa(rhs, eltype)) - jl_type_error("genericmemoryset", eltype, rhs); - JL_GC_POP(); - } - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; - if (layout->flags.arrayelem_isboxed) { - jl_atomic_store_release(((_Atomic(jl_value_t*)*)m->ptr) + i, rhs); - jl_gc_wb(jl_genericmemory_owner(m), rhs); - } - else { - int hasptr; - if (jl_is_uniontype(eltype)) { - uint8_t *psel = &((uint8_t*)jl_genericmemory_typetagdata(m))[i]; - unsigned nth = 0; - if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) - assert(0 && "invalid genericmemoryset to isbits union"); - *psel = nth; - if (jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(rhs))) - return; - hasptr = 0; - } - else { - hasptr = layout->first_ptr >= 0; - } - genericmemoryassign_safe(hasptr, jl_genericmemory_owner(m), &((char*)m->ptr)[i * layout->size], rhs); - } -} - -JL_DLLEXPORT void jl_genericmemoryunset(jl_genericmemory_t *m, size_t i) -{ - if (i >= m->length) - jl_bounds_error_int((jl_value_t*)m, i + 1); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; - if (layout->flags.arrayelem_isboxed) - jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)m->ptr) + i, NULL); - else if (layout->first_ptr >= 0) { - size_t elsize = layout->size; - jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); - memset((char*)m->ptr + elsize * i, 0, elsize); - } + int isatomic = (jl_tparam0(jl_typetagof(mem)) == (jl_value_t*)jl_atomic_sym); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout; + jl_genericmemoryref_t m; + m.mem = mem; + m.ptr_or_offset = (layout->flags.arrayelem_isunion || layout->size == 0) ? (void*)i : (void*)((char*)mem->ptr + layout->size * i); + return jl_memoryrefget(m, isatomic); } JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy_slice(jl_genericmemory_t *mem, void *data, size_t len) @@ -499,25 +392,27 @@ JL_DLLEXPORT jl_genericmemoryref_t jl_memoryrefindex(jl_genericmemoryref_t m JL_ return m; } -JL_DLLEXPORT jl_value_t *jl_ptrmemrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +static jl_value_t *jl_ptrmemrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, int isatomic) JL_NOTSAFEPOINT { assert((char*)m.ptr_or_offset - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); assert(((jl_datatype_t*)jl_typetagof(m.mem))->layout->flags.arrayelem_isboxed); - jl_value_t *elt = jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)m.ptr_or_offset); + _Atomic(jl_value_t*) *ptr = (_Atomic(jl_value_t*)*)m.ptr_or_offset; + jl_value_t *elt = isatomic ? jl_atomic_load(ptr) : jl_atomic_load_relaxed(ptr); if (elt == NULL) jl_throw(jl_undefref_exception); return elt; } -JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m) +JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m, int isatomic) { + assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; if (layout->flags.arrayelem_isboxed) - return jl_ptrmemrefget(m); - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); (void)isatomic; // TODO + return jl_ptrmemrefget(m, isatomic); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); char *data = (char*)m.ptr_or_offset; if (layout->flags.arrayelem_isunion) { + assert(!isatomic); assert(jl_is_uniontype(eltype)); size_t i = (size_t)data; assert(i < m.mem->length); @@ -531,33 +426,55 @@ JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m) return ((jl_datatype_t*)eltype)->instance; } assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); - jl_value_t *r = undefref_check((jl_datatype_t*)eltype, jl_new_bits(eltype, data)); + jl_value_t *r; + size_t fsz = jl_datatype_size(eltype); + int needlock = isatomic && fsz > MAX_ATOMIC_SIZE; + if (isatomic && !needlock) { + r = jl_atomic_new_bits(eltype, data); + } + else if (needlock) { + jl_task_t *ct = jl_current_task; + r = jl_gc_alloc(ct->ptls, fsz, eltype); + jl_lock_field((jl_mutex_t*)data); + memcpy((char*)r, data + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT), fsz); + jl_unlock_field((jl_mutex_t*)data); + } + else { + // TODO: a finalizer here could make the isunion case not quite right + r = jl_new_bits(eltype, data); + } + r = undefref_check((jl_datatype_t*)eltype, r); if (__unlikely(r == NULL)) jl_throw(jl_undefref_exception); return r; } -static int _jl_memoryref_isassigned(jl_genericmemoryref_t m) +static int _jl_memoryref_isassigned(jl_genericmemoryref_t m, int isatomic) { const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + _Atomic(jl_value_t*) *elem = (_Atomic(jl_value_t*)*)m.ptr_or_offset; if (layout->flags.arrayelem_isboxed) { - return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)m.ptr_or_offset) != NULL; } else if (layout->first_ptr >= 0) { - jl_value_t **elem = (jl_value_t**)m.ptr_or_offset; - return elem[layout->first_ptr] != NULL; + int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + if (needlock) + elem = elem + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT) / sizeof(jl_value_t*); + elem = &elem[layout->first_ptr]; + } + else { + return 1; } - return 1; + return (isatomic ? jl_atomic_load(elem) : jl_atomic_load_relaxed(elem)) != NULL; } -JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m) +JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m, int isatomic) { - return _jl_memoryref_isassigned(m) ? jl_true : jl_false; + return _jl_memoryref_isassigned(m, isatomic) ? jl_true : jl_false; } -JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED) +JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, int isatomic) { - jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); (void)isatomic; // TODO + assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { JL_GC_PUSH1(&rhs); @@ -568,47 +485,170 @@ JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, j const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; if (layout->flags.arrayelem_isboxed) { assert((char*)m.ptr_or_offset - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); - jl_atomic_store_release((_Atomic(jl_value_t*)*)m.ptr_or_offset, rhs); + if (isatomic) + jl_atomic_store((_Atomic(jl_value_t*)*)m.ptr_or_offset, rhs); + else + jl_atomic_store_release((_Atomic(jl_value_t*)*)m.ptr_or_offset, rhs); jl_gc_wb(jl_genericmemory_owner(m.mem), rhs); + return; + } + int hasptr; + char *data = (char*)m.ptr_or_offset; + if (layout->flags.arrayelem_isunion) { + assert(!isatomic); + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + uint8_t *psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; + unsigned nth = 0; + if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) + assert(0 && "invalid genericmemoryset to isbits union"); + *psel = nth; + hasptr = 0; + data = (char*)m.mem->ptr + i * layout->size; } else { - int hasptr; - char *data = (char*)m.ptr_or_offset; - if (layout->flags.arrayelem_isunion) { - assert(jl_is_uniontype(eltype)); - size_t i = (size_t)data; - assert(i < m.mem->length); - uint8_t *psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; - unsigned nth = 0; - if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) - assert(0 && "invalid genericmemoryset to isbits union"); - *psel = nth; - hasptr = 0; - data = (char*)m.mem->ptr + i * layout->size; + hasptr = layout->first_ptr >= 0; + } + if (layout->size != 0) { + assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); + int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + size_t fsz = jl_datatype_size((jl_datatype_t*)jl_typeof(rhs)); // need to shrink-wrap the final copy + if (isatomic && !needlock) { + jl_atomic_store_bits(data, rhs, fsz); } - else { - hasptr = layout->first_ptr >= 0; + else if (needlock) { + jl_lock_field((jl_mutex_t*)data); + memassign_safe(hasptr, data + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT), rhs, fsz); + jl_unlock_field((jl_mutex_t*)data); } - if (layout->size != 0) { - assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); - genericmemoryassign_safe(hasptr, jl_genericmemory_owner(m.mem), data, rhs); + else { + memassign_safe(hasptr, data, rhs, fsz); } + if (hasptr) + jl_gc_multi_wb(jl_genericmemory_owner(m.mem), rhs); // rhs is immutable } } -JL_DLLEXPORT void jl_memoryrefunset(jl_genericmemoryref_t m) +JL_DLLEXPORT jl_value_t *jl_memoryrefswap(jl_genericmemoryref_t m, jl_value_t *rhs, int isatomic) { - if (m.mem->length == 0) - jl_bounds_error_int((jl_value_t*)m.mem, 1); + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { + if (!jl_isa(rhs, eltype)) + jl_type_error("memoryrefswap!", eltype, rhs); + } const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + jl_value_t *owner = jl_genericmemory_owner(m.mem); + char *data = (char*)m.ptr_or_offset; if (layout->flags.arrayelem_isboxed) { - jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)m.ptr_or_offset, NULL); + assert(data - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); + jl_value_t *r; + if (isatomic) + r = jl_atomic_exchange((_Atomic(jl_value_t*)*)data, rhs); + else + r = jl_atomic_exchange_release((_Atomic(jl_value_t*)*)data, rhs); + jl_gc_wb(owner, rhs); + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + return r; + } + uint8_t *psel = NULL; + if (layout->flags.arrayelem_isunion) { + assert(!isatomic); + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; + data = (char*)m.mem->ptr + i * layout->size; } - else if (layout->first_ptr >= 0) { - size_t elsize = layout->size; - jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); - memset(m.ptr_or_offset, 0, elsize); + return swap_bits(eltype, data, psel, owner, rhs, isatomic ? isatomic_field : isatomic_none); +} + +JL_DLLEXPORT jl_value_t *jl_memoryrefmodify(jl_genericmemoryref_t m, jl_value_t *op, jl_value_t *rhs, int isatomic) +{ + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + jl_value_t *owner = jl_genericmemory_owner(m.mem); + char *data = (char*)m.ptr_or_offset; + if (layout->flags.arrayelem_isboxed) { + assert(data - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); + return modify_value(eltype, (_Atomic(jl_value_t*)*)data, owner, op, rhs, isatomic, NULL, NULL); + } + size_t fsz = layout->size; + uint8_t *psel = NULL; + if (layout->flags.arrayelem_isunion) { + assert(!isatomic); + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; + data = (char*)m.mem->ptr + i * fsz; + } + return modify_bits(eltype, data, psel, owner, op, rhs, isatomic ? isatomic_field : isatomic_none); +} + +JL_DLLEXPORT jl_value_t *jl_memoryrefreplace(jl_genericmemoryref_t m, jl_value_t *expected, jl_value_t *rhs, int isatomic) +{ + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { + if (!jl_isa(rhs, eltype)) + jl_type_error("memoryrefreplace!", eltype, rhs); + } + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + jl_value_t *owner = jl_genericmemory_owner(m.mem); + char *data = (char*)m.ptr_or_offset; + if (layout->flags.arrayelem_isboxed) { + assert(data - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); + return replace_value(eltype, (_Atomic(jl_value_t*)*)data, owner, expected, rhs, isatomic, NULL, NULL); + } + uint8_t *psel = NULL; + if (layout->flags.arrayelem_isunion) { + assert(!isatomic); + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; + data = (char*)m.mem->ptr + i * layout->size; + } + return replace_bits(eltype, data, psel, owner, expected, rhs, isatomic ? isatomic_field : isatomic_none); +} + +JL_DLLEXPORT jl_value_t *jl_memoryrefsetonce(jl_genericmemoryref_t m, jl_value_t *rhs, int isatomic) +{ + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { + if (!jl_isa(rhs, eltype)) + jl_type_error("memoryrefsetonce!", eltype, rhs); + } + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + jl_value_t *owner = jl_genericmemory_owner(m.mem); + char *data = (char*)m.ptr_or_offset; + int success; + if (layout->flags.arrayelem_isboxed) { + assert(data - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); + jl_value_t *r = NULL; + _Atomic(jl_value_t*) *px = (_Atomic(jl_value_t*)*)data; + success = isatomic ? jl_atomic_cmpswap(px, &r, rhs) : jl_atomic_cmpswap_release(px, &r, rhs); + if (success) + jl_gc_wb(owner, rhs); + } + else { + if (layout->flags.arrayelem_isunion) { + assert(!isatomic); + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + (void)i; + success = 0; + } + else if (layout->first_ptr < 0) { + success = 0; + } + else { + success = setonce_bits((jl_datatype_t*)eltype, data, owner, rhs, isatomic ? isatomic_field : isatomic_none); + } } + return success ? jl_true : jl_false; } JL_DLLEXPORT jl_value_t *ijl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index c784727c4f598..3a98850ddca68 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -535,7 +535,7 @@ static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) return NULL; } -static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, const jl_cgval_t *argv, size_t nargs) +static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, ArrayRef argv, size_t nargs) { Function *func = prepare_call(runtime_func()[f]); SmallVector argvalues(nargs); @@ -547,7 +547,7 @@ static jl_cgval_t emit_runtime_call(jl_codectx_t &ctx, JL_I::intrinsic f, const } // put a bits type tag on some value (despite the name, this doesn't necessarily actually change anything about the value however) -static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) +static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) { // Give the arguments names // const jl_cgval_t &bt_value = argv[0]; @@ -650,7 +650,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) static jl_cgval_t generic_cast( jl_codectx_t &ctx, intrinsic f, Instruction::CastOps Op, - const jl_cgval_t *argv, bool toint, bool fromint) + ArrayRef argv, bool toint, bool fromint) { auto &TT = ctx.emission_context.TargetTriple; auto &DL = ctx.emission_context.DL; @@ -707,12 +707,12 @@ static jl_cgval_t generic_cast( } } -static jl_cgval_t emit_runtime_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_runtime_pointerref(jl_codectx_t &ctx, ArrayRef argv) { return emit_runtime_call(ctx, pointerref, argv, 3); } -static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &e = argv[0]; const jl_cgval_t &i = argv[1]; @@ -769,7 +769,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) assert(!isboxed); if (!type_is_ghost(ptrty)) { Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); - auto load = typed_load(ctx, thePtr, im1, ety, ctx.tbaa().tbaa_data, nullptr, isboxed, AtomicOrdering::NotAtomic, false, align_nb); + thePtr = ctx.builder.CreateInBoundsGEP(ptrty, thePtr, im1); + auto load = typed_load(ctx, thePtr, nullptr, ety, ctx.tbaa().tbaa_data, nullptr, isboxed, AtomicOrdering::NotAtomic, false, align_nb); setName(ctx.emission_context, load.V, "pointerref"); return load; } @@ -779,13 +780,13 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) } } -static jl_cgval_t emit_runtime_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_runtime_pointerset(jl_codectx_t &ctx, ArrayRef argv) { return emit_runtime_call(ctx, pointerset, argv, 4); } // e[i] = x -static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &e = argv[0]; const jl_cgval_t &x = argv[1]; @@ -844,14 +845,15 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) assert(!isboxed); if (!type_is_ghost(ptrty)) { thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); - typed_store(ctx, thePtr, im1, x, jl_cgval_t(), ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, - AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, align_nb, false, true, false, false, false, false, nullptr, ""); + thePtr = ctx.builder.CreateInBoundsGEP(ptrty, thePtr, im1); + typed_store(ctx, thePtr, x, jl_cgval_t(), ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, + AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, align_nb, nullptr, true, false, false, false, false, false, nullptr, "atomic_pointerset", nullptr, nullptr); } } return e; } -static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &ord = argv[0]; if (ord.constant && jl_is_symbol(ord.constant)) { @@ -867,7 +869,7 @@ static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, jl_cgval_t *argv) return emit_runtime_call(ctx, atomic_fence, argv, 1); } -static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) +static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &e = argv[0]; const jl_cgval_t &ord = argv[1]; @@ -945,7 +947,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) // e[i] <= x (swap) // e[i] y => x (replace) // x(e[i], y) (modify) -static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl_cgval_t *argv, int nargs, const jl_cgval_t *modifyop) +static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef argv, int nargs, const jl_cgval_t *modifyop) { bool issetfield = f == atomic_pointerset; bool isreplacefield = f == atomic_pointerreplace; @@ -982,8 +984,8 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl // n.b.: the expected value (y) must be rooted, but not the others Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); bool isboxed = true; - jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, - llvm_order, llvm_failorder, sizeof(jl_value_t*), false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, modifyop, "atomic_pointermodify"); + jl_cgval_t ret = typed_store(ctx, thePtr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, + llvm_order, llvm_failorder, sizeof(jl_value_t*), nullptr, issetfield, isreplacefield, isswapfield, ismodifyfield, false, false, modifyop, "atomic_pointermodify", nullptr, nullptr); if (issetfield) ret = e; return ret; @@ -1021,8 +1023,8 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); else thePtr = nullptr; // could use any value here, since typed_store will not use it - jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, - llvm_order, llvm_failorder, nb, false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, modifyop, "atomic_pointermodify"); + jl_cgval_t ret = typed_store(ctx, thePtr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, + llvm_order, llvm_failorder, nb, nullptr, issetfield, isreplacefield, isswapfield, ismodifyfield, false, false, modifyop, "atomic_pointermodify", nullptr, nullptr); if (issetfield) ret = e; return ret; @@ -1082,7 +1084,7 @@ struct math_builder { } }; -static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, Value **argvalues, size_t nargs, +static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, ArrayRef argvalues, size_t nargs, jl_datatype_t **newtyp, jl_value_t *xtyp); @@ -1263,72 +1265,72 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar case pointerref: ++Emitted_pointerref; assert(nargs == 3); - return emit_pointerref(ctx, argv.data()); + return emit_pointerref(ctx, argv); case pointerset: ++Emitted_pointerset; assert(nargs == 4); - return emit_pointerset(ctx, argv.data()); + return emit_pointerset(ctx, argv); case atomic_fence: ++Emitted_atomic_fence; assert(nargs == 1); - return emit_atomicfence(ctx, argv.data()); + return emit_atomicfence(ctx, argv); case atomic_pointerref: ++Emitted_atomic_pointerref; assert(nargs == 2); - return emit_atomic_pointerref(ctx, argv.data()); + return emit_atomic_pointerref(ctx, argv); case atomic_pointerset: case atomic_pointerswap: case atomic_pointermodify: case atomic_pointerreplace: ++Emitted_atomic_pointerop; - return emit_atomic_pointerop(ctx, f, argv.data(), nargs, nullptr); + return emit_atomic_pointerop(ctx, f, argv, nargs, nullptr); case bitcast: ++Emitted_bitcast; assert(nargs == 2); - return generic_bitcast(ctx, argv.data()); + return generic_bitcast(ctx, argv); case trunc_int: ++Emitted_trunc_int; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::Trunc, argv.data(), true, true); + return generic_cast(ctx, f, Instruction::Trunc, argv, true, true); case sext_int: ++Emitted_sext_int; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::SExt, argv.data(), true, true); + return generic_cast(ctx, f, Instruction::SExt, argv, true, true); case zext_int: ++Emitted_zext_int; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::ZExt, argv.data(), true, true); + return generic_cast(ctx, f, Instruction::ZExt, argv, true, true); case uitofp: ++Emitted_uitofp; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::UIToFP, argv.data(), false, true); + return generic_cast(ctx, f, Instruction::UIToFP, argv, false, true); case sitofp: ++Emitted_sitofp; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::SIToFP, argv.data(), false, true); + return generic_cast(ctx, f, Instruction::SIToFP, argv, false, true); case fptoui: ++Emitted_fptoui; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::FPToUI, argv.data(), true, false); + return generic_cast(ctx, f, Instruction::FPToUI, argv, true, false); case fptosi: ++Emitted_fptosi; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::FPToSI, argv.data(), true, false); + return generic_cast(ctx, f, Instruction::FPToSI, argv, true, false); case fptrunc: ++Emitted_fptrunc; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::FPTrunc, argv.data(), false, false); + return generic_cast(ctx, f, Instruction::FPTrunc, argv, false, false); case fpext: ++Emitted_fpext; assert(nargs == 2); - return generic_cast(ctx, f, Instruction::FPExt, argv.data(), false, false); + return generic_cast(ctx, f, Instruction::FPExt, argv, false, false); case not_int: { ++Emitted_not_int; assert(nargs == 1); const jl_cgval_t &x = argv[0]; if (!jl_is_primitivetype(x.typ)) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); Type *xt = INTT(bitstype_to_llvm(x.typ, ctx.builder.getContext(), true), DL); Value *from = emit_unbox(ctx, xt, x, x.typ); Value *ans = ctx.builder.CreateNot(from); @@ -1340,7 +1342,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar assert(nargs == 1); const jl_cgval_t &x = argv[0]; if (!x.constant || !jl_is_datatype(x.constant)) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); jl_datatype_t *dt = (jl_datatype_t*) x.constant; // select the appropriated overloaded intrinsic @@ -1350,7 +1352,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar else if (dt == jl_float64_type) intr_name += "f64"; else - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); FunctionCallee intr = jl_Module->getOrInsertFunction(intr_name, getInt1Ty(ctx.builder.getContext())); auto ret = ctx.builder.CreateCall(intr); @@ -1363,14 +1365,14 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar // verify argument types if (!jl_is_primitivetype(xinfo.typ)) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); Type *xtyp = bitstype_to_llvm(xinfo.typ, ctx.builder.getContext(), true); if (float_func()[f]) xtyp = FLOATT(xtyp); else xtyp = INTT(xtyp, DL); if (!xtyp) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); ////Bool are required to be in the range [0,1] ////so while they are represented as i8, ////the operations need to be done in mod 1 @@ -1386,13 +1388,13 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar if (f == shl_int || f == lshr_int || f == ashr_int) { if (!jl_is_primitivetype(argv[1].typ)) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); argt[1] = INTT(bitstype_to_llvm(argv[1].typ, ctx.builder.getContext(), true), DL); } else { for (size_t i = 1; i < nargs; ++i) { if (xinfo.typ != argv[i].typ) - return emit_runtime_call(ctx, f, argv.data(), nargs); + return emit_runtime_call(ctx, f, argv, nargs); argt[i] = xtyp; } } @@ -1405,7 +1407,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar // call the intrinsic jl_value_t *newtyp = xinfo.typ; - Value *r = emit_untyped_intrinsic(ctx, f, argvalues.data(), nargs, (jl_datatype_t**)&newtyp, xinfo.typ); + Value *r = emit_untyped_intrinsic(ctx, f, argvalues, nargs, (jl_datatype_t**)&newtyp, xinfo.typ); // Turn Bool operations into mod 1 now, if needed if (newtyp == (jl_value_t*)jl_bool_type && !r->getType()->isIntegerTy(1)) r = ctx.builder.CreateTrunc(r, getInt1Ty(ctx.builder.getContext())); @@ -1415,7 +1417,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar assert(0 && "unreachable"); } -static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, Value **argvalues, size_t nargs, +static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, ArrayRef argvalues, size_t nargs, jl_datatype_t **newtyp, jl_value_t *xtyp) { ++EmittedUntypedIntrinsics; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index f9587d33518a2..138bde4a0830a 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -28,11 +28,8 @@ XX(jl_array_ptr) \ XX(jl_array_ptr_1d_append) \ XX(jl_array_ptr_1d_push) \ - XX(jl_genericmemory_isassigned) \ XX(jl_genericmemory_owner) \ XX(jl_genericmemoryref) \ - XX(jl_genericmemoryset) \ - XX(jl_genericmemoryunset) \ XX(jl_array_rank) \ XX(jl_array_to_string) \ XX(jl_atexit_hook) \ @@ -41,6 +38,7 @@ XX(jl_atomic_error) \ XX(jl_atomic_new_bits) \ XX(jl_atomic_store_bits) \ + XX(jl_atomic_storeonce_bits) \ XX(jl_atomic_swap_bits) \ XX(jl_backtrace_from_here) \ XX(jl_base_relative_to) \ diff --git a/src/julia.h b/src/julia.h index 43df87fd45266..7d143a3daa3fc 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1744,10 +1744,11 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, // constructors JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *bt, const void *src); JL_DLLEXPORT jl_value_t *jl_atomic_new_bits(jl_value_t *dt, const char *src); -JL_DLLEXPORT void jl_atomic_store_bits(char *dst, const jl_value_t *src, int nb); +JL_DLLEXPORT void jl_atomic_store_bits(char *dst, const jl_value_t *src, int nb) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_atomic_swap_bits(jl_value_t *dt, char *dst, const jl_value_t *src, int nb); -JL_DLLEXPORT int jl_atomic_bool_cmpswap_bits(char *dst, const jl_value_t *expected, const jl_value_t *src, int nb); -JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t *rettype, char *dst, const jl_value_t *expected, const jl_value_t *src, int nb); +JL_DLLEXPORT int jl_atomic_bool_cmpswap_bits(char *dst, const jl_value_t *expected, const jl_value_t *src, int nb) JL_NOTSAFEPOINT; +JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y, char *dst, const jl_value_t *expected, const jl_value_t *src, int nb) JL_NOTSAFEPOINT; +JL_DLLEXPORT int jl_atomic_storeonce_bits(jl_datatype_t *dt, char *dst, const jl_value_t *src, int nb) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...); JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na); JL_DLLEXPORT jl_value_t *jl_new_structt(jl_datatype_t *type, jl_value_t *tup); @@ -1875,17 +1876,17 @@ JL_DLLEXPORT jl_genericmemory_t *jl_pchar_to_memory(const char *str, size_t len) JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_t len); JL_DLLEXPORT jl_genericmemory_t *jl_alloc_memory_any(size_t n); JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *m, size_t i); // 0-indexed -JL_DLLEXPORT void jl_genericmemoryset(jl_genericmemory_t *m JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i); // 0-indexed -JL_DLLEXPORT void jl_genericmemoryunset(jl_genericmemory_t *m, size_t i); // 0-indexed -JL_DLLEXPORT int jl_genericmemory_isassigned(jl_genericmemory_t *m, size_t i); // 0-indexed JL_DLLEXPORT jl_genericmemoryref_t *jl_new_memoryref(jl_value_t *typ, jl_genericmemory_t *mem, void *data); -JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, int isatomic); JL_DLLEXPORT jl_value_t *jl_ptrmemoryrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m) JL_GLOBALLY_ROOTED; +JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m, int isatomic) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_genericmemoryref_t jl_memoryrefindex(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, size_t idx) JL_NOTSAFEPOINT; -JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); -JL_DLLEXPORT void jl_memoryrefunset(jl_genericmemoryref_t m); +JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, int isatomic); +JL_DLLEXPORT jl_value_t *jl_memoryrefswap(jl_genericmemoryref_t m, jl_value_t *v, int isatomic); +JL_DLLEXPORT jl_value_t *jl_memoryrefmodify(jl_genericmemoryref_t m, jl_value_t *op, jl_value_t *v, int isatomic); +JL_DLLEXPORT jl_value_t *jl_memoryrefreplace(jl_genericmemoryref_t m, jl_value_t *expected, jl_value_t *v, int isatomic); +JL_DLLEXPORT jl_value_t *jl_memoryrefsetonce(jl_genericmemoryref_t m, jl_value_t *v, int isatomic); // strings JL_DLLEXPORT const char *jl_string_ptr(jl_value_t *s); @@ -1925,6 +1926,10 @@ JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); +JL_DLLEXPORT jl_value_t *jl_checked_swap(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); +JL_DLLEXPORT jl_value_t *jl_checked_replace(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *expected, jl_value_t *rhs); +JL_DLLEXPORT jl_value_t *jl_checked_modify(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *op, jl_value_t *rhs); +JL_DLLEXPORT jl_value_t *jl_checked_assignonce(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT void jl_declare_constant(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var); JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from); JL_DLLEXPORT void jl_module_use(jl_module_t *to, jl_module_t *from, jl_sym_t *s); diff --git a/src/julia_atomics.h b/src/julia_atomics.h index 7e7c8052a278b..c094afcc54cd5 100644 --- a/src/julia_atomics.h +++ b/src/julia_atomics.h @@ -174,6 +174,11 @@ bool jl_atomic_cmpswap_acqrel(std::atomic *ptr, T *expected, S val) { return std::atomic_compare_exchange_strong_explicit(ptr, expected, val, memory_order_acq_rel, memory_order_acquire); } +template +bool jl_atomic_cmpswap_release(std::atomic *ptr, T *expected, S val) +{ + return std::atomic_compare_exchange_strong_explicit(ptr, expected, val, memory_order_release, memory_order_relaxed); +} #define jl_atomic_cmpswap_relaxed(ptr, expected, val) jl_atomic_cmpswap_explicit(ptr, expected, val, memory_order_relaxed) template T jl_atomic_exchange(std::atomic *ptr, S desired) @@ -185,6 +190,7 @@ T jl_atomic_exchange_explicit(std::atomic *ptr, S desired, std::memory_order { return std::atomic_exchange_explicit(ptr, desired, order); } +#define jl_atomic_exchange_release(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_reease) #define jl_atomic_exchange_relaxed(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_relaxed) extern "C" { #else @@ -205,11 +211,15 @@ extern "C" { atomic_compare_exchange_strong(obj, expected, desired) # define jl_atomic_cmpswap_relaxed(obj, expected, desired) \ atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_relaxed, memory_order_relaxed) -#define jl_atomic_cmpswap_acqrel(obj, expected, desired) \ +# define jl_atomic_cmpswap_release(obj, expected, desired) \ + atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_release, memory_order_relaxed) +# define jl_atomic_cmpswap_acqrel(obj, expected, desired) \ atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_acq_rel, memory_order_acquire) // TODO: Maybe add jl_atomic_cmpswap_weak for spin lock # define jl_atomic_exchange(obj, desired) \ atomic_exchange(obj, desired) +# define jl_atomic_exchange_release(obj, desired) \ + atomic_exchange_explicit(obj, desired, memory_order_release) # define jl_atomic_exchange_relaxed(obj, desired) \ atomic_exchange_explicit(obj, desired, memory_order_relaxed) # define jl_atomic_store(obj, val) \ @@ -256,6 +266,7 @@ extern "C" { #define _Atomic(T) T #undef jl_atomic_exchange +#undef jl_atomic_exchange_release #undef jl_atomic_exchange_relaxed #define jl_atomic_exchange(obj, desired) \ (__extension__({ \ @@ -264,10 +275,12 @@ extern "C" { *p__analyzer__ = (desired); \ temp__analyzer__; \ })) +#define jl_atomic_exchange_release jl_atomic_exchange #define jl_atomic_exchange_relaxed jl_atomic_exchange #undef jl_atomic_cmpswap #undef jl_atomic_cmpswap_acqrel +#undef jl_atomic_cmpswap_release #undef jl_atomic_cmpswap_relaxed #define jl_atomic_cmpswap(obj, expected, desired) \ (__extension__({ \ @@ -282,6 +295,7 @@ extern "C" { eq__analyzer__; \ })) #define jl_atomic_cmpswap_acqrel jl_atomic_cmpswap +#define jl_atomic_cmpswap_release jl_atomic_cmpswap #define jl_atomic_cmpswap_relaxed jl_atomic_cmpswap #undef jl_atomic_store diff --git a/src/julia_internal.h b/src/julia_internal.h index 94b7e5609f06d..e140dfa205d8e 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -308,6 +308,25 @@ static inline void memmove_refs(_Atomic(void*) *dstp, _Atomic(void*) *srcp, size } } +static inline void memassign_safe(int hasptr, char *dst, const jl_value_t *src, size_t nb) JL_NOTSAFEPOINT +{ + assert(nb == jl_datatype_size(jl_typeof(src))); + if (hasptr) { + size_t nptr = nb / sizeof(void*); + memmove_refs((_Atomic(void*)*)dst, (_Atomic(void*)*)src, nptr); + nb -= nptr * sizeof(void*); + if (__likely(nb == 0)) + return; + src = (jl_value_t*)((char*)src + nptr * sizeof(void*)); + dst = dst + nptr * sizeof(void*); + } + else if (nb >= 16) { + memcpy(dst, jl_assume_aligned(src, 16), nb); + return; + } + memcpy(dst, jl_assume_aligned(src, sizeof(void*)), nb); +} + // -- gc.c -- // #define GC_CLEAN 0 // freshly allocated @@ -708,6 +727,12 @@ typedef struct jl_typeenv_t { int jl_tuple_isa(jl_value_t **child, size_t cl, jl_datatype_t *pdt); int jl_tuple1_isa(jl_value_t *child1, jl_value_t **child, size_t cl, jl_datatype_t *pdt); +enum atomic_kind { + isatomic_none = 0, + isatomic_object = 1, + isatomic_field = 2 +}; + JL_DLLEXPORT int jl_has_intersect_type_not_kind(jl_value_t *t); int jl_subtype_invariant(jl_value_t *a, jl_value_t *b, int ta); int jl_has_concrete_subtype(jl_value_t *typ); @@ -753,6 +778,13 @@ void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic); jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *op, jl_value_t *rhs, int isatomic); jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *expected, jl_value_t *rhs, int isatomic); +int set_nth_fieldonce(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic); +jl_value_t *swap_bits(jl_value_t *ty, char *v, uint8_t *psel, jl_value_t *parent, jl_value_t *rhs, enum atomic_kind isatomic); +jl_value_t *replace_value(jl_value_t *ty, _Atomic(jl_value_t*) *p, jl_value_t *parent, jl_value_t *expected, jl_value_t *rhs, int isatomic, jl_module_t *mod, jl_sym_t *name); +jl_value_t *replace_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_t *parent, jl_value_t *expected, jl_value_t *rhs, enum atomic_kind isatomic); +jl_value_t *modify_value(jl_value_t *ty, _Atomic(jl_value_t*) *p, jl_value_t *parent, jl_value_t *op, jl_value_t *rhs, int isatomic, jl_module_t *mod, jl_sym_t *name); +jl_value_t *modify_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_t *parent, jl_value_t *op, jl_value_t *rhs, enum atomic_kind isatomic); +int setonce_bits(jl_datatype_t *rty, char *p, jl_value_t *owner, jl_value_t *rhs, enum atomic_kind isatomic); jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module); jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st); diff --git a/src/julia_locks.h b/src/julia_locks.h index 47e258f69aab2..5774ddada60c6 100644 --- a/src/julia_locks.h +++ b/src/julia_locks.h @@ -96,6 +96,11 @@ static inline void jl_mutex_init(jl_mutex_t *lock, const char *name) JL_NOTSAFEP #define JL_LOCK_NOGC(m) jl_mutex_lock_nogc(m) #define JL_UNLOCK_NOGC(m) jl_mutex_unlock_nogc(m) +JL_DLLEXPORT void jl_lock_value(jl_mutex_t *v) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_unlock_value(jl_mutex_t *v) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_lock_field(jl_mutex_t *v) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_unlock_field(jl_mutex_t *v) JL_NOTSAFEPOINT; + #ifdef __cplusplus } #endif diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index edb3aad8f2328..e081c6f067245 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1620,6 +1620,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { callee == gc_preserve_end_func || callee == typeof_func || callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || + callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || callee == write_barrier_func || callee == gc_loaded_func || callee->getName() == "memcmp") { continue; diff --git a/src/module.c b/src/module.c index 787ea947d0480..44b89bc236afc 100644 --- a/src/module.c +++ b/src/module.c @@ -669,10 +669,10 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) b->exportp |= exported; } -JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var) +JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var) // unlike most queries here, this is currently seq_cst { jl_binding_t *b = jl_get_binding(m, var); - return b && (jl_atomic_load_relaxed(&b->value) != NULL); + return b && (jl_atomic_load(&b->value) != NULL); } JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var) @@ -875,7 +875,7 @@ void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *s, jl_binding_t *b } } -JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs) +jl_value_t *jl_check_binding_wr(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED, int reassign) { jl_value_t *old_ty = NULL; if (!jl_atomic_cmpswap_relaxed(&b->ty, &old_ty, (jl_value_t*)jl_any_type)) { @@ -887,24 +887,73 @@ JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sy JL_GC_POP(); } } + else { + old_ty = (jl_value_t*)jl_any_type; + } if (b->constp) { - jl_value_t *old = NULL; - if (jl_atomic_cmpswap(&b->value, &old, rhs)) { - jl_gc_wb(b, rhs); - return; + if (reassign) { + jl_value_t *old = NULL; + if (jl_atomic_cmpswap(&b->value, &old, rhs)) { + jl_gc_wb(b, rhs); + return NULL; + } + if (jl_egal(rhs, old)) + return NULL; + if (jl_typeof(rhs) != jl_typeof(old) || jl_is_type(rhs) || jl_is_module(rhs)) + reassign = 0; + else + jl_safe_printf("WARNING: redefinition of constant %s.%s. This may fail, cause incorrect answers, or produce other errors.\n", + jl_symbol_name(mod->name), jl_symbol_name(var)); } - if (jl_egal(rhs, old)) - return; - if (jl_typeof(rhs) != jl_typeof(old) || jl_is_type(rhs) || jl_is_module(rhs)) { + if (!reassign) jl_errorf("invalid redefinition of constant %s.%s", jl_symbol_name(mod->name), jl_symbol_name(var)); + } + return old_ty; +} - } - jl_safe_printf("WARNING: redefinition of constant %s.%s. This may fail, cause incorrect answers, or produce other errors.\n", - jl_symbol_name(mod->name), jl_symbol_name(var)); +JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs) +{ + if (jl_check_binding_wr(b, mod, var, rhs, 1) != NULL) { + jl_atomic_store_release(&b->value, rhs); + jl_gc_wb(b, rhs); } - jl_atomic_store_release(&b->value, rhs); +} + +JL_DLLEXPORT jl_value_t *jl_checked_swap(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs) +{ + jl_check_binding_wr(b, mod, var, rhs, 0); + jl_value_t *old = jl_atomic_exchange(&b->value, rhs); jl_gc_wb(b, rhs); + if (__unlikely(old == NULL)) + jl_undefined_var_error(var, (jl_value_t*)mod); + return old; +} + +JL_DLLEXPORT jl_value_t *jl_checked_replace(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *expected, jl_value_t *rhs) +{ + jl_value_t *ty = jl_check_binding_wr(b, mod, var, rhs, 0); + return replace_value(ty, &b->value, (jl_value_t*)b, expected, rhs, 1, mod, var); +} + +JL_DLLEXPORT jl_value_t *jl_checked_modify(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *op, jl_value_t *rhs) +{ + jl_value_t *ty = NULL; + if (jl_atomic_cmpswap_relaxed(&b->ty, &ty, (jl_value_t*)jl_any_type)) + ty = (jl_value_t*)jl_any_type; + if (b->constp) + jl_errorf("invalid redefinition of constant %s.%s", + jl_symbol_name(mod->name), jl_symbol_name(var)); + return modify_value(ty, &b->value, (jl_value_t*)b, op, rhs, 1, mod, var); +} + +JL_DLLEXPORT jl_value_t *jl_checked_assignonce(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs ) +{ + jl_check_binding_wr(b, mod, var, rhs, 0); + jl_value_t *old = NULL; + if (jl_atomic_cmpswap(&b->value, &old, rhs)) + jl_gc_wb(b, rhs); + return old; } JL_DLLEXPORT void jl_declare_constant(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 2a84532c4c76b..ff479a4d30959 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -584,9 +584,9 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerreplace(jl_value_t *p, jl_value_t *exp char *pp = (char*)jl_unbox_long(p); jl_datatype_t *rettyp = jl_apply_cmpswap_type(ety); JL_GC_PROMISE_ROOTED(rettyp); // (JL_ALWAYS_LEAFTYPE) + jl_value_t *result = NULL; + JL_GC_PUSH1(&result); if (ety == (jl_value_t*)jl_any_type) { - jl_value_t *result; - JL_GC_PUSH1(&result); result = expected; int success; while (1) { @@ -595,8 +595,6 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerreplace(jl_value_t *p, jl_value_t *exp break; } result = jl_new_struct(rettyp, result, success ? jl_true : jl_false); - JL_GC_POP(); - return result; } else { if (jl_typeof(x) != ety) @@ -604,8 +602,20 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointerreplace(jl_value_t *p, jl_value_t *exp size_t nb = jl_datatype_size(ety); if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE) jl_error("atomic_pointerreplace: invalid pointer for atomic operation"); - return jl_atomic_cmpswap_bits((jl_datatype_t*)ety, rettyp, pp, expected, x, nb); + int isptr = jl_field_isptr(rettyp, 0); + jl_task_t *ct = jl_current_task; + result = jl_gc_alloc(ct->ptls, isptr ? nb : jl_datatype_size(rettyp), isptr ? ety : (jl_value_t*)rettyp); + int success = jl_atomic_cmpswap_bits((jl_datatype_t*)ety, result, pp, expected, x, nb); + if (isptr) { + jl_value_t *z = jl_gc_alloc(ct->ptls, jl_datatype_size(rettyp), rettyp); + *(jl_value_t**)z = result; + result = z; + nb = sizeof(jl_value_t*); + } + *((uint8_t*)result + nb) = success ? 1 : 0; } + JL_GC_POP(); + return result; } JL_DLLEXPORT jl_value_t *jl_atomic_fence(jl_value_t *order_sym) diff --git a/src/staticdata.c b/src/staticdata.c index e7ebcea7bd061..8489fa116688e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -100,7 +100,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 179 +#define NUM_TAGS 188 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -283,12 +283,17 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin_swapfield); INSERT_TAG(jl_builtin_modifyfield); INSERT_TAG(jl_builtin_replacefield); + INSERT_TAG(jl_builtin_setfieldonce); INSERT_TAG(jl_builtin_fieldtype); INSERT_TAG(jl_builtin_memoryref); INSERT_TAG(jl_builtin_memoryrefoffset); INSERT_TAG(jl_builtin_memoryrefget); INSERT_TAG(jl_builtin_memoryrefset); INSERT_TAG(jl_builtin_memoryref_isassigned); + INSERT_TAG(jl_builtin_memoryrefswap); + INSERT_TAG(jl_builtin_memoryrefmodify); + INSERT_TAG(jl_builtin_memoryrefreplace); + INSERT_TAG(jl_builtin_memoryrefsetonce); INSERT_TAG(jl_builtin_apply_type); INSERT_TAG(jl_builtin_applicable); INSERT_TAG(jl_builtin_invoke); @@ -299,6 +304,10 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin_compilerbarrier); INSERT_TAG(jl_builtin_getglobal); INSERT_TAG(jl_builtin_setglobal); + INSERT_TAG(jl_builtin_swapglobal); + INSERT_TAG(jl_builtin_modifyglobal); + INSERT_TAG(jl_builtin_replaceglobal); + INSERT_TAG(jl_builtin_setglobalonce); // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG assert(i == NUM_TAGS - 1); @@ -461,14 +470,16 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_typeassert, &jl_f__apply_iterate, &jl_f__apply_pure, &jl_f__call_latest, &jl_f__call_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, - &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, + &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_setfieldonce, &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, - &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryrefset, &jl_f_memoryref_isassigned, + &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryref_isassigned, + &jl_f_memoryrefset, &jl_f_memoryrefswap, &jl_f_memoryrefmodify, &jl_f_memoryrefreplace, &jl_f_memoryrefsetonce, &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_get_binding_type, &jl_f_set_binding_type, &jl_f_opaque_closure_call, &jl_f_donotdelete, &jl_f_compilerbarrier, - &jl_f_getglobal, &jl_f_setglobal, &jl_f_finalizer, &jl_f__compute_sparams, &jl_f__svec_ref, + &jl_f_getglobal, &jl_f_setglobal, &jl_f_swapglobal, &jl_f_modifyglobal, &jl_f_replaceglobal, &jl_f_setglobalonce, + &jl_f_finalizer, &jl_f__compute_sparams, &jl_f__svec_ref, &jl_f_current_scope, NULL }; diff --git a/test/atomics.jl b/test/atomics.jl index dd50fb96be49f..0d6c841073ad5 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -44,6 +44,13 @@ copy(r::Union{Refxy,ARefxy}) = typeof(r)(r.x, r.y) function add(x::T, y)::T where {T}; x + y; end swap(x, y) = y +struct UndefComplex{T} + re::T + im::T + UndefComplex{T}() where {T} = new{T}() +end +Base.convert(T::Type{<:UndefComplex}, S) = T() + let T1 = Refxy{NTuple{3,UInt8}}, T2 = ARefxy{NTuple{3,UInt8}} @test sizeof(T1) == 6 @@ -60,10 +67,13 @@ end # check that very large types are getting locks let (x, y) = (Complex{Int128}(10, 30), Complex{Int128}(20, 40)) - ar = ARefxy(x, y) r = Refxy(x, y) + ar = ARefxy(x, y) + mr = AtomicMemory{Pair{typeof(x),typeof(y)}}(undef, 20) @test 64 == sizeof(r) < sizeof(ar) - @test sizeof(r) == sizeof(ar) - Int(fieldoffset(typeof(ar), 1)) + @test sizeof(ar) == sizeof(r) + Int(fieldoffset(typeof(ar), 1)) + @test_broken Base.elsize(mr) == sizeof(ar) + @test sizeof(mr) == length(mr) * (sizeof(r) + 16) end struct PadIntA <: Number # internal padding @@ -87,6 +97,8 @@ Base.show(io::IO, x::PadIntA) = print(io, "PadIntA(", x.b, ")") Base.show(io::IO, x::PadIntB) = print(io, "PadIntB(", Int(x), ")") Base.show(io::IO, x::Int24) = print(io, "Int24(", Core.Intrinsics.zext_int(Int, x), ")") +## Fields + @noinline function _test_field_operators(r) r = r[] TT = fieldtype(typeof(r), :x) @@ -264,6 +276,26 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test replacefield!(r, :x, x, y, :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((x, true)) @test replacefield!(r, :x, x, y, :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((y, x === y)) @test replacefield!(r, :x, y, x, :sequentially_consistent) === ReplaceType{TT}((y, true)) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :u, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be written non-atomically") setfieldonce!(r, :x, x) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be written non-atomically") setfieldonce!(r, :x, y, :not_atomic, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :unordered, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be accessed non-atomically") setfieldonce!(r, :x, x, :monotonic, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be accessed non-atomically") setfieldonce!(r, :x, x, :acquire, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be accessed non-atomically") setfieldonce!(r, :x, x, :release, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be accessed non-atomically") setfieldonce!(r, :x, x, :acquire_release, :not_atomic) + @test_throws ConcurrencyViolationError("setfieldonce!: atomic field cannot be accessed non-atomically") setfieldonce!(r, :x, x, :sequentially_consistent, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :u) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :unordered) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :monotonic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :acquire) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :acquire_release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setfieldonce!(r, :x, x, :not_atomic, :sequentially_consistent) + @test setfieldonce!(r, :x, y, :sequentially_consistent, :sequentially_consistent) === false + @test setfieldonce!(r, :x, y, :sequentially_consistent, :sequentially_consistent) === false + @test setfieldonce!(r, :x, x, :sequentially_consistent) === false nothing end @noinline function test_field_orderings(r, x, y) @@ -287,12 +319,6 @@ test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) -struct UndefComplex{T} - re::T - im::T - UndefComplex{T}() where {T} = new{T}() -end -Base.convert(T::Type{<:UndefComplex}, S) = T() @noinline function _test_field_undef(r) r = r[] TT = fieldtype(typeof(r), :x) @@ -318,6 +344,29 @@ test_field_undef(ARefxy{Union{Nothing,Integer}}) test_field_undef(ARefxy{UndefComplex{Any}}) test_field_undef(ARefxy{UndefComplex{UndefComplex{Any}}}) +@noinline function _test_once_undef(r) + r = r[] + TT = fieldtype(typeof(r), :x) + x = convert(TT, 12345_10) + @test_throws UndefRefError getfield(r, :x) + @test setfieldonce!(r, :x, x, :sequentially_consistent) === true + @test getfield(r, :x, :sequentially_consistent) === x + @test setfieldonce!(r, :x, convert(TT, 12345_20), :sequentially_consistent) === false + nothing +end + +@noinline function test_once_undef(TT) + _test_once_undef(Ref(TT())) + _test_once_undef(Ref{Any}(TT())) + nothing +end + +test_once_undef(ARefxy{BigInt}) +test_once_undef(ARefxy{Any}) +test_once_undef(ARefxy{Union{Nothing,Integer}}) +test_once_undef(ARefxy{UndefComplex{Any}}) +test_once_undef(ARefxy{UndefComplex{UndefComplex{Any}}}) + @test_throws ErrorException @macroexpand @atomic foo() @test_throws ErrorException @macroexpand @atomic foo += bar @test_throws ErrorException @macroexpand @atomic foo += bar @@ -374,6 +423,13 @@ let a = ARefxy(1, -1) @test_throws ConcurrencyViolationError @atomicreplace :monotonic :acquire a.x xchg end +let a = ARefxy{Union{Nothing,Integer}}() + @test_throws ConcurrencyViolationError @atomiconce :not_atomic a.x = 2 + @test true === @atomiconce a.x = 1 + @test 1 === @atomic a.x + @test false === @atomiconce a.x = 2 +end + # atomic getfield with boundcheck # via codegen getx(a, boundcheck) = getfield(a, :x, :sequentially_consistent, boundcheck) @@ -384,3 +440,558 @@ ans = getfield(ARefxy{Any}(42, 42), :x, :sequentially_consistent, true) @test ans == 42 ans = getfield(ARefxy{Any}(42, 42), :x, :sequentially_consistent, false) @test ans == 42 + + +## Globals + +# the optimizer is terrible at handling PhiC nodes, so this must be a function +# generator with a custom inlining here of r, instead of being able to assume +# the inlining pass can inline a constant value correctly +function gen_test_global_operators(@nospecialize r) + M = @__MODULE__ + return quote + TT = Core.get_binding_type($M, $r) + T = typeof(getglobal($M, $r)) + @test getglobal($M, $r, :sequentially_consistent) === T(123_10) + @test setglobal!($M, $r, T(123_1), :sequentially_consistent) === T(123_1) + @test getglobal($M, $r, :sequentially_consistent) === T(123_1) + @test replaceglobal!($M, $r, 123_1 % UInt, T(123_30), :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((T(123_1), false)) + @test replaceglobal!($M, $r, T(123_1), T(123_30), :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((T(123_1), true)) + @test getglobal($M, $r, :sequentially_consistent) === T(123_30) + @test replaceglobal!($M, $r, T(123_1), T(123_1), :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((T(123_30), false)) + @test getglobal($M, $r, :sequentially_consistent) === T(123_30) + @test modifyglobal!($M, $r, add, 1, :sequentially_consistent) === Pair{TT,TT}(T(123_30), T(123_31)) + @test modifyglobal!($M, $r, add, 1, :sequentially_consistent) === Pair{TT,TT}(T(123_31), T(123_32)) + @test getglobal($M, $r, :sequentially_consistent) === T(123_32) + @test swapglobal!($M, $r, T(123_1), :sequentially_consistent) === T(123_32) + @test getglobal($M, $r, :sequentially_consistent) === T(123_1) + end +end +@noinline function test_global_operators(T::Type) + r = Symbol("g1_$T") + @eval global $r::$T = 123_10 + invokelatest(@eval(() -> $(gen_test_global_operators(QuoteNode(r))))) + r = Symbol("g2_$T") + @eval global $r::$T = 123_10 + invokelatest(@eval(r -> $(gen_test_global_operators(:r))), r) + nothing +end +test_global_operators(Int) +test_global_operators(Any) +test_global_operators(Union{Nothing,Int}) +test_global_operators(Complex{Int32}) +test_global_operators(Complex{Int128}) +test_global_operators(PadIntA) +test_global_operators(PadIntB) +#FIXME: test_global_operators(Int24) +test_global_operators(Float64) + +function gen_test_global_orderings(@nospecialize r) + M = @__MODULE__ + return quote + @nospecialize x y + TT = Core.get_binding_type($M, $r) + + @test getglobal($M, $r) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") getglobal($M, $r, :u) + @test_throws ConcurrencyViolationError("getglobal: module binding cannot be read non-atomically") getglobal($M, $r, :not_atomic) + @test getglobal($M, $r, :unordered) === x + @test getglobal($M, $r, :monotonic) === x + @test getglobal($M, $r, :acquire) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") getglobal($M, $r, :release) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") getglobal($M, $r, :acquire_release) === x + @test getglobal($M, $r, :sequentially_consistent) === x + @test isdefined($M, $r) + @test_throws ConcurrencyViolationError("invalid atomic ordering") isdefined($M, $r, :u) + @test_throws ConcurrencyViolationError("isdefined: module binding cannot be accessed non-atomically") isdefined($M, $r, :not_atomic) + @test isdefined($M, $r, :unordered) + @test isdefined($M, $r, :monotonic) + @test isdefined($M, $r, :acquire) + @test_throws ConcurrencyViolationError("invalid atomic ordering") isdefined($M, $r, :release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") isdefined($M, $r, :acquire_release) + @test isdefined($M, $r, :sequentially_consistent) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobal!($M, $r, y, :u) + @test_throws ConcurrencyViolationError("setglobal!: module binding cannot be written non-atomically") setglobal!($M, $r, y, :not_atomic) + @test getglobal($M, $r) === x + @test setglobal!($M, $r, y) === y + @test setglobal!($M, $r, y, :unordered) === y + @test setglobal!($M, $r, y, :monotonic) === y + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobal!($M, $r, y, :acquire) === y + @test setglobal!($M, $r, y, :release) === y + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobal!($M, $r, y, :acquire_release) === y + @test setglobal!($M, $r, y, :sequentially_consistent) === y + @test getglobal($M, $r) === y + + @test_throws ConcurrencyViolationError("invalid atomic ordering") swapglobal!($M, $r, x, :u) + @test_throws ConcurrencyViolationError("swapglobal!: module binding cannot be written non-atomically") swapglobal!($M, $r, x, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") swapglobal!($M, $r, x, :unordered) === y + @test swapglobal!($M, $r, x, :monotonic) === y + @test swapglobal!($M, $r, x, :acquire) === x + @test swapglobal!($M, $r, x, :release) === x + @test swapglobal!($M, $r, x, :acquire_release) === x + @test swapglobal!($M, $r, x, :sequentially_consistent) === x + @test swapglobal!($M, $r, x) === x + + @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyglobal!($M, $r, swap, x, :u) + @test_throws ConcurrencyViolationError("modifyglobal!: module binding cannot be written non-atomically") modifyglobal!($M, $r, swap, x, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyglobal!($M, $r, swap, x, :unordered) + @test modifyglobal!($M, $r, swap, x, :monotonic) === Pair{TT,TT}(x, x) + @test modifyglobal!($M, $r, swap, x, :acquire) === Pair{TT,TT}(x, x) + @test modifyglobal!($M, $r, swap, x, :release) === Pair{TT,TT}(x, x) + @test modifyglobal!($M, $r, swap, x, :acquire_release) === Pair{TT,TT}(x, x) + @test modifyglobal!($M, $r, swap, x, :sequentially_consistent) === Pair{TT,TT}(x, x) + @test modifyglobal!($M, $r, swap, x) === Pair{TT,TT}(x, x) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :u, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be written non-atomically") replaceglobal!($M, $r, y, x, :not_atomic, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :unordered, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be accessed non-atomically") replaceglobal!($M, $r, x, x, :monotonic, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be accessed non-atomically") replaceglobal!($M, $r, x, x, :acquire, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be accessed non-atomically") replaceglobal!($M, $r, x, x, :release, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be accessed non-atomically") replaceglobal!($M, $r, x, x, :acquire_release, :not_atomic) + @test_throws ConcurrencyViolationError("replaceglobal!: module binding cannot be accessed non-atomically") replaceglobal!($M, $r, x, x, :sequentially_consistent, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :u) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :unordered) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :monotonic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :acquire) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :acquire_release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replaceglobal!($M, $r, x, x, :not_atomic, :sequentially_consistent) + @test replaceglobal!($M, $r, x, y, :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((x, true)) + @test replaceglobal!($M, $r, x, y, :sequentially_consistent, :sequentially_consistent) === ReplaceType{TT}((y, x === y)) + @test replaceglobal!($M, $r, y, x, :sequentially_consistent) === ReplaceType{TT}((y, true)) + @test replaceglobal!($M, $r, x, x) === ReplaceType{TT}((x, true)) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :u, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be written non-atomically") setglobalonce!($M, $r, y, :not_atomic, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :unordered, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be accessed non-atomically") setglobalonce!($M, $r, x, :monotonic, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be accessed non-atomically") setglobalonce!($M, $r, x, :acquire, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be accessed non-atomically") setglobalonce!($M, $r, x, :release, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be accessed non-atomically") setglobalonce!($M, $r, x, :acquire_release, :not_atomic) + @test_throws ConcurrencyViolationError("setglobalonce!: module binding cannot be accessed non-atomically") setglobalonce!($M, $r, x, :sequentially_consistent, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :u) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :unordered) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :monotonic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :acquire) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :acquire_release) + @test_throws ConcurrencyViolationError("invalid atomic ordering") setglobalonce!($M, $r, x, :not_atomic, :sequentially_consistent) + @test setglobalonce!($M, $r, x) === false + @test setglobalonce!($M, $r, y, :sequentially_consistent, :sequentially_consistent) === false + @test setglobalonce!($M, $r, y, :sequentially_consistent, :sequentially_consistent) === false + @test setglobalonce!($M, $r, x, :sequentially_consistent) === false + end +end +@noinline function test_global_orderings(T::Type, x, y) + @nospecialize + r = Symbol("h1_$T") + @eval global $r::$T = $(QuoteNode(x)) + invokelatest(@eval((x, y) -> $(gen_test_global_orderings(QuoteNode(r)))), x, y) + r = Symbol("h2_$T") + @eval global $r::$T = $(QuoteNode(x)) + invokelatest(@eval((r, x, y) -> $(gen_test_global_orderings(:r))), r, x, y) + nothing +end +test_global_orderings(Int, 10, 20) +test_global_orderings(Bool, true, false) +test_global_orderings(String, "hi", "bye") +test_global_orderings(Symbol, :hi, :bye) +test_global_orderings(Nothing, nothing, nothing) +test_global_orderings(Any, 123_10, 123_20) +test_global_orderings(Any, true, false) +test_global_orderings(Union{Nothing,Missing}, nothing, missing) +test_global_orderings(Union{Nothing,Int}, nothing, 123_1) +test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_global_orderings(Float64, 10.0, 20.0) +test_global_orderings(Float64, NaN, Inf) + +function gen_test_global_undef(@nospecialize r) + M = @__MODULE__ + return quote + TT = Core.get_binding_type($M, $r) + x = convert(TT, 12345_10) + @test_throws UndefVarError getglobal($M, $r) + @test_throws UndefVarError getglobal($M, $r, :sequentially_consistent) + @test_throws UndefVarError modifyglobal!($M, $r, add, 1, :sequentially_consistent) + @test_throws (TT === Any ? UndefVarError : Union{TypeError,ErrorException}) replaceglobal!($M, $r, 1, 1.0, :sequentially_consistent) # TODO: should this be TypeError or ErrorException + @test_throws UndefVarError replaceglobal!($M, $r, 1, x, :sequentially_consistent) + @test_throws UndefVarError getglobal($M, $r, :sequentially_consistent) + @test_throws UndefVarError swapglobal!($M, $r, x, :sequentially_consistent) + @test getglobal($M, $r, :sequentially_consistent) === x === getglobal($M, $r) + end +end +@noinline function test_global_undef(T) + r = Symbol("u1_$T") + @eval global $r::$T + invokelatest(@eval(() -> $(gen_test_global_undef(QuoteNode(r))))) + r = Symbol("u2_$T") + @eval global $r::$T + invokelatest(@eval(r -> $(gen_test_global_undef(:r))), r) + nothing +end +test_global_undef(BigInt) +test_global_undef(Any) +test_global_undef(Union{Nothing,Integer}) +test_global_undef(UndefComplex{Any}) +test_global_undef(UndefComplex{UndefComplex{Any}}) +test_global_undef(Int) + +function gen_test_globalonce(@nospecialize r) + M = @__MODULE__ + return quote + TT = Core.get_binding_type($M, $r) + x = convert(TT, 12345_10) + @test_throws UndefVarError getglobal($M, $r) + @test setglobalonce!($M, $r, x, :sequentially_consistent) === true + @test getglobal($M, $r, :sequentially_consistent) === x + @test setglobalonce!($M, $r, convert(TT, 12345_20), :sequentially_consistent) === false + end +end +@noinline function test_globalonce(T) + r = Symbol("o1_$T") + @eval global $r::$T + invokelatest(@eval(() -> $(gen_test_globalonce(QuoteNode(r))))) + r = Symbol("o2_$T") + @eval global $r::$T + invokelatest(@eval(r -> $(gen_test_globalonce(:r))), r) + nothing +end +test_globalonce(BigInt) +test_globalonce(Any) +test_globalonce(Union{Nothing,Integer}) +test_globalonce(UndefComplex{Any}) +test_globalonce(UndefComplex{UndefComplex{Any}}) +test_globalonce(Int) + +# test macroexpansions +global x::Int +let a = @__MODULE__ + @test_throws ConcurrencyViolationError @atomiconce :not_atomic a.x = 2 + @test true === @atomiconce a.x = 1 + @test 1 === @atomic a.x + @test false === @atomiconce a.x = 2 +end +let a = @__MODULE__ + @test 1 === @atomic a.x + @test 2 === @atomic :sequentially_consistent a.x = 2 + @test 3 === @atomic :monotonic a.x = 3 + local four = 4 + @test 4 === @atomic :monotonic a.x = four + @test 3 === @atomic :monotonic a.x = four - 1 + @test_throws ConcurrencyViolationError @atomic :not_atomic a.x = 2 + @test_throws ConcurrencyViolationError @atomic :not_atomic a.x + @test_throws ConcurrencyViolationError @atomic :not_atomic a.x += 1 + + @test 3 === @atomic :monotonic a.x + @test 5 === @atomic a.x += 2 + @test 4 === @atomic :monotonic a.x -= 1 + @test 12 === @atomic :monotonic a.x *= 3 + + @test 12 === @atomic a.x + @test (12 => 13) === @atomic a.x + 1 + @test (13 => 15) === @atomic :monotonic a.x + 2 + @test (15 => 19) === @atomic a.x max 19 + @test (19 => 20) === @atomic :monotonic a.x max 20 + @test_throws ConcurrencyViolationError @atomic :not_atomic a.x + 1 + @test_throws ConcurrencyViolationError @atomic :not_atomic a.x max 30 + + @test 20 === @atomic a.x + @test 20 === @atomicswap a.x = 1 + @test 1 === @atomicswap :monotonic a.x = 2 + @test_throws ConcurrencyViolationError @atomicswap :not_atomic a.x = 1 + + @test 2 === @atomic a.x + @test ReplaceType{Int}((2, true)) === @atomicreplace a.x 2 => 1 + @test ReplaceType{Int}((1, false)) === @atomicreplace :monotonic a.x 2 => 1 + @test ReplaceType{Int}((1, false)) === @atomicreplace :monotonic :monotonic a.x 2 => 1 + @test_throws ConcurrencyViolationError @atomicreplace :not_atomic a.x 1 => 2 + @test_throws ConcurrencyViolationError @atomicreplace :monotonic :acquire a.x 1 => 2 + + @test 1 === @atomic a.x + xchg = 1 => 2 + @test ReplaceType{Int}((1, true)) === @atomicreplace a.x xchg + @test ReplaceType{Int}((2, false)) === @atomicreplace :monotonic a.x xchg + @test ReplaceType{Int}((2, false)) === @atomicreplace :acquire_release :monotonic a.x xchg + @test_throws ConcurrencyViolationError @atomicreplace :not_atomic a.x xchg + @test_throws ConcurrencyViolationError @atomicreplace :monotonic :acquire a.x xchg +end + +## Memory + +using InteractiveUtils +using Core: memoryrefget, memoryrefset!, memoryrefswap!, memoryrefreplace!, memoryrefmodify!, memoryrefsetonce!, memoryref_isassigned + +@noinline function _test_memory_operators(r) + r = r[] + TT = eltype(r) + T = typeof(r[]) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_10) + @test memoryrefset!(r, T(123_1), :sequentially_consistent, true) === T(123_1) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_1) + @test memoryrefreplace!(r, 123_1 % UInt, T(123_30), :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((T(123_1), false)) + @test memoryrefreplace!(r, T(123_1), T(123_30), :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((T(123_1), true)) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_30) + @test memoryrefreplace!(r, T(123_1), T(123_1), :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((T(123_30), false)) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_30) + @test memoryrefmodify!(r, add, 1, :sequentially_consistent, true) === Pair{TT,TT}(T(123_30), T(123_31)) + @test memoryrefmodify!(r, add, 1, :sequentially_consistent, true) === Pair{TT,TT}(T(123_31), T(123_32)) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_32) + @test memoryrefswap!(r, T(123_1), :sequentially_consistent, true) === T(123_32) + @test memoryrefget(r, :sequentially_consistent, true) === T(123_1) + nothing +end +@noinline function test_memory_operators(T::Type) + x = convert(T, 123_10) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + memoryrefset!(r, x, :unordered, true) # @atomic r[] = x + _test_memory_operators(Ref(r)) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + memoryrefset!(r, x, :unordered, true) # @atomic r[] = x + _test_memory_operators(Ref{Any}(r)) + nothing +end +test_memory_operators(Int) +test_memory_operators(Any) +test_memory_operators(Union{Nothing,Int}) +test_memory_operators(Complex{Int32}) +test_memory_operators(Complex{Int128}) +test_memory_operators(PadIntA) +test_memory_operators(PadIntB) +#FIXME: test_memory_operators(Int24) +test_memory_operators(Float64) + +@noinline function _test_memory_orderings(xr, yr, x, y) + @nospecialize x y + xr = xr[] + yr = yr[] + TT = eltype(yr) + @test TT == eltype(xr) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(xr, :u, true) + @test_throws ConcurrencyViolationError("memoryrefget: atomic memory cannot be accessed non-atomically") memoryrefget(xr, :not_atomic, true) + @test memoryrefget(xr, :unordered, true) === x + @test memoryrefget(xr, :monotonic, true) === x + @test memoryrefget(xr, :acquire, true) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(xr, :release, true) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(xr, :acquire_release, true) === x + @test memoryrefget(xr, :sequentially_consistent, true) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(xr, :u, true) + @test_throws ConcurrencyViolationError("memoryref_isassigned: atomic memory cannot be accessed non-atomically") memoryref_isassigned(xr, :not_atomic, true) + @test memoryref_isassigned(xr, :unordered, true) + @test memoryref_isassigned(xr, :monotonic, true) + @test memoryref_isassigned(xr, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(xr, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(xr, :acquire_release, true) + @test memoryref_isassigned(xr, :sequentially_consistent, true) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(yr, :u, true) + @test memoryrefget(yr, :not_atomic, true) === y + @test_throws ConcurrencyViolationError("memoryrefget: non-atomic memory cannot be accessed atomically") memoryrefget(yr, :unordered, true) + @test_throws ConcurrencyViolationError("memoryrefget: non-atomic memory cannot be accessed atomically") memoryrefget(yr, :monotonic, true) + @test_throws ConcurrencyViolationError("memoryrefget: non-atomic memory cannot be accessed atomically") memoryrefget(yr, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(yr, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefget(yr, :acquire_release, true) + @test_throws ConcurrencyViolationError("memoryrefget: non-atomic memory cannot be accessed atomically") memoryrefget(yr, :sequentially_consistent, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(yr, :u, true) + @test memoryref_isassigned(yr, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryref_isassigned: non-atomic memory cannot be accessed atomically") memoryref_isassigned(yr, :unordered, true) + @test_throws ConcurrencyViolationError("memoryref_isassigned: non-atomic memory cannot be accessed atomically") memoryref_isassigned(yr, :monotonic, true) + @test_throws ConcurrencyViolationError("memoryref_isassigned: non-atomic memory cannot be accessed atomically") memoryref_isassigned(yr, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(yr, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryref_isassigned(yr, :acquire_release, true) + @test_throws ConcurrencyViolationError("memoryref_isassigned: non-atomic memory cannot be accessed atomically") memoryref_isassigned(yr, :sequentially_consistent, true) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(xr, y, :u, true) + @test_throws ConcurrencyViolationError("memoryrefset!: atomic memory cannot be written non-atomically") memoryrefset!(xr, y, :not_atomic, true) + @test memoryrefget(xr, :unordered, true) === x + @test memoryrefset!(xr, y, :unordered, true) === y + @test memoryrefset!(xr, y, :monotonic, true) === y + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(xr, y, :acquire, true) === y + @test memoryrefset!(xr, y, :release, true) === y + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(xr, y, :acquire_release, true) === y + @test memoryrefset!(xr, y, :sequentially_consistent, true) === y + @test memoryrefget(xr, :unordered, true) === y + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(yr, x, :u, true) + @test_throws ConcurrencyViolationError("memoryrefset!: non-atomic memory cannot be written atomically") memoryrefset!(yr, x, :unordered, true) + @test_throws ConcurrencyViolationError("memoryrefset!: non-atomic memory cannot be written atomically") memoryrefset!(yr, x, :monotonic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(yr, x, :acquire, true) + @test_throws ConcurrencyViolationError("memoryrefset!: non-atomic memory cannot be written atomically") memoryrefset!(yr, x, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefset!(yr, x, :acquire_release, true) + @test_throws ConcurrencyViolationError("memoryrefset!: non-atomic memory cannot be written atomically") memoryrefset!(yr, x, :sequentially_consistent, true) + @test memoryrefget(yr, :not_atomic, true) === y + @test memoryrefset!(yr, x, :not_atomic, true) === x + @test memoryrefset!(yr, x, :not_atomic, true) === x + @test memoryrefget(yr, :not_atomic, true) === x + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefswap!(yr, y, :u, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefswap!(yr, y, :unordered, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: non-atomic memory cannot be written atomically") memoryrefswap!(yr, y, :monotonic, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: non-atomic memory cannot be written atomically") memoryrefswap!(yr, y, :acquire, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: non-atomic memory cannot be written atomically") memoryrefswap!(yr, y, :release, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: non-atomic memory cannot be written atomically") memoryrefswap!(yr, y, :acquire_release, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: non-atomic memory cannot be written atomically") memoryrefswap!(yr, y, :sequentially_consistent, true) + @test memoryrefswap!(yr, y, :not_atomic, true) === x + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefmodify!(yr, swap, y, :u, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefmodify!(yr, swap, y, :unordered, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: non-atomic memory cannot be written atomically") memoryrefmodify!(yr, swap, y, :monotonic, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: non-atomic memory cannot be written atomically") memoryrefmodify!(yr, swap, y, :acquire, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: non-atomic memory cannot be written atomically") memoryrefmodify!(yr, swap, y, :release, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: non-atomic memory cannot be written atomically") memoryrefmodify!(yr, swap, y, :acquire_release, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: non-atomic memory cannot be written atomically") memoryrefmodify!(yr, swap, y, :sequentially_consistent, true) + @test memoryrefmodify!(yr, swap, x, :not_atomic, true) === Pair{TT,TT}(y, x) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :u, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :unordered, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: non-atomic memory cannot be written atomically") memoryrefreplace!(yr, y, y, :monotonic, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: non-atomic memory cannot be written atomically") memoryrefreplace!(yr, y, y, :acquire, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: non-atomic memory cannot be written atomically") memoryrefreplace!(yr, y, y, :release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: non-atomic memory cannot be written atomically") memoryrefreplace!(yr, y, y, :acquire_release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: non-atomic memory cannot be written atomically") memoryrefreplace!(yr, y, y, :sequentially_consistent, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :u, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :unordered, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :monotonic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :acquire_release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(yr, y, y, :not_atomic, :sequentially_consistent, true) + @test memoryrefreplace!(yr, x, y, :not_atomic, :not_atomic, true) === ReplaceType{TT}((x, true)) + @test memoryrefreplace!(yr, x, y, :not_atomic, :not_atomic, true) === ReplaceType{TT}((y, x === y)) + @test memoryrefreplace!(yr, y, y, :not_atomic, :not_atomic, true) === ReplaceType{TT}((y, true)) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefswap!(xr, x, :u, true) + @test_throws ConcurrencyViolationError("memoryrefswap!: atomic memory cannot be written non-atomically") memoryrefswap!(xr, x, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefswap!(xr, x, :unordered, true) === y + @test memoryrefswap!(xr, x, :monotonic, true) === y + @test memoryrefswap!(xr, x, :acquire, true) === x + @test memoryrefswap!(xr, x, :release, true) === x + @test memoryrefswap!(xr, x, :acquire_release, true) === x + @test memoryrefswap!(xr, x, :sequentially_consistent, true) === x + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefmodify!(xr, swap, x, :u, true) + @test_throws ConcurrencyViolationError("memoryrefmodify!: atomic memory cannot be written non-atomically") memoryrefmodify!(xr, swap, x, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefmodify!(xr, swap, x, :unordered, true) + @test memoryrefmodify!(xr, swap, x, :monotonic, true) === Pair{TT,TT}(x, x) + @test memoryrefmodify!(xr, swap, x, :acquire, true) === Pair{TT,TT}(x, x) + @test memoryrefmodify!(xr, swap, x, :release, true) === Pair{TT,TT}(x, x) + @test memoryrefmodify!(xr, swap, x, :acquire_release, true) === Pair{TT,TT}(x, x) + @test memoryrefmodify!(xr, swap, x, :sequentially_consistent, true) === Pair{TT,TT}(x, x) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :u, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be written non-atomically") memoryrefreplace!(xr, y, x, :not_atomic, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :unordered, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be accessed non-atomically") memoryrefreplace!(xr, x, x, :monotonic, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be accessed non-atomically") memoryrefreplace!(xr, x, x, :acquire, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be accessed non-atomically") memoryrefreplace!(xr, x, x, :release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be accessed non-atomically") memoryrefreplace!(xr, x, x, :acquire_release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefreplace!: atomic memory cannot be accessed non-atomically") memoryrefreplace!(xr, x, x, :sequentially_consistent, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :u, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :unordered, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :monotonic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :acquire_release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefreplace!(xr, x, x, :not_atomic, :sequentially_consistent, true) + @test memoryrefreplace!(xr, x, y, :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((x, true)) + @test memoryrefreplace!(xr, x, y, :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((y, x === y)) + @test memoryrefreplace!(xr, y, x, :sequentially_consistent, :sequentially_consistent, true) === ReplaceType{TT}((y, true)) + + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :u, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be written non-atomically") memoryrefsetonce!(xr, y, :not_atomic, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :unordered, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be accessed non-atomically") memoryrefsetonce!(xr, x, :monotonic, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be accessed non-atomically") memoryrefsetonce!(xr, x, :acquire, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be accessed non-atomically") memoryrefsetonce!(xr, x, :release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be accessed non-atomically") memoryrefsetonce!(xr, x, :acquire_release, :not_atomic, true) + @test_throws ConcurrencyViolationError("memoryrefsetonce!: atomic memory cannot be accessed non-atomically") memoryrefsetonce!(xr, x, :sequentially_consistent, :not_atomic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :u, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :unordered, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :monotonic, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :acquire, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :acquire_release, true) + @test_throws ConcurrencyViolationError("invalid atomic ordering") memoryrefsetonce!(xr, x, :not_atomic, :sequentially_consistent, true) + @test memoryrefsetonce!(xr, y, :sequentially_consistent, :sequentially_consistent, true) === false + @test memoryrefsetonce!(xr, y, :sequentially_consistent, :sequentially_consistent, true) === false + @test memoryrefsetonce!(xr, x, :sequentially_consistent, :sequentially_consistent, true) === false + nothing +end +@noinline function test_memory_orderings(T::Type, x, y) + @nospecialize + xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x + yr = GenericMemoryRef(Memory{T}(undef, 1)) + yr[] = y + _test_memory_orderings(Ref(xr), Ref(yr), x, y) + xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x + yr = GenericMemoryRef(Memory{T}(undef, 1)) + yr[] = y + _test_memory_orderings(Ref{Any}(xr), Ref{Any}(yr), x, y) + nothing +end +@noinline test_memory_orderings(x, y) = (@nospecialize; test_memory_orderings(typeof(x), x, y)) +test_memory_orderings(10, 20) +test_memory_orderings(true, false) +test_memory_orderings("hi", "bye") +test_memory_orderings(:hi, :bye) +test_memory_orderings(nothing, nothing) +test_memory_orderings(Any, 123_10, 123_20) +test_memory_orderings(Any, true, false) +test_memory_orderings(Union{Nothing,Missing}, nothing, missing) +test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) +test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_memory_orderings(10.0, 20.0) +test_memory_orderings(NaN, Inf) + +@noinline function _test_memory_undef(r) + r = r[] + TT = eltype(r) + x = convert(TT, 12345_10) + @test_throws UndefRefError memoryrefget(r, :sequentially_consistent, true) + @test_throws UndefRefError memoryrefmodify!(r, add, 1, :sequentially_consistent, true) + @test_throws (TT === Any ? UndefRefError : TypeError) memoryrefreplace!(r, 1, 1.0, :sequentially_consistent, :sequentially_consistent, true) + @test_throws UndefRefError memoryrefreplace!(r, 1, x, :sequentially_consistent, :sequentially_consistent, true) + @test_throws UndefRefError memoryrefget(r, :sequentially_consistent, true) + @test_throws UndefRefError memoryrefswap!(r, x, :sequentially_consistent, true) + @test memoryrefget(r, :sequentially_consistent, true) === x + nothing +end +@noinline function test_memory_undef(T) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + _test_memory_undef(Ref(r)) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + _test_memory_undef(Ref{Any}(r)) + nothing +end +test_memory_undef(BigInt) +test_memory_undef(Any) +test_memory_undef(Union{Nothing,Integer}) +test_memory_undef(UndefComplex{Any}) +test_memory_undef(UndefComplex{UndefComplex{Any}}) + +@noinline function _test_once_undef(r) + r = r[] + TT = eltype(r) + x = convert(TT, 12345_10) + @test_throws UndefRefError memoryrefget(r, :sequentially_consistent, true) + @test memoryrefsetonce!(r, x, :sequentially_consistent, :sequentially_consistent, true) === true + @test memoryrefget(r, :sequentially_consistent, true) === x + @test memoryrefsetonce!(r, convert(TT, 12345_20), :sequentially_consistent, :sequentially_consistent, true) === false + nothing +end +@noinline function test_once_undef(T) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + _test_once_undef(Ref(r)) + r = GenericMemoryRef(AtomicMemory{T}(undef, 1)) + _test_once_undef(Ref{Any}(r)) + nothing +end +test_once_undef(BigInt) +test_once_undef(Any) +test_once_undef(Union{Nothing,Integer}) +test_once_undef(UndefComplex{Any}) +test_once_undef(UndefComplex{UndefComplex{Any}}) diff --git a/test/compiler/effects.jl b/test/compiler/effects.jl index 71eb50cd42760..f5ecb8a0d347c 100644 --- a/test/compiler/effects.jl +++ b/test/compiler/effects.jl @@ -446,7 +446,7 @@ let effects = Base.infer_effects() do end @test !Core.Compiler.is_nothrow(effects) end -@test_throws ErrorException setglobal!_nothrow_undefinedyet() +@test_throws Union{ErrorException,TypeError} setglobal!_nothrow_undefinedyet() # TODO: what kind of error should this be? # Nothrow for setfield! mutable struct SetfieldNothrow diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 0d46ed1319ae5..65729ab01cb9f 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -1627,24 +1627,24 @@ let memoryref_tfunc(@nospecialize xs...) = Core.Compiler.memoryref_tfunc(Core.Co @test memoryref_isassigned_tfunc(Any, Any, Any) === Bool @test builtin_tfunction(Core.memoryref_isassigned, Any[MemoryRef{Int}, Vararg{Any}]) == Bool @test builtin_tfunction(Core.memoryref_isassigned, Any[MemoryRef{Int}, Symbol, Bool, Vararg{Bool}]) == Bool - @test memoryrefset!_tfunc(MemoryRef{Int}, Int, Symbol, Bool) === MemoryRef{Int} + @test memoryrefset!_tfunc(MemoryRef{Int}, Int, Symbol, Bool) === Int let ua = MemoryRef{<:Integer} - @test memoryrefset!_tfunc(ua, Int, Symbol, Bool) === ua + @test memoryrefset!_tfunc(ua, Int, Symbol, Bool) === Int end - @test memoryrefset!_tfunc(GenericMemoryRef, Int, Symbol, Bool) === GenericMemoryRef - @test memoryrefset!_tfunc(GenericMemoryRef{:not_atomic}, Int, Symbol, Bool) === GenericMemoryRef{:not_atomic} - @test memoryrefset!_tfunc(Any, Int, Symbol, Bool) === Any + @test memoryrefset!_tfunc(GenericMemoryRef, Int, Symbol, Bool) === Int + @test memoryrefset!_tfunc(GenericMemoryRef{:not_atomic}, Int, Symbol, Bool) === Int + @test memoryrefset!_tfunc(Any, Int, Symbol, Bool) === Int @test memoryrefset!_tfunc(MemoryRef{String}, Int, Symbol, Bool) === Union{} @test memoryrefset!_tfunc(String, Char, Symbol, Bool) === Union{} - @test memoryrefset!_tfunc(MemoryRef{Int}, Any, Symbol, Bool) === MemoryRef{Int} - @test memoryrefset!_tfunc(MemoryRef{Int}, Any, Any, Any) === MemoryRef{Int} - @test memoryrefset!_tfunc(GenericMemoryRef{:not_atomic}, Any, Any, Any) === GenericMemoryRef{:not_atomic} - @test memoryrefset!_tfunc(GenericMemoryRef, Any, Any, Any) === GenericMemoryRef - @test memoryrefset!_tfunc(Any, Any, Any, Any) === Any # also probably could be GenericMemoryRef - @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Vararg{Any}]) == MemoryRef{Int} + @test memoryrefset!_tfunc(MemoryRef{Int}, Any, Symbol, Bool) === Any # could improve this to Int + @test memoryrefset!_tfunc(MemoryRef{Int}, Any, Any, Any) === Any # could improve this to Int + @test memoryrefset!_tfunc(GenericMemoryRef{:not_atomic}, Any, Any, Any) === Any + @test memoryrefset!_tfunc(GenericMemoryRef, Any, Any, Any) === Any + @test memoryrefset!_tfunc(Any, Any, Any, Any) === Any + @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Vararg{Any}]) == Any @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Vararg{Symbol}]) == Union{} - @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Any, Symbol, Vararg{Bool}]) == MemoryRef{Int} - @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Any, Symbol, Bool, Vararg{Any}]) == MemoryRef{Int} + @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Any, Symbol, Vararg{Bool}]) === Any # could improve this to Int + @test builtin_tfunction(Core.memoryrefset!, Any[MemoryRef{Int}, Any, Symbol, Bool, Vararg{Any}]) === Any # could improve this to Int @test memoryrefoffset_tfunc(MemoryRef) == memoryrefoffset_tfunc(GenericMemoryRef) == Int @test memoryrefoffset_tfunc(Memory) == memoryrefoffset_tfunc(GenericMemory) == Union{} @test builtin_tfunction(Core.memoryrefoffset, Any[Vararg{MemoryRef}]) == Int diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 0ef7f6def1057..71331fd30fb3a 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -1793,6 +1793,24 @@ let src = code_typed1((Atomic{Int},Union{Int,Float64})) do a, b end @test count(isinvokemodify(:mymax), src.code) == 2 end +global x_global_inc::Int = 1 +let src = code_typed1(()) do + @atomic (@__MODULE__).x_global_inc += 1 + end + @test count(isinvokemodify(:+), src.code) == 1 +end +let src = code_typed1((Ptr{Int},)) do a + unsafe_modify!(a, +, 1) + end + @test count(isinvokemodify(:+), src.code) == 1 +end +let src = code_typed1((AtomicMemoryRef{Int},)) do a + Core.memoryrefmodify!(a, +, 1, :sequentially_consistent, true) + end + @test count(isinvokemodify(:+), src.code) == 1 +end + + # apply `ssa_inlining_pass` multiple times let interp = Core.Compiler.NativeInterpreter()