From 4dfc9a7d2d45fc08b9cdc13f62b07b1f70b96468 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Wed, 28 Jul 2021 11:17:10 +0200 Subject: [PATCH] Make jl_cumulative_compile_time_ns global (and reentrant). Now, multiple tasks (on the same or different Threads) can start and stop compilation time measurement, without interrupting each other. Add `cumulative_compile_time_ns()` to return the cumulative compile time for a thread without stopping measurement. ```julia julia> Base.cumulative_compile_time_ns_before() # enable constantly measuring compilation time 0x0000000000000000 julia> @time @eval module M ; f(x) = 2+x; end 0.006730 seconds (919 allocations: 55.212 KiB, 57.20% compilation time) Main.M julia> Base.cumulative_compile_time_ns() 0x00000000075246b3 julia> @time 2+2 0.000000 seconds 4 julia> Base.cumulative_compile_time_ns() 0x0000000007fe4a46 julia> @time @eval M.f(2) 0.003538 seconds (750 allocations: 46.247 KiB, 94.64% compilation time) 4 julia> Base.cumulative_compile_time_ns() 0x000000000831619e ``` Make jl_cumulative_compile_time_ns into a global, atomic variable. Instead of keeping per-task compilation time, this change keeps a global counter of compilation time, protected with atomic mutations. Fixes #41739 ```julia julia> include("./compilation-task-migration-17-example.jl") start thread: 2 end thread: 2 5.185706 seconds (3.53 M allocations: 2.570 GiB, 7.34% gc time, 15.57% compilation time) julia> include("./compilation-task-migration-17-example.jl") start thread: 3 WARNING: replacing module M. end thread: 1 4.110316 seconds (18.23 k allocations: 2.391 GiB, 5.67% gc time, 0.24% compilation time) ``` --- .gitignore | 1 + HISTORY.md | 2 + Make.inc | 17 ++ Makefile | 3 + NEWS.md | 1 + base/compiler/abstractinterpretation.jl | 109 +++++++----- base/compiler/inferencestate.jl | 2 +- base/compiler/ssair/legacy.jl | 2 +- base/compiler/ssair/passes.jl | 2 +- base/compiler/ssair/slot2ssa.jl | 2 +- base/compiler/tfuncs.jl | 4 +- base/compiler/typeinfer.jl | 30 ++-- base/compiler/typelattice.jl | 4 +- base/compiler/validation.jl | 9 +- base/coreio.jl | 1 + base/exports.jl | 1 + base/io.jl | 124 +++++++------ base/iobuffer.jl | 6 + base/libuv.jl | 1 + base/process.jl | 1 + base/range.jl | 17 +- base/rational.jl | 18 ++ base/set.jl | 6 + base/stream.jl | 132 ++++++++++---- base/timing.jl | 4 +- contrib/bpftrace/gc_all.bt | 44 +++++ contrib/bpftrace/gc_simple.bt | 23 +++ contrib/bpftrace/gc_stop_the_world_latency.bt | 23 +++ contrib/refresh_checksums.mk | 5 + deps/checksums/libuv | 68 +++---- deps/libuv.version | 4 +- doc/make.jl | 1 + doc/src/base/io-network.md | 1 + doc/src/devdocs/probes.md | 168 ++++++++++++++++++ src/Makefile | 21 ++- src/aotcompile.cpp | 14 +- src/gc.c | 11 ++ src/gf.c | 7 +- src/jitlayers.cpp | 45 ++--- src/julia_internal.h | 40 ++++- src/task.c | 2 +- src/threading.c | 6 +- src/uprobes.d | 18 ++ stdlib/Distributed/src/cluster.jl | 66 ++----- stdlib/Distributed/src/macros.jl | 10 +- stdlib/Distributed/src/managers.jl | 2 +- stdlib/Distributed/src/messages.jl | 28 +-- stdlib/Distributed/src/process_messages.jl | 4 +- stdlib/Distributed/src/remotecall.jl | 46 ++--- stdlib/Distributed/test/distributed_exec.jl | 1 - stdlib/Distributed/test/threads.jl | 63 ------- stdlib/LibUV_jll/Project.toml | 2 +- stdlib/Logging/docs/src/index.md | 3 + stdlib/Sockets/src/Sockets.jl | 15 +- stdlib/Sockets/test/runtests.jl | 35 +++- test/iobuffer.jl | 21 ++- test/misc.jl | 17 ++ test/ranges.jl | 15 +- test/spawn.jl | 8 + 59 files changed, 893 insertions(+), 443 deletions(-) create mode 100755 contrib/bpftrace/gc_all.bt create mode 100755 contrib/bpftrace/gc_simple.bt create mode 100755 contrib/bpftrace/gc_stop_the_world_latency.bt create mode 100644 doc/src/devdocs/probes.md create mode 100644 src/uprobes.d delete mode 100644 stdlib/Distributed/test/threads.jl diff --git a/.gitignore b/.gitignore index 2c5ee63bc3ee97..2fabd2bff211aa 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ *.so *.dylib *.dSYM +*.h.gen *.jl.cov *.jl.*.cov *.jl.mem diff --git a/HISTORY.md b/HISTORY.md index b5ebc03bbf1d9f..7ac06836750c8b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -143,6 +143,8 @@ Standard library changes * `replace(::String)` now accepts multiple patterns, which will be applied left-to-right simultaneously, so only one pattern will be applied to any character, and the patterns will only be applied to the input text, not the replacements ([#40484]). +* New `replace` methods to replace elements of a `Tuple`. + #### Package Manager diff --git a/Make.inc b/Make.inc index 3d61f5255ef1ad..33bfac180a0fa6 100644 --- a/Make.inc +++ b/Make.inc @@ -75,6 +75,9 @@ HAVE_SSP := 0 WITH_GC_VERIFY := 0 WITH_GC_DEBUG_ENV := 0 +# Enable DTrace support +WITH_DTRACE := 0 + # Prevent picking up $ARCH from the environment variables ARCH:= @@ -87,6 +90,10 @@ endef COMMA:=, SPACE:=$(eval) $(eval) +# force a sane / stable configuration +export LC_ALL=C +export LANG=C + # We need python for things like BB triplet recognition and relative path computation. # We don't really care about version, generally, so just find something that works: PYTHON := "$(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo "{python|python3|python2} not found")" @@ -755,6 +762,13 @@ JCXXFLAGS += -DGC_DEBUG_ENV JCFLAGS += -DGC_DEBUG_ENV endif +ifeq ($(WITH_DTRACE), 1) +JCXXFLAGS += -DUSE_DTRACE +JCFLAGS += -DUSE_DTRACE +DTRACE := dtrace +else +endif + # =========================================================================== # Select the cpu architecture to target, or automatically detects the user's compiler @@ -1551,6 +1565,7 @@ LINKCOLOR:="\033[34;1m" PERLCOLOR:="\033[35m" FLISPCOLOR:="\033[32m" JULIACOLOR:="\033[32;1m" +DTRACECOLOR:="\033[32;1m" SRCCOLOR:="\033[33m" BINCOLOR:="\033[37;1m" @@ -1564,6 +1579,7 @@ PRINT_LINK = printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$(GOAL) PRINT_PERL = printf ' %b %b\n' $(PERLCOLOR)PERL$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1) PRINT_FLISP = printf ' %b %b\n' $(FLISPCOLOR)FLISP$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1) PRINT_JULIA = printf ' %b %b\n' $(JULIACOLOR)JULIA$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1) +PRINT_DTRACE = printf ' %b %b\n' $(DTRACECOLOR)DTRACE$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1) else QUIET_MAKE = @@ -1573,6 +1589,7 @@ PRINT_LINK = echo '$(subst ','\'',$(1))'; $(1) PRINT_PERL = echo '$(subst ','\'',$(1))'; $(1) PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1) PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1) +PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1) endif diff --git a/Makefile b/Makefile index 51bb09b094f1c0..62afa8e6855290 100644 --- a/Makefile +++ b/Makefile @@ -573,3 +573,6 @@ endif @time $(call spawn,$(build_bindir)/julia$(EXE) -e '') @time $(call spawn,$(build_bindir)/julia$(EXE) -e '') @time $(call spawn,$(build_bindir)/julia$(EXE) -e '') + +print-locale: + @locale diff --git a/NEWS.md b/NEWS.md index 84941b377daeaf..d64511283107dd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -45,6 +45,7 @@ Standard library changes overflow in most cases. The new function `checked_length` is now available, which will try to use checked arithmetic to error if the result may be wrapping. Or use a package such as SaferIntegers.jl when constructing the range. ([#40382]) +* TCP socket objects now expose `shutdown` functionality and support half-open mode usage ([#40783]). #### InteractiveUtils * A new macro `@time_imports` for reporting any time spent importing packages and their dependencies ([#41612]) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 52a9eb1a02ccd5..296a30cdb2e0ff 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -85,9 +85,12 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), push!(edges, edge) end this_argtypes = isa(matches, MethodMatches) ? argtypes : matches.applicable_argtypes[i] - const_rt, const_result = abstract_call_method_with_const_args(interp, result, f, this_argtypes, match, sv, false) - if const_rt !== rt && const_rt ⊑ rt - rt = const_rt + const_result = abstract_call_method_with_const_args(interp, result, f, this_argtypes, match, sv, false) + if const_result !== nothing + const_rt, const_result = const_result + if const_rt !== rt && const_rt ⊑ rt + rt = const_rt + end end push!(const_results, const_result) if const_result !== nothing @@ -107,9 +110,12 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # try constant propagation with argtypes for this match # this is in preparation for inlining, or improving the return result this_argtypes = isa(matches, MethodMatches) ? argtypes : matches.applicable_argtypes[i] - const_this_rt, const_result = abstract_call_method_with_const_args(interp, result, f, this_argtypes, match, sv, false) - if const_this_rt !== this_rt && const_this_rt ⊑ this_rt - this_rt = const_this_rt + const_result = abstract_call_method_with_const_args(interp, result, f, this_argtypes, match, sv, false) + if const_result !== nothing + const_this_rt, const_result = const_result + if const_this_rt !== this_rt && const_this_rt ⊑ this_rt + this_rt = const_this_rt + end end push!(const_results, const_result) if const_result !== nothing @@ -523,33 +529,35 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, resul @nospecialize(f), argtypes::Vector{Any}, match::MethodMatch, sv::InferenceState, va_override::Bool) mi = maybe_get_const_prop_profitable(interp, result, f, argtypes, match, sv) - mi === nothing && return Any, nothing + mi === nothing && return nothing # try constant prop' inf_cache = get_inference_cache(interp) inf_result = cache_lookup(mi, argtypes, inf_cache) if inf_result === nothing # if there might be a cycle, check to make sure we don't end up # calling ourselves here. - if result.edgecycle && _any(InfStackUnwind(sv)) do infstate - # if the type complexity limiting didn't decide to limit the call signature (`result.edgelimited = false`) - # we can relax the cycle detection by comparing `MethodInstance`s and allow inference to - # propagate different constant elements if the recursion is finite over the lattice - return (result.edgelimited ? match.method === infstate.linfo.def : mi === infstate.linfo) && - any(infstate.result.overridden_by_const) + let result = result # prevent capturing + if result.edgecycle && _any(InfStackUnwind(sv)) do infstate + # if the type complexity limiting didn't decide to limit the call signature (`result.edgelimited = false`) + # we can relax the cycle detection by comparing `MethodInstance`s and allow inference to + # propagate different constant elements if the recursion is finite over the lattice + return (result.edgelimited ? match.method === infstate.linfo.def : mi === infstate.linfo) && + any(infstate.result.overridden_by_const) + end + add_remark!(interp, sv, "[constprop] Edge cycle encountered") + return nothing end - add_remark!(interp, sv, "[constprop] Edge cycle encountered") - return Any, nothing end inf_result = InferenceResult(mi, argtypes, va_override) frame = InferenceState(inf_result, #=cache=#false, interp) - frame === nothing && return Any, nothing # this is probably a bad generated function (unsound), but just ignore it + frame === nothing && return nothing # this is probably a bad generated function (unsound), but just ignore it frame.parent = sv push!(inf_cache, inf_result) - typeinf(interp, frame) || return Any, nothing + typeinf(interp, frame) || return nothing end result = inf_result.result # if constant inference hits a cycle, just bail out - isa(result, InferenceState) && return Any, nothing + isa(result, InferenceState) && return nothing add_backedge!(mi, sv) return result, inf_result end @@ -1174,7 +1182,8 @@ function abstract_invoke(interp::AbstractInterpreter, argtypes::Vector{Any}, sv: nargtype === Bottom && return CallMeta(Bottom, false) nargtype isa DataType || return CallMeta(Any, false) # other cases are not implemented below isdispatchelem(ft) || return CallMeta(Any, false) # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below - types = rewrap_unionall(Tuple{ft, unwrap_unionall(types).parameters...}, types) + ft = ft::DataType + types = rewrap_unionall(Tuple{ft, unwrap_unionall(types).parameters...}, types)::Type nargtype = Tuple{ft, nargtype.parameters...} argtype = Tuple{ft, argtype.parameters...} result = findsup(types, method_table(interp)) @@ -1196,12 +1205,14 @@ function abstract_invoke(interp::AbstractInterpreter, argtypes::Vector{Any}, sv: # t, a = ti.parameters[i], argtypes′[i] # argtypes′[i] = t ⊑ a ? t : a # end - const_rt, const_result = abstract_call_method_with_const_args(interp, result, argtype_to_function(ft′), argtypes′, match, sv, false) - if const_rt !== rt && const_rt ⊑ rt - return CallMeta(const_rt, InvokeCallInfo(match, const_result)) - else - return CallMeta(rt, InvokeCallInfo(match, nothing)) + const_result = abstract_call_method_with_const_args(interp, result, argtype_to_function(ft′), argtypes′, match, sv, false) + if const_result !== nothing + const_rt, const_result = const_result + if const_rt !== rt && const_rt ⊑ rt + return CallMeta(const_rt, InvokeCallInfo(match, const_result)) + end end + return CallMeta(rt, InvokeCallInfo(match, nothing)) end # call where the function is known exactly @@ -1301,19 +1312,20 @@ end function abstract_call_opaque_closure(interp::AbstractInterpreter, closure::PartialOpaque, argtypes::Vector{Any}, sv::InferenceState) pushfirst!(argtypes, closure.env) sig = argtypes_to_type(argtypes) - (; rt, edge) = result = abstract_call_method(interp, closure.source::Method, sig, Core.svec(), false, sv) + (; rt, edge) = result = abstract_call_method(interp, closure.source, sig, Core.svec(), false, sv) edge !== nothing && add_backedge!(edge, sv) tt = closure.typ - sigT = unwrap_unionall(tt).parameters[1] - match = MethodMatch(sig, Core.svec(), closure.source::Method, sig <: rewrap_unionall(sigT, tt)) + sigT = (unwrap_unionall(tt)::DataType).parameters[1] + match = MethodMatch(sig, Core.svec(), closure.source, sig <: rewrap_unionall(sigT, tt)) info = OpaqueClosureCallInfo(match) if !result.edgecycle - const_rettype, const_result = abstract_call_method_with_const_args(interp, result, closure, argtypes, + const_result = abstract_call_method_with_const_args(interp, result, closure, argtypes, match, sv, closure.isva) - if const_rettype ⊑ rt - rt = const_rettype - end if const_result !== nothing + const_rettype, const_result = const_result + if const_rettype ⊑ rt + rt = const_rettype + end info = ConstCallInfo(info, Union{Nothing,InferenceResult}[const_result]) end end @@ -1323,7 +1335,7 @@ end function most_general_argtypes(closure::PartialOpaque) ret = Any[] cc = widenconst(closure) - argt = unwrap_unionall(cc).parameters[1] + argt = (unwrap_unionall(cc)::DataType).parameters[1] if !isa(argt, DataType) || argt.name !== typename(Tuple) argt = Tuple end @@ -1338,8 +1350,8 @@ function abstract_call(interp::AbstractInterpreter, fargs::Union{Nothing,Vector{ f = argtype_to_function(ft) if isa(ft, PartialOpaque) return abstract_call_opaque_closure(interp, ft, argtypes[2:end], sv) - elseif isa(unwrap_unionall(ft), DataType) && unwrap_unionall(ft).name === typename(Core.OpaqueClosure) - return CallMeta(rewrap_unionall(unwrap_unionall(ft).parameters[2], ft), false) + elseif (uft = unwrap_unionall(ft); isa(uft, DataType) && uft.name === typename(Core.OpaqueClosure)) + return CallMeta(rewrap_unionall((uft::DataType).parameters[2], ft), false) elseif f === nothing # non-constant function, but the number of arguments is known # and the ft is not a Builtin or IntrinsicFunction @@ -1534,12 +1546,12 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), if length(e.args) == 2 && isconcretetype(t) && !ismutabletype(t) at = abstract_eval_value(interp, e.args[2], vtypes, sv) n = fieldcount(t) - if isa(at, Const) && isa(at.val, Tuple) && n == length(at.val) && - let t = t; _all(i->getfield(at.val, i) isa fieldtype(t, i), 1:n); end + if isa(at, Const) && isa(at.val, Tuple) && n == length(at.val::Tuple) && + let t = t; _all(i->getfield(at.val::Tuple, i) isa fieldtype(t, i), 1:n); end t = Const(ccall(:jl_new_structt, Any, (Any, Any), t, at.val)) - elseif isa(at, PartialStruct) && at ⊑ Tuple && n == length(at.fields) && - let t = t, at = at; _all(i->at.fields[i] ⊑ fieldtype(t, i), 1:n); end - t = PartialStruct(t, at.fields) + elseif isa(at, PartialStruct) && at ⊑ Tuple && n == length(at.fields::Vector{Any}) && + let t = t, at = at; _all(i->(at.fields::Vector{Any})[i] ⊑ fieldtype(t, i), 1:n); end + t = PartialStruct(t, at.fields::Vector{Any}) end end elseif e.head === :new_opaque_closure @@ -1587,7 +1599,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), sym = e.args[1] t = Bool if isa(sym, SlotNumber) - vtyp = vtypes[slot_id(sym)] + vtyp = vtypes[slot_id(sym)]::VarState if vtyp.typ === Bottom t = Const(false) # never assigned previously elseif !vtyp.undef @@ -1602,7 +1614,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), t = Const(true) end elseif isa(sym, Expr) && sym.head === :static_parameter - n = sym.args[1] + n = sym.args[1]::Int if 1 <= n <= length(sv.sptypes) spty = sv.sptypes[n] if isa(spty, Const) @@ -1637,7 +1649,7 @@ function abstract_eval_global(M::Module, s::Symbol) end function abstract_eval_ssavalue(s::SSAValue, src::CodeInfo) - typ = src.ssavaluetypes[s.id] + typ = (src.ssavaluetypes::Vector{Any})[s.id] if typ === NOT_FOUND return Bottom end @@ -1725,6 +1737,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) isva = isa(def, Method) && def.isva nslots = nargs - isva slottypes = frame.slottypes + ssavaluetypes = frame.src.ssavaluetypes::Vector{Any} while frame.pc´´ <= n # make progress on the active ip set local pc::Int = frame.pc´´ # current program-counter @@ -1828,7 +1841,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) for (caller, caller_pc) in frame.cycle_backedges # notify backedges of updated type information typeassert(caller.stmt_types[caller_pc], VarTable) # we must have visited this statement before - if !(caller.src.ssavaluetypes[caller_pc] === Any) + if !((caller.src.ssavaluetypes::Vector{Any})[caller_pc] === Any) # no reason to revisit if that call-site doesn't affect the final result if caller_pc < caller.pc´´ caller.pc´´ = caller_pc @@ -1838,6 +1851,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) end end elseif hd === :enter + stmt = stmt::Expr l = stmt.args[1]::Int frame.cur_hand = Pair{Any,Any}(l, frame.cur_hand) # propagate type info to exception handler @@ -1853,21 +1867,24 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) typeassert(states[l], VarTable) frame.handler_at[l] = frame.cur_hand elseif hd === :leave + stmt = stmt::Expr for i = 1:((stmt.args[1])::Int) frame.cur_hand = (frame.cur_hand::Pair{Any,Any}).second end else if hd === :(=) + stmt = stmt::Expr t = abstract_eval_statement(interp, stmt.args[2], changes, frame) if t === Bottom break end - frame.src.ssavaluetypes[pc] = t + ssavaluetypes[pc] = t lhs = stmt.args[1] if isa(lhs, SlotNumber) changes = StateUpdate(lhs, VarState(t, false), changes, false) end elseif hd === :method + stmt = stmt::Expr fname = stmt.args[1] if isa(fname, SlotNumber) changes = StateUpdate(fname, VarState(Any, false), changes, false) @@ -1882,7 +1899,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) if !isempty(frame.ssavalue_uses[pc]) record_ssa_assign(pc, t, frame) else - frame.src.ssavaluetypes[pc] = t + ssavaluetypes[pc] = t end end if frame.cur_hand !== nothing && isa(changes, StateUpdate) @@ -1903,7 +1920,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) if t === nothing # mark other reached expressions as `Any` to indicate they don't throw - frame.src.ssavaluetypes[pc] = Any + ssavaluetypes[pc] = Any end pc´ > n && break # can't proceed with the fast-path fall-through diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index a3f2b1ea95d27e..d76a4ebd18f6e0 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -179,7 +179,7 @@ function sptypes_from_meth_instance(linfo::MethodInstance) while temp isa UnionAll temp = temp.body end - sigtypes = temp.parameters + sigtypes = (temp::DataType).parameters for j = 1:length(sigtypes) tj = sigtypes[j] if isType(tj) && tj.parameters[1] === Pi diff --git a/base/compiler/ssair/legacy.jl b/base/compiler/ssair/legacy.jl index a921c2eac70c51..a832daa8ed4184 100644 --- a/base/compiler/ssair/legacy.jl +++ b/base/compiler/ssair/legacy.jl @@ -47,7 +47,7 @@ function replace_code_newstyle!(ci::CodeInfo, ir::IRCode, nargs::Int) for metanode in ir.meta push!(ci.code, metanode) push!(ci.codelocs, 1) - push!(ci.ssavaluetypes, Any) + push!(ci.ssavaluetypes::Vector{Any}, Any) push!(ci.ssaflags, 0x00) end # Translate BB Edges to statement edges diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 37f5d38940e053..838ab5bd217553 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1066,7 +1066,7 @@ function type_lift_pass!(ir::IRCode) if haskey(processed, id) val = processed[id] else - push!(worklist, (id, up_id, new_phi, i)) + push!(worklist, (id, up_id, new_phi::SSAValue, i)) continue end else diff --git a/base/compiler/ssair/slot2ssa.jl b/base/compiler/ssair/slot2ssa.jl index 3c3e7fcfc48b39..6ca17be2f0a83a 100644 --- a/base/compiler/ssair/slot2ssa.jl +++ b/base/compiler/ssair/slot2ssa.jl @@ -873,7 +873,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, domtree::DomTree, defuse, changed = false for new_idx in type_refine_phi node = new_nodes.stmts[new_idx] - new_typ = recompute_type(node[:inst], ci, ir, ir.sptypes, slottypes) + new_typ = recompute_type(node[:inst]::Union{PhiNode,PhiCNode}, ci, ir, ir.sptypes, slottypes) if !(node[:type] ⊑ new_typ) || !(new_typ ⊑ node[:type]) node[:type] = new_typ changed = true diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 09aa05f35b9f08..772db09417393f 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1564,7 +1564,7 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp if length(argtypes) - 1 == tf[2] argtypes = argtypes[1:end-1] else - vatype = argtypes[end] + vatype = argtypes[end]::Core.TypeofVararg argtypes = argtypes[1:end-1] while length(argtypes) < tf[1] push!(argtypes, unwrapva(vatype)) @@ -1670,7 +1670,7 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s aft = argtypes[2] if isa(aft, Const) || (isType(aft) && !has_free_typevars(aft)) || (isconcretetype(aft) && !(aft <: Builtin)) - af_argtype = isa(tt, Const) ? tt.val : tt.parameters[1] + af_argtype = isa(tt, Const) ? tt.val : (tt::DataType).parameters[1] if isa(af_argtype, DataType) && af_argtype <: Tuple argtypes_vec = Any[aft, af_argtype.parameters...] if contains_is(argtypes_vec, Union{}) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 246f66fc64ed86..1d2559cf10c3a7 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -243,7 +243,7 @@ function _typeinf(interp::AbstractInterpreter, frame::InferenceState) # collect results for the new expanded frame results = Tuple{InferenceResult, Vector{Any}, Bool}[ ( frames[i].result, - frames[i].stmt_edges[1], + frames[i].stmt_edges[1]::Vector{Any}, frames[i].cached ) for i in 1:length(frames) ] empty!(frames) @@ -341,7 +341,7 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInsta if cache_the_tree if may_compress(interp) nslots = length(ci.slotflags) - resize!(ci.slottypes, nslots) + resize!(ci.slottypes::Vector{Any}, nslots) resize!(ci.slotnames, nslots) return ccall(:jl_compress_ir, Any, (Any, Any), def, ci) else @@ -438,7 +438,7 @@ function finish(me::InferenceState, interp::AbstractInterpreter) empty!(edges) end if me.src.edges !== nothing - append!(s_edges, me.src.edges) + append!(s_edges, me.src.edges::Vector) me.src.edges = nothing end # inspect whether our inference had a limited result accuracy, @@ -447,7 +447,7 @@ function finish(me::InferenceState, interp::AbstractInterpreter) limited_ret = me.bestguess isa LimitedAccuracy limited_src = false if !limited_ret - gt = me.src.ssavaluetypes + gt = me.src.ssavaluetypes::Vector{Any} for j = 1:length(gt) gt[j] = gtj = cycle_fix_limited(gt[j], me) if gtj isa LimitedAccuracy && me.parent !== nothing @@ -511,8 +511,9 @@ end # widen all Const elements in type annotations function widen_all_consts!(src::CodeInfo) - for i = 1:length(src.ssavaluetypes) - src.ssavaluetypes[i] = widenconst(src.ssavaluetypes[i]) + ssavaluetypes = src.ssavaluetypes::Vector{Any} + for i = 1:length(ssavaluetypes) + ssavaluetypes[i] = widenconst(ssavaluetypes[i]) end for i = 1:length(src.code) @@ -577,6 +578,7 @@ function record_slot_assign!(sv::InferenceState) states = sv.stmt_types body = sv.src.code::Vector{Any} slottypes = sv.slottypes::Vector{Any} + ssavaluetypes = sv.src.ssavaluetypes::Vector{Any} for i = 1:length(body) expr = body[i] st_i = states[i] @@ -585,7 +587,7 @@ function record_slot_assign!(sv::InferenceState) lhs = expr.args[1] rhs = expr.args[2] if isa(lhs, SlotNumber) - vt = widenconst(sv.src.ssavaluetypes[i]) + vt = widenconst(ssavaluetypes[i]) if vt !== Bottom id = slot_id(lhs) otherTy = slottypes[id] @@ -608,12 +610,11 @@ function type_annotate!(sv::InferenceState, run_optimizer::Bool) # (otherwise, we'll perhaps run the optimization passes later, outside of inference) # remove all unused ssa values - gt = sv.src.ssavaluetypes - for j = 1:length(gt) - if gt[j] === NOT_FOUND - gt[j] = Union{} - end - gt[j] = widenconditional(gt[j]) + src = sv.src + ssavaluetypes = src.ssavaluetypes::Vector{Any} + for j = 1:length(ssavaluetypes) + t = ssavaluetypes[j] + ssavaluetypes[j] = t === NOT_FOUND ? Union{} : widenconditional(t) end # compute the required type for each slot @@ -626,7 +627,6 @@ function type_annotate!(sv::InferenceState, run_optimizer::Bool) # annotate variables load types # remove dead code optimization # and compute which variables may be used undef - src = sv.src states = sv.stmt_types nargs = sv.nargs nslots = length(states[1]::VarTable) @@ -669,7 +669,7 @@ function type_annotate!(sv::InferenceState, run_optimizer::Bool) elseif run_optimizer deleteat!(body, i) deleteat!(states, i) - deleteat!(src.ssavaluetypes, i) + deleteat!(ssavaluetypes, i) deleteat!(src.codelocs, i) deleteat!(sv.stmt_info, i) nexpr -= 1 diff --git a/base/compiler/typelattice.jl b/base/compiler/typelattice.jl index 6391d4029b58e7..9712e023c70042 100644 --- a/base/compiler/typelattice.jl +++ b/base/compiler/typelattice.jl @@ -299,7 +299,7 @@ function smerge(sa::Union{NotFound,VarState}, sb::Union{NotFound,VarState}) end @inline tchanged(@nospecialize(n), @nospecialize(o)) = o === NOT_FOUND || (n !== NOT_FOUND && !(n ⊑ o)) -@inline schanged(@nospecialize(n), @nospecialize(o)) = (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !issubstate(n, o))) +@inline schanged(@nospecialize(n), @nospecialize(o)) = (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !issubstate(n::VarState, o::VarState))) widenconditional(@nospecialize typ) = typ function widenconditional(typ::AnyConditional) @@ -396,7 +396,7 @@ function stupdate1!(state::VarTable, change::StateUpdate) if isa(oldtypetyp, Conditional) && slot_id(oldtypetyp.var) == changeid oldtypetyp = widenconditional(oldtypetyp) if oldtype.typ isa LimitedAccuracy - oldtypetyp = LimitedAccuracy(oldtypetyp, oldtype.typ.causes) + oldtypetyp = LimitedAccuracy(oldtypetyp, (oldtype.typ::LimitedAccuracy).causes) end state[i] = VarState(oldtypetyp, oldtype.undef) end diff --git a/base/compiler/validation.jl b/base/compiler/validation.jl index f6b89f8f5cd04d..c152dfb9fa6a54 100644 --- a/base/compiler/validation.jl +++ b/base/compiler/validation.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Expr head => argument count bounds -const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange}( +const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :call => 1:typemax(Int), :invoke => 2:typemax(Int), :static_parameter => 1:1, @@ -180,10 +180,11 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_ !is_top_level && nslotnames == 0 && push!(errors, InvalidCodeError(EMPTY_SLOTNAMES)) nslotnames < nslotflags && push!(errors, InvalidCodeError(SLOTFLAGS_MISMATCH, (nslotnames, nslotflags))) if c.inferred - nssavaluetypes = length(c.ssavaluetypes) + nssavaluetypes = length(c.ssavaluetypes::Vector{Any}) nssavaluetypes < nssavals && push!(errors, InvalidCodeError(SSAVALUETYPES_MISMATCH, (nssavals, nssavaluetypes))) else - c.ssavaluetypes != nssavals && push!(errors, InvalidCodeError(SSAVALUETYPES_MISMATCH_UNINFERRED, (nssavals, c.ssavaluetypes))) + ssavaluetypes = c.ssavaluetypes::Int + ssavaluetypes != nssavals && push!(errors, InvalidCodeError(SSAVALUETYPES_MISMATCH_UNINFERRED, (nssavals, ssavaluetypes))) end return errors end @@ -205,7 +206,7 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, mi::Core.MethodInsta else m = mi.def::Method mnargs = m.nargs - n_sig_params = length(Core.Compiler.unwrap_unionall(m.sig).parameters) + n_sig_params = length((unwrap_unionall(m.sig)::DataType).parameters) if (m.isva ? (n_sig_params < (mnargs - 1)) : (n_sig_params != mnargs)) push!(errors, InvalidCodeError(SIGNATURE_NARGS_MISMATCH, (m.isva, n_sig_params, mnargs))) end diff --git a/base/coreio.jl b/base/coreio.jl index 9ef717383dedd5..d0f8df290b41b7 100644 --- a/base/coreio.jl +++ b/base/coreio.jl @@ -13,6 +13,7 @@ write(::DevNull, ::UInt8) = 1 unsafe_write(::DevNull, ::Ptr{UInt8}, n::UInt)::Int = n close(::DevNull) = nothing wait_close(::DevNull) = wait() +bytesavailable(io::DevNull) = 0 let CoreIO = Union{Core.CoreSTDOUT, Core.CoreSTDERR} global write(io::CoreIO, x::UInt8) = Core.write(io, x) diff --git a/base/exports.jl b/base/exports.jl index 0a8e473055b4b2..da0019ab469877 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -803,6 +803,7 @@ export # I/O and events close, + shutdown, countlines, eachline, readeach, diff --git a/base/io.jl b/base/io.jl index 30a87aa9e1cf33..2eface1ca94581 100644 --- a/base/io.jl +++ b/base/io.jl @@ -60,9 +60,49 @@ function isopen end Close an I/O stream. Performs a [`flush`](@ref) first. """ function close end + +""" + shutdown(stream) + +Shutdown the write half of a full-duplex I/O stream. Performs a [`flush`](@ref) +first. Notify the other end that no more data will be written to the underlying +file. This is not supported by all IO types. + +# Examples +```jldoctest +julia> io = Base.BufferStream(); # this never blocks, so we can read and write on the same Task + +julia> write(io, "request"); + +julia> # calling `read(io)` here would block forever + +julia> shutdown(io); + +julia> read(io, String) +"request" +""" +function shutdown end + +""" + flush(stream) + +Commit all currently buffered writes to the given stream. +""" function flush end -function wait_readnb end -function wait_close end + +""" + bytesavailable(io) + +Return the number of bytes available for reading before a read from this stream or buffer will block. + +# Examples +```jldoctest +julia> io = IOBuffer("JuliaLang is a GitHub organization"); + +julia> bytesavailable(io) +34 +``` +""" function bytesavailable end """ @@ -81,7 +121,7 @@ function readavailable end """ isreadable(io) -> Bool -Return `true` if the specified IO object is readable (if that can be determined). +Return `false` if the specified IO object is not readable. # Examples ```jldoctest @@ -99,12 +139,12 @@ true julia> rm("myfile.txt") ``` """ -function isreadable end +isreadable(io::IO) = isopen(io) """ iswritable(io) -> Bool -Return `true` if the specified IO object is writable (if that can be determined). +Return `false` if the specified IO object is not writable. # Examples ```jldoctest @@ -122,10 +162,23 @@ false julia> rm("myfile.txt") ``` """ -function iswritable end -function copy end +iswritable(io::IO) = isopen(io) + +""" + eof(stream) -> Bool + +Test whether an I/O stream is at end-of-file. If the stream is not yet exhausted, this +function will block to wait for more data if necessary, and then return `false`. Therefore +it is always safe to read one byte after seeing `eof` return `false`. `eof` will return +`false` as long as buffered data is still available, even if the remote end of a connection +is closed. +""" function eof end +function copy end +function wait_readnb end +function wait_close end + """ read(io::IO, T) @@ -357,65 +410,37 @@ end function pipe_reader end function pipe_writer end +for f in (:flush, :shutdown, :iswritable) + @eval $(f)(io::AbstractPipe) = $(f)(pipe_writer(io)::IO) +end write(io::AbstractPipe, byte::UInt8) = write(pipe_writer(io)::IO, byte) write(to::IO, from::AbstractPipe) = write(to, pipe_reader(from)) unsafe_write(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_write(pipe_writer(io)::IO, p, nb)::Union{Int,UInt} buffer_writes(io::AbstractPipe, args...) = buffer_writes(pipe_writer(io)::IO, args...) -flush(io::AbstractPipe) = flush(pipe_writer(io)::IO) +for f in ( + # peek/mark interface + :mark, :unmark, :reset, :ismarked, + # Simple reader functions + :read, :readavailable, :bytesavailable, :reseteof, :isreadable) + @eval $(f)(io::AbstractPipe) = $(f)(pipe_reader(io)::IO) +end read(io::AbstractPipe, byte::Type{UInt8}) = read(pipe_reader(io)::IO, byte)::UInt8 unsafe_read(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_read(pipe_reader(io)::IO, p, nb) -read(io::AbstractPipe) = read(pipe_reader(io)::IO) readuntil(io::AbstractPipe, arg::UInt8; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) readuntil(io::AbstractPipe, arg::AbstractChar; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) readuntil(io::AbstractPipe, arg::AbstractString; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) readuntil(io::AbstractPipe, arg::AbstractVector; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) readuntil_vector!(io::AbstractPipe, target::AbstractVector, keep::Bool, out) = readuntil_vector!(pipe_reader(io)::IO, target, keep, out) readbytes!(io::AbstractPipe, target::AbstractVector{UInt8}, n=length(target)) = readbytes!(pipe_reader(io)::IO, target, n) - -for f in ( - # peek/mark interface - :mark, :unmark, :reset, :ismarked, - # Simple reader functions - :readavailable, :isreadable) - @eval $(f)(io::AbstractPipe) = $(f)(pipe_reader(io)::IO) -end peek(io::AbstractPipe, ::Type{T}) where {T} = peek(pipe_reader(io)::IO, T)::T +wait_readnb(io::AbstractPipe, nb::Int) = wait_readnb(pipe_reader(io)::IO, nb) +eof(io::AbstractPipe) = eof(pipe_reader(io)::IO)::Bool -iswritable(io::AbstractPipe) = iswritable(pipe_writer(io)::IO) isopen(io::AbstractPipe) = isopen(pipe_writer(io)::IO) || isopen(pipe_reader(io)::IO) close(io::AbstractPipe) = (close(pipe_writer(io)::IO); close(pipe_reader(io)::IO)) -wait_readnb(io::AbstractPipe, nb::Int) = wait_readnb(pipe_reader(io)::IO, nb) wait_close(io::AbstractPipe) = (wait_close(pipe_writer(io)::IO); wait_close(pipe_reader(io)::IO)) -""" - bytesavailable(io) - -Return the number of bytes available for reading before a read from this stream or buffer will block. - -# Examples -```jldoctest -julia> io = IOBuffer("JuliaLang is a GitHub organization"); - -julia> bytesavailable(io) -34 -``` -""" -bytesavailable(io::AbstractPipe) = bytesavailable(pipe_reader(io)::IO) -bytesavailable(io::DevNull) = 0 - -""" - eof(stream) -> Bool - -Test whether an I/O stream is at end-of-file. If the stream is not yet exhausted, this -function will block to wait for more data if necessary, and then return `false`. Therefore -it is always safe to read one byte after seeing `eof` return `false`. `eof` will return -`false` as long as buffered data is still available, even if the remote end of a connection -is closed. -""" -eof(io::AbstractPipe) = eof(pipe_reader(io)::IO)::Bool -reseteof(io::AbstractPipe) = reseteof(pipe_reader(io)::IO) - # Exception-safe wrappers (io = open(); try f(io) finally close(io)) @@ -1119,11 +1144,6 @@ ismarked(io::IO) = io.mark >= 0 # Make sure all IO streams support flush, even if only as a no-op, # to make it easier to write generic I/O code. -""" - flush(stream) - -Commit all currently buffered writes to the given stream. -""" flush(io::IO) = nothing """ diff --git a/base/iobuffer.jl b/base/iobuffer.jl index e204eca906cbf0..f65ed3f894fe00 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -334,6 +334,12 @@ end eof(io::GenericIOBuffer) = (io.ptr-1 == io.size) +function shutdown(io::GenericIOBuffer) + io.writable = false + # OR throw(_UVError("shutdown", UV_ENOTSOCK)) + nothing +end + @noinline function close(io::GenericIOBuffer{T}) where T io.readable = false io.writable = false diff --git a/base/libuv.jl b/base/libuv.jl index c63045f4b1b68c..c64cbff564b66d 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -107,6 +107,7 @@ end function uv_alloc_buf end function uv_readcb end function uv_writecb_task end +function uv_shutdowncb_task end function uv_return_spawn end function uv_asynccb end function uv_timercb end diff --git a/base/process.jl b/base/process.jl index b3ec79fa1ab4ed..6be7099f94f358 100644 --- a/base/process.jl +++ b/base/process.jl @@ -275,6 +275,7 @@ function setup_stdio(stdio::Union{IOBuffer, BufferStream}, child_readable::Bool) @warn "Process error" exception=(ex, catch_backtrace()) finally close(parent) + child_readable || shutdown(stdio) end end catch ex diff --git a/base/range.jl b/base/range.jl index b6a8e333cf52e1..4ddecd7bdd91c1 100644 --- a/base/range.jl +++ b/base/range.jl @@ -663,8 +663,8 @@ function checked_length(r::OrdinalRange{T}) where T else diff = checked_sub(stop, start) end - a = Integer(div(diff, s)) - return checked_add(a, oneunit(a)) + a = div(diff, s) + return Integer(checked_add(a, oneunit(a))) end function checked_length(r::AbstractUnitRange{T}) where T @@ -672,8 +672,8 @@ function checked_length(r::AbstractUnitRange{T}) where T if isempty(r) return Integer(first(r) - first(r)) end - a = Integer(checked_add(checked_sub(last(r), first(r)))) - return checked_add(a, oneunit(a)) + a = checked_sub(last(r), first(r)) + return Integer(checked_add(a, oneunit(a))) end function length(r::OrdinalRange{T}) where T @@ -690,15 +690,14 @@ function length(r::OrdinalRange{T}) where T else diff = stop - start end - a = Integer(div(diff, s)) - return a + oneunit(a) + a = div(diff, s) + return Integer(a + oneunit(a)) end - function length(r::AbstractUnitRange{T}) where T @_inline_meta - a = Integer(last(r) - first(r)) # even when isempty, by construction (with overflow) - return a + oneunit(a) + a = last(r) - first(r) # even when isempty, by construction (with overflow) + return Integer(a + oneunit(a)) end length(r::OneTo) = Integer(r.stop - zero(r.stop)) diff --git a/base/rational.jl b/base/rational.jl index 23c9298962f53c..1c83443d10df3a 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -533,3 +533,21 @@ function hash(x::Rational{<:BitInteger64}, h::UInt) h = hash_integer(num, h) return h end + +# These methods are only needed for performance. Since `first(r)` and `last(r)` have the +# same denominator (because their difference is an integer), `length(r)` can be calulated +# without calling `gcd`. +function length(r::AbstractUnitRange{T}) where T<:Rational + @_inline_meta + f = first(r) + l = last(r) + return div(l.num - f.num + f.den, f.den) +end +function checked_length(r::AbstractUnitRange{T}) where T<:Rational + f = first(r) + l = last(r) + if isempty(r) + return f.num - f.num + end + return div(checked_add(checked_sub(l.num, f.num), f.den), f.den) +end diff --git a/base/set.jl b/base/set.jl index 5a744c556432c3..e2e6a83b0a4c5b 100644 --- a/base/set.jl +++ b/base/set.jl @@ -548,6 +548,9 @@ replaced. See also [`replace!`](@ref), [`splice!`](@ref), [`delete!`](@ref), [`insert!`](@ref). +!!! compat "Julia 1.7" + Version 1.7 is required to replace elements of a `Tuple`. + # Examples ```jldoctest julia> replace([1, 2, 1, 3], 1=>0, 2=>4, count=2) @@ -596,6 +599,9 @@ Return a copy of `A` where each value `x` in `A` is replaced by `new(x)`. If `count` is specified, then replace at most `count` values in total (replacements being defined as `new(x) !== x`). +!!! compat "Julia 1.7" + Version 1.7 is required to replace elements of a `Tuple`. + # Examples ```jldoctest julia> replace(x -> isodd(x) ? 2x : x, [1, 2, 3, 4]) diff --git a/base/stream.jl b/base/stream.jl index 6cbd1d3b86a280..6e433b771f0d2f 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -109,7 +109,7 @@ function eof(s::LibuvStream) # and that we won't return true if there's a readerror pending (it'll instead get thrown). # This requires some careful ordering here (TODO: atomic loads) bytesavailable(s) > 0 && return false - open = isopen(s) # must precede readerror check + open = isreadable(s) # must precede readerror check s.readerror === nothing || throw(s.readerror) return !open end @@ -270,6 +270,7 @@ show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(", function isreadable(io::LibuvStream) bytesavailable(io) > 0 && return true isopen(io) || return false + io.status == StatusEOF && return false return ccall(:uv_is_readable, Cint, (Ptr{Cvoid},), io.handle) != 0 end @@ -378,7 +379,7 @@ function isopen(x::Union{LibuvStream, LibuvServer}) if x.status == StatusUninit || x.status == StatusInit throw(ArgumentError("$x is not initialized")) end - return x.status != StatusClosed && x.status != StatusEOF + return x.status != StatusClosed end function check_open(x::Union{LibuvStream, LibuvServer}) @@ -390,13 +391,13 @@ end function wait_readnb(x::LibuvStream, nb::Int) # fast path before iolock acquire bytesavailable(x.buffer) >= nb && return - open = isopen(x) # must precede readerror check + open = isopen(x) && x.status != StatusEOF # must precede readerror check x.readerror === nothing || throw(x.readerror) open || return iolock_begin() # repeat fast path after iolock acquire, before other expensive work bytesavailable(x.buffer) >= nb && (iolock_end(); return) - open = isopen(x) + open = isopen(x) && x.status != StatusEOF x.readerror === nothing || throw(x.readerror) open || (iolock_end(); return) # now do the "real" work @@ -407,6 +408,7 @@ function wait_readnb(x::LibuvStream, nb::Int) while bytesavailable(x.buffer) < nb x.readerror === nothing || throw(x.readerror) isopen(x) || break + x.status != StatusEOF || break x.throttle = max(nb, x.throttle) start_reading(x) # ensure we are reading iolock_end() @@ -431,6 +433,50 @@ function wait_readnb(x::LibuvStream, nb::Int) nothing end +function shutdown(s::LibuvStream) + iolock_begin() + check_open(s) + req = Libc.malloc(_sizeof_uv_shutdown) + uv_req_set_data(req, C_NULL) # in case we get interrupted before arriving at the wait call + err = ccall(:uv_shutdown, Int32, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}), + req, s, @cfunction(uv_shutdowncb_task, Cvoid, (Ptr{Cvoid}, Cint))) + if err < 0 + Libc.free(req) + uv_error("shutdown", err) + end + ct = current_task() + preserve_handle(ct) + sigatomic_begin() + uv_req_set_data(req, ct) + iolock_end() + status = try + sigatomic_end() + wait()::Cint + finally + # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end + sigatomic_end() + iolock_begin() + ct.queue === nothing || list_deletefirst!(ct.queue, ct) + if uv_req_data(req) != C_NULL + # req is still alive, + # so make sure we won't get spurious notifications later + uv_req_set_data(req, C_NULL) + else + # done with req + Libc.free(req) + end + iolock_end() + unpreserve_handle(ct) + if isopen(s) && (s.status == StatusEOF && !isa(s, TTY)) || ccall(:uv_is_readable, Cint, (Ptr{Cvoid},), s.handle) == 0 + close(s) + end + end + if status < 0 + throw(_UVError("shutdown", status)) + end + nothing +end + function wait_close(x::Union{LibuvStream, LibuvServer}) preserve_handle(x) lock(x.cond) @@ -451,7 +497,7 @@ function close(stream::Union{LibuvStream, LibuvServer}) if stream.status == StatusInit ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), stream.handle) stream.status = StatusClosing - elseif isopen(stream) || stream.status == StatusEOF + elseif isopen(stream) should_wait = uv_handle_data(stream) != C_NULL if stream.status != StatusClosing ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), stream.handle) @@ -606,35 +652,33 @@ function uv_readcb(handle::Ptr{Cvoid}, nread::Cssize_t, buf::Ptr{Cvoid}) nrequested = ccall(:jl_uv_buf_len, Csize_t, (Ptr{Cvoid},), buf) function readcb_specialized(stream::LibuvStream, nread::Int, nrequested::UInt) lock(stream.cond) - try - if nread < 0 - if nread == UV_ENOBUFS && nrequested == 0 - # remind the client that stream.buffer is full - notify(stream.cond) - elseif nread == UV_EOF - if isa(stream, TTY) - stream.status = StatusEOF # libuv called uv_stop_reading already + if nread < 0 + if nread == UV_ENOBUFS && nrequested == 0 + # remind the client that stream.buffer is full + notify(stream.cond) + elseif nread == UV_EOF # libuv called uv_stop_reading already + if stream.status != StatusClosing + if stream isa TTY || ccall(:uv_is_writable, Cint, (Ptr{Cvoid},), stream.handle) != 0 + # stream can still be used either by reseteof or write + stream.status = StatusEOF notify(stream.cond) - elseif stream.status != StatusClosing - # begin shutdown of the stream + else + # underlying stream is no longer useful: begin finalization ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), stream.handle) stream.status = StatusClosing end - else - stream.readerror = _UVError("read", nread) - # This is a fatal connection error. Shutdown requests as per the usual - # close function won't work and libuv will fail with an assertion failure - ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), stream) - stream.status = StatusClosing - notify(stream.cond) end else - notify_filled(stream.buffer, nread) - notify(stream.cond) + stream.readerror = _UVError("read", nread) + # This is a fatal connection error + ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), stream.handle) + stream.status = StatusClosing end - finally - unlock(stream.cond) + else + notify_filled(stream.buffer, nread) + notify(stream.cond) end + unlock(stream.cond) # Stop background reading when # 1) there's nobody paying attention to the data we are reading @@ -651,6 +695,7 @@ function uv_readcb(handle::Ptr{Cvoid}, nread::Cssize_t, buf::Ptr{Cvoid}) nothing end readcb_specialized(stream_unknown_type, Int(nread), UInt(nrequested)) + nothing end function reseteof(x::TTY) @@ -844,6 +889,7 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) while bytesavailable(buf) < nb s.readerror === nothing || throw(s.readerror) isopen(s) || break + s.status != StatusEOF || break iolock_end() wait_readnb(s, nb) iolock_begin() @@ -890,6 +936,7 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) while bytesavailable(buf) < nb s.readerror === nothing || throw(s.readerror) isopen(s) || throw(EOFError()) + s.status != StatusEOF || throw(EOFError()) iolock_end() wait_readnb(s, nb) iolock_begin() @@ -946,13 +993,14 @@ function readuntil(x::LibuvStream, c::UInt8; keep::Bool=false) @assert buf.seekable == false if !occursin(c, buf) # fast path checks first x.readerror === nothing || throw(x.readerror) - if isopen(x) + if isopen(x) && x.status != StatusEOF preserve_handle(x) lock(x.cond) try while !occursin(c, x.buffer) x.readerror === nothing || throw(x.readerror) isopen(x) || break + x.status != StatusEOF || break start_reading(x) # ensure we are reading iolock_end() wait(x.cond) @@ -1115,6 +1163,20 @@ function uv_writecb_task(req::Ptr{Cvoid}, status::Cint) nothing end +function uv_shutdowncb_task(req::Ptr{Cvoid}, status::Cint) + d = uv_req_data(req) + if d != C_NULL + uv_req_set_data(req, C_NULL) # let the Task know we got the shutdowncb + t = unsafe_pointer_to_objref(d)::Task + schedule(t, status) + else + # no owner for this req, safe to just free it + Libc.free(req) + end + nothing +end + + _fd(x::IOStream) = RawFD(fd(x)) _fd(x::Union{OS_HANDLE, RawFD}) = x @@ -1405,18 +1467,20 @@ mutable struct BufferStream <: LibuvStream buffer::IOBuffer cond::Threads.Condition readerror::Any - is_open::Bool buffer_writes::Bool lock::ReentrantLock # advisory lock + status::Int - BufferStream() = new(PipeBuffer(), Threads.Condition(), nothing, true, false, ReentrantLock()) + BufferStream() = new(PipeBuffer(), Threads.Condition(), nothing, false, ReentrantLock(), StatusActive) end -isopen(s::BufferStream) = s.is_open +isopen(s::BufferStream) = s.status != StatusClosed + +shutdown(s::BufferStream) = close(s) function close(s::BufferStream) lock(s.cond) do - s.is_open = false + s.status = StatusClosed notify(s.cond) nothing end @@ -1439,8 +1503,8 @@ function unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt) end bytesavailable(s::BufferStream) = bytesavailable(s.buffer) -isreadable(s::BufferStream) = s.buffer.readable -iswritable(s::BufferStream) = s.buffer.writable +isreadable(s::BufferStream) = (isopen(s) || bytesavailable(s) > 0) && s.buffer.readable +iswritable(s::BufferStream) = isopen(s) && s.buffer.writable function wait_readnb(s::BufferStream, nb::Int) lock(s.cond) do @@ -1450,7 +1514,7 @@ function wait_readnb(s::BufferStream, nb::Int) end end -show(io::IO, s::BufferStream) = print(io, "BufferStream() bytes waiting:", bytesavailable(s.buffer), ", isopen:", s.is_open) +show(io::IO, s::BufferStream) = print(io, "BufferStream(bytes waiting=", bytesavailable(s.buffer), ", isopen=", isopen(s), ")") function readuntil(s::BufferStream, c::UInt8; keep::Bool=false) bytes = lock(s.cond) do diff --git a/base/timing.jl b/base/timing.jl index ab7af23048305b..8e7303a8e1361d 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -55,9 +55,11 @@ function gc_alloc_count(diff::GC_Diff) diff.malloc + diff.realloc + diff.poolalloc + diff.bigalloc end -# cumulative total time spent on compilation +# cumulative total time spent on compilation, in nanoseconds cumulative_compile_time_ns_before() = ccall(:jl_cumulative_compile_time_ns_before, UInt64, ()) cumulative_compile_time_ns_after() = ccall(:jl_cumulative_compile_time_ns_after, UInt64, ()) +# cumulative total time the process has spent on compilation while measurement was enabled. +cumulative_compile_time_ns() = ccall(:jl_cumulative_compile_time_ns, UInt64, ()) # total time spend in garbage collection, in nanoseconds gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ()) diff --git a/contrib/bpftrace/gc_all.bt b/contrib/bpftrace/gc_all.bt new file mode 100755 index 00000000000000..f78e8f3aa607d8 --- /dev/null +++ b/contrib/bpftrace/gc_all.bt @@ -0,0 +1,44 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Tracing Julia GC Times... Hit Ctrl-C to end.\n"); +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__begin +{ + $now = nsecs; + @time[pid] = $now; + @start[pid] = $now; +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__stop_the_world +/@start[pid]/ +{ + $now = nsecs; + @stop_the_world_usecs[pid] = hist(($now - @time[pid]) / 1000); + @time[pid] = $now; +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__end +/@start[pid]/ +{ + $now = nsecs; + @gc_total_usecs[pid] = hist(($now - @start[pid]) / 1000); + @gc_phase_usecs[pid] = hist(($now - @time[pid]) / 1000); + @time[pid] = $now; + delete(@start[pid]); +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__finalizer +/@time[pid]/ +{ + @finalizer[pid] = hist((nsecs - @time[pid]) / 1000); + delete(@time[pid]); +} + +END +{ + clear(@start); + clear(@time); +} diff --git a/contrib/bpftrace/gc_simple.bt b/contrib/bpftrace/gc_simple.bt new file mode 100755 index 00000000000000..559f41c41cf72c --- /dev/null +++ b/contrib/bpftrace/gc_simple.bt @@ -0,0 +1,23 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Tracing Julia GC Times... Hit Ctrl-C to end.\n"); +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__begin +{ + @start[pid] = nsecs; +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__end +/@start[pid]/ +{ + @usecs[pid] = hist((nsecs - @start[pid]) / 1000); + delete(@start[pid]); +} + +END +{ + clear(@start); +} diff --git a/contrib/bpftrace/gc_stop_the_world_latency.bt b/contrib/bpftrace/gc_stop_the_world_latency.bt new file mode 100755 index 00000000000000..8e541bcb421e2d --- /dev/null +++ b/contrib/bpftrace/gc_stop_the_world_latency.bt @@ -0,0 +1,23 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Tracing Julia GC Stop-The-World Latency... Hit Ctrl-C to end.\n"); +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__begin +{ + @start[pid] = nsecs; +} + +usdt:usr/lib/libjulia-internal.so:julia:gc__stop_the_world +/@start[pid]/ +{ + @usecs[pid] = hist((nsecs - @start[pid]) / 1000); + delete(@start[pid]); +} + +END +{ + clear(@start); +} diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index 1ccbb16a9ba4f2..5b8a25ab79b917 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -8,6 +8,11 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) +# force a sane / stable configuration +export LC_ALL=C +export LANG=C +.SUFFIXES: + # Default target that will have everything else added to it as a dependency all: checksum pack-checksum diff --git a/deps/checksums/libuv b/deps/checksums/libuv index f03a3c6ce9a359..f3ecef65865474 100644 --- a/deps/checksums/libuv +++ b/deps/checksums/libuv @@ -1,34 +1,34 @@ -libuv-fb3e3364c33ae48c827f6b103e05c3f0e78b79a9.tar.gz/md5/dc93ae5119c8934f374570342ef036ed -libuv-fb3e3364c33ae48c827f6b103e05c3f0e78b79a9.tar.gz/sha512/29947c236aef8931be4767df1cd8404ee9b036ee107b31cbce6fad9a97743df57d068b15bc4bd00320b9b81cd879258a9ec9dc675853e424ccdb8d6bdd226240 -LibUV.v2.0.1+2.aarch64-apple-darwin.tar.gz/md5/ed00585eb80fd82c014e2a431269ccec -LibUV.v2.0.1+2.aarch64-apple-darwin.tar.gz/sha512/a98ffde4ff49a71699f798622c62b5f95d0dc010f1de88ad57ee437baa73cb25e263a8a6c4de86364fb31076993326d9bd0223db3e1ecf6904c1aa6e7e1f0120 -LibUV.v2.0.1+2.aarch64-linux-gnu.tar.gz/md5/3e75495795d5a4eee8ec9c1619a5caaa -LibUV.v2.0.1+2.aarch64-linux-gnu.tar.gz/sha512/c0f1396ccc7784772d4c40f3a62d6bb22c6859a3258b07727348f436b7991a8f6d51ec46c09569f17a7bd600a321ab3b3cd59538d39c228cd3e205c33e755a51 -LibUV.v2.0.1+2.aarch64-linux-musl.tar.gz/md5/c2899ea791cfcd37ff85c1182330168e -LibUV.v2.0.1+2.aarch64-linux-musl.tar.gz/sha512/b340ff4e28a5e566ee2640926265b2070acfcc4b3c87fc2e414e2b2a9ff23be852d92ff4f51e36e21de029f23bca5524e7e267ba091401b070f4d5cd9bd03c54 -LibUV.v2.0.1+2.armv6l-linux-gnueabihf.tar.gz/md5/6b6c080a88050051100af58a4e96f25d -LibUV.v2.0.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/936a4f4baf21a0fe7492bc3fab3475f653824daa184030df764af1eb5e71d152aa5dd3449b1cf31f77a460169853d7371597056e641c440c7b3d4f5c6be1ce10 -LibUV.v2.0.1+2.armv6l-linux-musleabihf.tar.gz/md5/032010ad683931906d2467753cebea9e -LibUV.v2.0.1+2.armv6l-linux-musleabihf.tar.gz/sha512/b8050662775d75e59b072c688ae44f7a3d3f54d114270902a825e01f4d74c8e131a3a75cd95e31b9ebf4488d64ff6170a67e5986e02e5fcb105bf5d3cc28706b -LibUV.v2.0.1+2.armv7l-linux-gnueabihf.tar.gz/md5/e558bedc0b69d6575e43df0eec958ad9 -LibUV.v2.0.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/d3404b20b7e8e8fe935ca1e7da55823a6ff6703c822acf622638dc5c744bfefe1745e8e3a67054abc3aec0c10793ac46dbab29ccf7269d8a3a0d857e3a1a93e7 -LibUV.v2.0.1+2.armv7l-linux-musleabihf.tar.gz/md5/1aa605d9930ba63874483defb35a96ba -LibUV.v2.0.1+2.armv7l-linux-musleabihf.tar.gz/sha512/927e710191e6b8d1c09054780b4af6336c6744ceb0885c7a5a0cec5e08bfb0d53ede75cc8bb145fda08f720d98a77e102a2903e08a0fef75a0b630631db6f35e -LibUV.v2.0.1+2.i686-linux-gnu.tar.gz/md5/4ec3415ef12615581f8b26ec374a35bf -LibUV.v2.0.1+2.i686-linux-gnu.tar.gz/sha512/fd37bb83cda297ec80332cdaed2a704ea43c3ec72fb539042ef09aa510275a0418c750278fca9e463bdecdca957f8457103f0be6eeae1017387141eb2b906694 -LibUV.v2.0.1+2.i686-linux-musl.tar.gz/md5/f24ea24837ef06be346d239cbb33ae7e -LibUV.v2.0.1+2.i686-linux-musl.tar.gz/sha512/e9388568b20fa71d95e331a336aa3b17396e87d99aef6d752cb48416cdc9501e2ea887702a5765a22dcf6b5f7b730f5666ed3a639e7fe0113e9032f0d760b352 -LibUV.v2.0.1+2.i686-w64-mingw32.tar.gz/md5/890d1f7963a5dc927c15f8433b69dcf7 -LibUV.v2.0.1+2.i686-w64-mingw32.tar.gz/sha512/7e7d2b6405bbb1b62725a61d649fcbd53c2dcb65b8a6deea5a186717f88dbab4198a0f58d4223500aa991976725f8e1c4272ab29866174c5f555ba75a2e9b0ee -LibUV.v2.0.1+2.powerpc64le-linux-gnu.tar.gz/md5/7fa0d0e9344f4a4c4d5075ec5d368b0a -LibUV.v2.0.1+2.powerpc64le-linux-gnu.tar.gz/sha512/b5587e9e1072bc6becd5d1354294a3afcfda1c52e9a5f56387d43c7300369106059a2bac8669a919ce25d888b2302711c7433a82e366648935481568420daeef -LibUV.v2.0.1+2.x86_64-apple-darwin.tar.gz/md5/716960539cbae1e38e1cf88c2670927d -LibUV.v2.0.1+2.x86_64-apple-darwin.tar.gz/sha512/7b064d99428b312302c698e73e8a7919147c0522857a24e08d16144aea83429c5ac9526b6553697f28784457a5b417958fc5e4e28b4191861004dddc3f95566c -LibUV.v2.0.1+2.x86_64-linux-gnu.tar.gz/md5/ff70887943a3fc68eddcb66ed941417e -LibUV.v2.0.1+2.x86_64-linux-gnu.tar.gz/sha512/00610022d700dd6b33c97decea43490fcd4218fde2e57c0d6317abec046adf220fdf4d03f132938ec78af85653a5262d1344527c632c06aec53750710a6b317c -LibUV.v2.0.1+2.x86_64-linux-musl.tar.gz/md5/a5834444d0b7e7d88cc87e5eb458bca3 -LibUV.v2.0.1+2.x86_64-linux-musl.tar.gz/sha512/e2e6e6726e8ef0962c35d7ff54a60b3370cd5b927fda8b4415e8d2f19b098ed9bd00e262eb18d11a73e2e27c88aefa72c3a6e9c193d27eab436c4d9d6531cd47 -LibUV.v2.0.1+2.x86_64-unknown-freebsd.tar.gz/md5/951d9da43208d2c48eb00c7ce300b4cf -LibUV.v2.0.1+2.x86_64-unknown-freebsd.tar.gz/sha512/87e578f6cf34c9cc1c965f4958048967740b4ab530836aff33b3339c0d927beccf1f0c58f7e256c9ba98bf1fa0362186a24fcc5bb79ae1f149f86183b4b7f5c1 -LibUV.v2.0.1+2.x86_64-w64-mingw32.tar.gz/md5/4e9c2f078ed7b617a1aa447e1c44abbf -LibUV.v2.0.1+2.x86_64-w64-mingw32.tar.gz/sha512/b3b14c5d447cd742cade43b56bf3867d530dd391c105ddbd7f2b9e0e26ee6a1f3e6fa11148a9ba1540fa598b155da3e56d369a96273a5ea1343b5c3cd4821953 +LibUV.v2.0.1+3.aarch64-apple-darwin.tar.gz/md5/7c1f08965b45726099a940ab28a79207 +LibUV.v2.0.1+3.aarch64-apple-darwin.tar.gz/sha512/9f9d170fc36e2e30c53e96cdf9ae7cd4eb969905db65119627d4a9e10abb5e41aeb4ab07705d571300d44d4894c9d179c6a9450e3e6415042a6131bae0971c26 +LibUV.v2.0.1+3.aarch64-linux-gnu.tar.gz/md5/8d59850a63ea71c4007d08f7d23ceda2 +LibUV.v2.0.1+3.aarch64-linux-gnu.tar.gz/sha512/f662b36ce6e336e724c6fd03482d91a70e7ae8cfe7a1fff98cbca2cdb99b0cd3475bf32e57547097ca625c1fceb57c989871f391faea9227a6d6d4faf7649175 +LibUV.v2.0.1+3.aarch64-linux-musl.tar.gz/md5/3ad43e49a7a996b50341e7150064f13d +LibUV.v2.0.1+3.aarch64-linux-musl.tar.gz/sha512/69c6675fee6647eb7a2c8c680bd49ba31c1dcda676530af1ad1ca979c0cf8d9c908e0cb246608718d4bbca4d73174751c608aa80e3f11557ed4d05cc1d270021 +LibUV.v2.0.1+3.armv6l-linux-gnueabihf.tar.gz/md5/fec375c1c45fbd1b2eb22fbd5f727e5f +LibUV.v2.0.1+3.armv6l-linux-gnueabihf.tar.gz/sha512/1b279406cdb43bf1c3fd6b1d2d24fe9b5ca75a65212b5720d5406af26a6b7551b18efb0471c884e98e97c50693412344fd733c3ef19ea8fecf1c2c26ae888492 +LibUV.v2.0.1+3.armv6l-linux-musleabihf.tar.gz/md5/07790f330f3394d0a3ea47aa56529be1 +LibUV.v2.0.1+3.armv6l-linux-musleabihf.tar.gz/sha512/4fd4bf7c1b333907fbbbfdca213c91cb6b387e56886b6349271d7a3c1ddb3f9349f0799c60178de507e039289662c310829d81be804c5076ce9ae16eb61c8cb1 +LibUV.v2.0.1+3.armv7l-linux-gnueabihf.tar.gz/md5/2ce38a69564e279b0deb32a6af1acc52 +LibUV.v2.0.1+3.armv7l-linux-gnueabihf.tar.gz/sha512/d520ef8c556db6a42534c4abdd59f4b64ebd77cdb7b2972385d6a90463e0a490ca168bb83128e66b13555128305f85d5c979a739f7c369a11f8217e49505ce0a +LibUV.v2.0.1+3.armv7l-linux-musleabihf.tar.gz/md5/034d903cb13a45278df742df23c46275 +LibUV.v2.0.1+3.armv7l-linux-musleabihf.tar.gz/sha512/f900fd47320628ac3cf0ba0b9d9408ef4e46ed508f5efc00b94b532c5d713fcf4695827f978f605b3d9f0a1db42abcd0b1fec6a6c5bb6a502ed5f0fed55f4a32 +LibUV.v2.0.1+3.i686-linux-gnu.tar.gz/md5/60c51d15d9ae2ac8418b71730dc093d0 +LibUV.v2.0.1+3.i686-linux-gnu.tar.gz/sha512/84841f6fb4c744e335d6ce25e3059d9aeeaa54b823fe714f43fa8b11caf209e539afb9605e96992d075ae16a664da6c3a3c3d4cdb905d42c0ddc8722267c19af +LibUV.v2.0.1+3.i686-linux-musl.tar.gz/md5/d70ffffd8e57dfbcb4265ad202722fca +LibUV.v2.0.1+3.i686-linux-musl.tar.gz/sha512/c76dc0df03f7a683e08313baa0d8471b45c64099e13cf71a6595cb50c9e8ff1e96c31c400f83144ee816e5a7c8f01ad6f48d8b50a6cd161e0d246737780c2a9e +LibUV.v2.0.1+3.i686-w64-mingw32.tar.gz/md5/aa634a6e49c5683dcca287f2cf1ac71e +LibUV.v2.0.1+3.i686-w64-mingw32.tar.gz/sha512/ac1b2b53cbd7e317866259c30de349c453deda2c970587e4f4b40605f966f722fe8250e9dd7677ded0c928c9398240b7e4867805bb1404fb9d0c7dfab8493c79 +LibUV.v2.0.1+3.powerpc64le-linux-gnu.tar.gz/md5/9233d6bb298bd5020b680c25f742de98 +LibUV.v2.0.1+3.powerpc64le-linux-gnu.tar.gz/sha512/172253d1e6ce888865f5fd2182aad6426ff9988313937c7abe80ccdba859289f7ec8997574bb853f9786596d90414c4a926a475c1817c17574e0dd2ea8ad68ad +LibUV.v2.0.1+3.x86_64-apple-darwin.tar.gz/md5/09b83fe0ced427136ea5680cea64b004 +LibUV.v2.0.1+3.x86_64-apple-darwin.tar.gz/sha512/1842df6b14f23cc42c1d29546aa00b4b07390646beb312092bceb00d1de855116368ddcdd4ccd8525741fb6ecd26312c90dc9342d2e1ef4d36fbb896bdd2cbd3 +LibUV.v2.0.1+3.x86_64-linux-gnu.tar.gz/md5/d54b4fbfd998228e80c42c89df2c14a1 +LibUV.v2.0.1+3.x86_64-linux-gnu.tar.gz/sha512/290b2432e72712e73f20aa0ea6f787f9ffbf9ab5e3249c708355f7d57fb3b40ec3913e0c9a7141197caf3a741d6edbb1788cdef7e1a9a8547117dcf40559e567 +LibUV.v2.0.1+3.x86_64-linux-musl.tar.gz/md5/7690b83baecf1b614f30c7c0b661d86d +LibUV.v2.0.1+3.x86_64-linux-musl.tar.gz/sha512/53950b56f87ea0acc13790bf3a82441ca4ccfcd66cf272c81915e999487e4973c8c474caa811584d5fa8873cdd18ac0944f3d6f415c483a26e38645eb1701cdb +LibUV.v2.0.1+3.x86_64-unknown-freebsd.tar.gz/md5/6d2a330a2e87b9b8564708fba963845f +LibUV.v2.0.1+3.x86_64-unknown-freebsd.tar.gz/sha512/f15daf2f5a1bda49234fe546e13f97280c843d6939c26c9f05133302ec81e38bd47a5098d4900e39290f913835cffbdaa0b2c6406d6a86dc2eb8d0159f461e9f +LibUV.v2.0.1+3.x86_64-w64-mingw32.tar.gz/md5/4f934cf8dd1d45b206af6a72c3e679f7 +LibUV.v2.0.1+3.x86_64-w64-mingw32.tar.gz/sha512/36ef56923d7cf5c31aba87fb75774ce68976f7b6b7971a4c86011da275a5472e28033a6d97b965887b7bbe335bedfb6970618b88f20c095228ffa5f783ab8eb1 +libuv-c6869fba163a1e04af64ede438a8fd0191e75e9e.tar.gz/md5/b60fc7b00bdfafcbbc66317858882058 +libuv-c6869fba163a1e04af64ede438a8fd0191e75e9e.tar.gz/sha512/197b386af51eb4456ce65e2951e033731e1194fca8bed08755a78360ebb3431ab4d8d69a75279e7995d2e4197133d613892e5b9b5d6411bffa692df35542420f diff --git a/deps/libuv.version b/deps/libuv.version index 339cba4441875c..045f329a0c9f39 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ -LIBUV_BRANCH=julia-uv2-1.39.0 -LIBUV_SHA1=fb3e3364c33ae48c827f6b103e05c3f0e78b79a9 +LIBUV_BRANCH=julia-uv2-1.42.0 +LIBUV_SHA1=c6869fba163a1e04af64ede438a8fd0191e75e9e diff --git a/doc/make.jl b/doc/make.jl index c58b3eebed7165..03798b50888dd2 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -155,6 +155,7 @@ DevDocs = [ "devdocs/debuggingtips.md", "devdocs/valgrind.md", "devdocs/sanitizers.md", + "devdocs/probes.md" ] ] diff --git a/doc/src/base/io-network.md b/doc/src/base/io-network.md index 2d6a462400813d..8b2ea3c4c2412f 100644 --- a/doc/src/base/io-network.md +++ b/doc/src/base/io-network.md @@ -13,6 +13,7 @@ Base.take!(::Base.GenericIOBuffer) Base.fdio Base.flush Base.close +Base.shutdown Base.write Base.read Base.read! diff --git a/doc/src/devdocs/probes.md b/doc/src/devdocs/probes.md new file mode 100644 index 00000000000000..7e2fb96df5ac8d --- /dev/null +++ b/doc/src/devdocs/probes.md @@ -0,0 +1,168 @@ +# Instrumenting Julia with DTrace, and bpftrace + +DTrace and bpftrace are tools that enable lightweight instrumentation of processes. +You can turn the instrumentation on and off while the process is running, +and with instrumentation off the overhead is minimal. + +!!! compat "Julia 1.8" + Support for probes was added in Julia 1.8 + +!!! note + This documentation has been written from a Linux perspective, most of this + should hold on Mac OS/Darwin and FreeBSD. + +## Enabling support + +On Linux install the `systemtap` package that has a version of `dtrace`. + +``` +WITH_DTRACE=1 +``` + +### Verifying + +``` +> readelf -n usr/lib/libjulia-internal.so.1 + +Displaying notes found in: .note.gnu.build-id + Owner Data size Description + GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID bitstring) + Build ID: 57161002f35548772a87418d2385c284ceb3ead8 + +Displaying notes found in: .note.stapsdt + Owner Data size Description + stapsdt 0x00000029 NT_STAPSDT (SystemTap probe descriptors) + Provider: julia + Name: gc__begin + Location: 0x000000000013213e, Base: 0x00000000002bb4da, Semaphore: 0x0000000000346cac + Arguments: + stapsdt 0x00000032 NT_STAPSDT (SystemTap probe descriptors) + Provider: julia + Name: gc__stop_the_world + Location: 0x0000000000132144, Base: 0x00000000002bb4da, Semaphore: 0x0000000000346cae + Arguments: + stapsdt 0x00000027 NT_STAPSDT (SystemTap probe descriptors) + Provider: julia + Name: gc__end + Location: 0x000000000013214a, Base: 0x00000000002bb4da, Semaphore: 0x0000000000346cb0 + Arguments: + stapsdt 0x0000002d NT_STAPSDT (SystemTap probe descriptors) + Provider: julia + Name: gc__finalizer + Location: 0x0000000000132150, Base: 0x00000000002bb4da, Semaphore: 0x0000000000346cb2 + Arguments: +``` + +## Adding probes in libjulia + +Probes are declared in dtraces format in the file `src/uprobes.d`. The generated +header file is included in `src/julia_internal.h` and if you add probes you should +provide a noop implementation there. + +The header will contain a semaphore `*_ENABLED` and the actual call to the probe. +If the probe arguments are expensive to compute you should first check if the +probe is enabled and then compute the arguments and call the probe. + +```c + if (JL_PROBE_{PROBE}_ENABLED()) + auto expensive_arg = ...; + JL_PROBE_{PROBE}(expensive_arg); +``` + +If your probe has no arguments it is preferred to not include the semaphore check. +With USDT probes enabled the cost of a semaphore is a memory load, irrespective of +the fact that the probe is enabled or not. + +```c +#define JL_PROBE_GC_BEGIN_ENABLED() __builtin_expect (julia_gc__begin_semaphore, 0) +__extension__ extern unsigned short julia_gc__begin_semaphore __attribute__ ((unused)) __attribute__ ((section (".probes"))); +``` + +Whereas the probe itself is a noop sled that will be patched to a trampoline to +the probe handler. + +## Available probes + +### GC probes + +1. `julia:gc__begin`: GC begins running on one thread and triggers stop-the-world. +2. `julia:gc__stop_the_world`: All threads have reached a safepoint and GC runs. +3. `julia:gc__mark__begin`: Beginning the mark phase +4. `julia:gc__mark_end(scanned_bytes, perm_scanned)`: Mark phase ended +5. `julia:gc__sweep_begin(full)`: Starting sweep +6. `julia:gc__sweep_end()`: Sweep phase finished +7. `julia:gc__end`: GC is finished, other threads continue work +8. `julia:gc__finalizer`: Initial GC thread has finished running finalizers + +#### GC stop-the-world latency + +An example `bpftrace` script is given in `contrib/gc_stop_the_world_latency.bt` +and it creates a histogram of the latency for all threads to reach a safepoint. + +Running this Julia code, with `julia -t 2` + +``` +using Base.Threads + +fib(x) = x <= 1 ? 1 : fib(x-1) + fib(x-2) + +beaver = @spawn begin + while true + fib(30) + # This safepoint is necessary until #41616, since otherwise this + # loop will never yield to GC. + GC.safepoint() + end +end + +allocator = @spawn begin + while true + zeros(1024) + end +end + +wait(allocator) +``` + +and in a second terminal + +``` +> sudo contrib/bpftrace/gc_stop_the_world_latency.bt +Attaching 4 probes... +Tracing Julia GC Stop-The-World Latency... Hit Ctrl-C to end. +^C + + +@usecs[1743412]: +[4, 8) 971 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[8, 16) 837 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | +[16, 32) 129 |@@@@@@ | +[32, 64) 10 | | +[64, 128) 1 | | +``` + +We can see the latency distribution of the stop-the-world phase in the executed Julia process. + +## Notes on using `bpftrace` + +An example probe in the bpftrace format looks like: + +``` +usdt:usr/lib/libjulia-internal.so:julia:gc__begin +{ + @start[pid] = nsecs; +} +``` + +The probe declaration takes the kind `usdt`, then either the +path to the library or the PID, the provider name `julia` +and the probe name `gc__begin`. Note that I am using a +relative path to the `libjulia-internal.so`, but this might +need to be an absolute path on a production system. + +## Useful references: + +- [Julia Evans blog on Linux tracing systems](https://jvns.ca/blog/2017/07/05/linux-tracing-systems) +- [LWN article on USDT and BPF](https://lwn.net/Articles/753601/) +- [GDB support for probes](https://sourceware.org/gdb/onlinedocs/gdb/Static-Probe-Points.html) +- [Brendan Gregg -- Linux Performance](https://www.brendangregg.com/linuxperf.html) diff --git a/src/Makefile b/src/Makefile index 8da3e4e6687d58..c61523a2bacf77 100644 --- a/src/Makefile +++ b/src/Makefile @@ -84,10 +84,18 @@ endif SRCS += $(RUNTIME_SRCS) +ifeq ($(WITH_DTRACE),1) +DTRACE_HEADERS := uprobes.h.gen +ifneq ($(OS),Darwin) +SRCS += uprobes +endif +else +DTRACE_HEADERS := +endif # headers are used for dependency tracking, while public headers will be part of the dist UV_HEADERS := -HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h locks.h atomics.h julia_internal.h options.h timing.h) +HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h locks.h atomics.h julia_internal.h options.h timing.h) $(addprefix $(BUILDDIR)/, $(DTRACE_HEADERS)) PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,julia.h julia_assert.h julia_threads.h julia_fasttls.h locks.h atomics.h julia_gcext.h) ifeq ($(USE_SYSTEM_LIBUV),0) UV_HEADERS += uv.h @@ -163,6 +171,13 @@ $(BUILDDIR): LLVM_CONFIG_ABSOLUTE := $(shell which $(LLVM_CONFIG)) +# Generate the DTrace header file, while also renaming the macros from +# JULIA_ to JL_PROBE to clearly delinate them. +$(BUILDDIR)/%.h.gen : $(SRCDIR)/%.d + @$(call PRINT_DTRACE, $(DTRACE) -h -s $< -o $@) + sed 's/JULIA_/JL_PROBE_/' $@ > $@.tmp + mv $@.tmp $@ + # source file rules $(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(JCPPFLAGS) $(JCFLAGS) $(SHIPFLAGS) $(DISABLE_ASSERTIONS) -c $< -o $@) @@ -172,6 +187,10 @@ $(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(SRCDIR)/llvm-version.h $(HEADERS) $(LLVM_CONF @$(call PRINT_CC, $(CXX) $(LLVM_CXXFLAGS) $(JCPPFLAGS) $(JCXXFLAGS) $(SHIPFLAGS) $(CXX_DISABLE_ASSERTION) -c $< -o $@) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(SRCDIR)/llvm-version.h $(HEADERS) $(LLVM_CONFIG_ABSOLUTE) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(LLVM_CXXFLAGS) $(JCPPFLAGS) $(JCXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) +$(BUILDDIR)/%.o : $(SRCDIR)/%.d + @$(call PRINT_DTRACE, $(DTRACE) -G -s $< -o $@) +$(BUILDDIR)/%.dbg.obj : $(SRCDIR)/%.d + @$(call PRINT_DTRACE, $(DTRACE) -G -s $< -o $@) # public header rules $(eval $(call dir_target,$(build_includedir)/julia)) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 96364d99048160..6906d81b805d42 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -286,8 +286,7 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p JL_GC_PUSH1(&src); JL_LOCK(&codegen_lock); uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); CompilationPolicy policy = (CompilationPolicy) _policy; @@ -415,8 +414,8 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p } data->M = std::move(clone); - if (jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_measure_compile_time, (jl_hrtime() - compiler_start_time)); if (policy == CompilationPolicy::ImagingMode) imaging_mode = 0; JL_UNLOCK(&codegen_lock); // Might GC @@ -916,8 +915,7 @@ void *jl_get_llvmf_defn(jl_method_instance_t *mi, size_t world, char getwrapper, jl_llvm_functions_t decls; JL_LOCK(&codegen_lock); uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); std::tie(m, decls) = jl_emit_code(mi, src, jlrettype, output); @@ -942,8 +940,8 @@ void *jl_get_llvmf_defn(jl_method_instance_t *mi, size_t world, char getwrapper, m.release(); // the return object `llvmf` will be the owning pointer } JL_GC_POP(); - if (jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_measure_compile_time, (jl_hrtime() - compiler_start_time)); JL_UNLOCK(&codegen_lock); // Might GC if (F) return F; diff --git a/src/gc.c b/src/gc.c index 0f761b90ccfd81..f923b826de544d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3004,6 +3004,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) uint64_t t0 = jl_hrtime(); int64_t last_perm_scanned_bytes = perm_scanned_bytes; + JL_PROBE_GC_MARK_BEGIN(); // 1. fix GC bits of objects in the remset. for (int t_i = 0; t_i < jl_n_threads; t_i++) @@ -3030,6 +3031,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) gc_mark_loop(ptls, sp); gc_mark_sp_init(gc_cache, &sp); gc_num.since_sweep += gc_num.allocd; + JL_PROBE_GC_MARK_END(scanned_bytes, perm_scanned_bytes); gc_settime_premark_end(); gc_time_mark_pause(t0, scanned_bytes, perm_scanned_bytes); int64_t actual_allocd = gc_num.since_sweep; @@ -3135,6 +3137,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) perm_scanned_bytes = 0; scanned_bytes = 0; // 5. start sweeping + JL_PROBE_GC_SWEEP_BEGIN(sweep_full); sweep_weak_refs(); sweep_stack_pools(); gc_sweep_foreign_objs(); @@ -3144,6 +3147,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) gc_sweep_pool(sweep_full); if (sweep_full) gc_sweep_perm_alloc(); + JL_PROBE_GC_SWEEP_END(); // sweeping is over // 6. if it is a quick sweep, put back the remembered objects in queued state // so that we don't trigger the barrier again on them. @@ -3201,6 +3205,8 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { + JL_PROBE_GC_BEGIN(collection); + jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; if (jl_gc_disable_counter) { @@ -3230,6 +3236,8 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) // TODO (concurrently queue objects) // no-op for non-threading jl_gc_wait_for_the_world(); + JL_PROBE_GC_STOP_THE_WORLD(); + gc_invoke_callbacks(jl_gc_cb_pre_gc_t, gc_cblist_pre_gc, (collection)); @@ -3247,6 +3255,7 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) // no-op for non-threading jl_safepoint_end_gc(); jl_gc_state_set(ptls, old_state, JL_GC_STATE_WAITING); + JL_PROBE_GC_END(); // Only disable finalizers on current thread // Doing this on all threads is racy (it's impossible to check @@ -3257,6 +3266,8 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) run_finalizers(ct); ptls->in_finalizer = was_in_finalizer; } + JL_PROBE_GC_FINALIZER(); + gc_invoke_callbacks(jl_gc_cb_post_gc_t, gc_cblist_post_gc, (collection)); #ifdef _OS_WINDOWS_ diff --git a/src/gf.c b/src/gf.c index 41381ccc5178e8..5809721ceaadd0 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3164,15 +3164,14 @@ static uint64_t inference_start_time = 0; JL_DLLEXPORT void jl_typeinf_begin(void) { JL_LOCK(&typeinf_lock); - if (jl_measure_compile_time[jl_threadid()]) + if (jl_atomic_load(&jl_measure_compile_time)) inference_start_time = jl_hrtime(); } JL_DLLEXPORT void jl_typeinf_end(void) { - int tid = jl_threadid(); - if (typeinf_lock.count == 1 && jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - inference_start_time); + if (typeinf_lock.count == 1 && jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_measure_compile_time, (jl_hrtime() - inference_start_time)); JL_UNLOCK(&typeinf_lock); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index e86d6109ff4275..e20daae5cb1d82 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -78,16 +78,21 @@ void jl_jit_globals(std::map &globals) extern "C" JL_DLLEXPORT uint64_t jl_cumulative_compile_time_ns_before() { - int tid = jl_threadid(); - jl_measure_compile_time[tid] = 1; - return jl_cumulative_compile_time[tid]; + // Increment the flag to allow reentrant callers to `@time`. + jl_atomic_fetch_add(&jl_measure_compile_time, 1); + return jl_atomic_load(&jl_cumulative_compile_time); } extern "C" JL_DLLEXPORT uint64_t jl_cumulative_compile_time_ns_after() { - int tid = jl_threadid(); - jl_measure_compile_time[tid] = 0; - return jl_cumulative_compile_time[tid]; + // Decrement the flag when done measuring, allowing other callers to continue measuring. + jl_atomic_fetch_add(&jl_measure_compile_time, -1); + return jl_atomic_load(&jl_cumulative_compile_time); +} + +extern "C" JL_DLLEXPORT +uint64_t jl_cumulative_compile_time_ns() { + return jl_atomic_load(&jl_cumulative_compile_time); } // this generates llvm code for the lambda info @@ -233,8 +238,7 @@ int jl_compile_extern_c(void *llvmmod, void *p, void *sysimg, jl_value_t *declrt { JL_LOCK(&codegen_lock); uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); jl_codegen_params_t params; jl_codegen_params_t *pparams = (jl_codegen_params_t*)p; @@ -258,8 +262,8 @@ int jl_compile_extern_c(void *llvmmod, void *p, void *sysimg, jl_value_t *declrt if (success && llvmmod == NULL) jl_add_to_ee(std::unique_ptr(into)); } - if (codegen_lock.count == 1 && jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (codegen_lock.count == 1 && jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_cumulative_compile_time, (jl_hrtime() - compiler_start_time)); JL_UNLOCK(&codegen_lock); return success; } @@ -315,8 +319,7 @@ jl_code_instance_t *jl_generate_fptr(jl_method_instance_t *mi JL_PROPAGATES_ROOT { JL_LOCK(&codegen_lock); // also disables finalizers, to prevent any unexpected recursion uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); // if we don't have any decls already, try to generate it now jl_code_info_t *src = NULL; @@ -354,8 +357,8 @@ jl_code_instance_t *jl_generate_fptr(jl_method_instance_t *mi JL_PROPAGATES_ROOT else { codeinst = NULL; } - if (codegen_lock.count == 1 && jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (codegen_lock.count == 1 && jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_cumulative_compile_time, (jl_hrtime() - compiler_start_time)); JL_UNLOCK(&codegen_lock); JL_GC_POP(); return codeinst; @@ -369,8 +372,7 @@ void jl_generate_fptr_for_unspecialized(jl_code_instance_t *unspec) } JL_LOCK(&codegen_lock); uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); if (unspec->invoke == NULL) { jl_code_info_t *src = NULL; @@ -398,8 +400,8 @@ void jl_generate_fptr_for_unspecialized(jl_code_instance_t *unspec) } JL_GC_POP(); } - if (codegen_lock.count == 1 && jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (codegen_lock.count == 1 && jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_cumulative_compile_time, (jl_hrtime() - compiler_start_time)); JL_UNLOCK(&codegen_lock); // Might GC } @@ -422,8 +424,7 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, // so create an exception here so we can print pretty our lies JL_LOCK(&codegen_lock); // also disables finalizers, to prevent any unexpected recursion uint64_t compiler_start_time = 0; - int tid = jl_threadid(); - if (jl_measure_compile_time[tid]) + if (jl_atomic_load(&jl_measure_compile_time)) compiler_start_time = jl_hrtime(); specfptr = (uintptr_t)codeinst->specptr.fptr; if (specfptr == 0) { @@ -448,8 +449,8 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, } JL_GC_POP(); } - if (jl_measure_compile_time[tid]) - jl_cumulative_compile_time[tid] += (jl_hrtime() - compiler_start_time); + if (jl_atomic_load(&jl_measure_compile_time)) + jl_atomic_fetch_add(&jl_cumulative_compile_time, (jl_hrtime() - compiler_start_time)); JL_UNLOCK(&codegen_lock); } if (specfptr != 0) diff --git a/src/julia_internal.h b/src/julia_internal.h index 68bee5b120b6e8..ebe037e21dd4da 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -154,8 +154,9 @@ static inline uint64_t cycleclock(void) #include "timing.h" -extern uint8_t *jl_measure_compile_time; -extern uint64_t *jl_cumulative_compile_time; +// Global *atomic* integers controlling *process-wide* measurement of compilation time. +extern volatile uint8_t jl_measure_compile_time; +extern volatile uint64_t jl_cumulative_compile_time; #ifdef _COMPILER_MICROSOFT_ # define jl_return_address() ((uintptr_t)_ReturnAddress()) @@ -1448,4 +1449,39 @@ uint16_t __gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; } #endif +#ifdef USE_DTRACE +#include "uprobes.h.gen" + +// uprobes.h.gen on systems with DTrace, is auto-generated to include +// `JL_PROBE_{PROBE}` and `JL_PROBE_{PROBE}_ENABLED()` macros for every probe +// defined in uprobes.d +// +// If the arguments to `JL_PROBE_{PROBE}` are expensive to compute, the call to +// these functions must be guarded by a JL_PROBE_{PROBE}_ENABLED() check, to +// minimize performance impact when probing is off. As an example: +// +// if (JL_PROBE_GC_STOP_THE_WORLD_ENABLED()) +// JL_PROBE_GC_STOP_THE_WORLD(); + +#else +// define a dummy version of the probe functions +#define JL_PROBE_GC_BEGIN(collection) do ; while (0) +#define JL_PROBE_GC_STOP_THE_WORLD() do ; while (0) +#define JL_PROBE_GC_MARK_BEGIN() do ; while (0) +#define JL_PROBE_GC_MARK_END(scanned_bytes, perm_scanned_bytes) do ; while (0) +#define JL_PROBE_GC_SWEEP_BEGIN(full) do ; while (0) +#define JL_PROBE_GC_SWEEP_END() do ; while (0) +#define JL_PROBE_GC_END() do ; while (0) +#define JL_PROBE_GC_FINALIZER() do ; while (0) + +#define JL_PROBE_GC_BEGIN_ENABLED() (0) +#define JL_PROBE_GC_STOP_THE_WORLD_ENABLED() (0) +#define JL_PROBE_GC_MARK_BEGIN_ENABLED() (0) +#define JL_PROBE_GC_MARK_END_ENABLED() (0) +#define JL_PROBE_GC_SWEEP_BEGIN_ENABLED() (0) +#define JL_PROBE_GC_SWEEP_END_ENABLED() (0) +#define JL_PROBE_GC_END_ENABLED() (0) +#define JL_PROBE_GC_FINALIZER_ENABLED() (0) +#endif + #endif diff --git a/src/task.c b/src/task.c index c3ac26fbcf5119..e9bbd4ecb5e867 100644 --- a/src/task.c +++ b/src/task.c @@ -561,7 +561,7 @@ static void JL_NORETURN throw_internal(jl_task_t *ct, jl_value_t *exception JL_M ptls->io_wait = 0; // @time needs its compile timer disabled on error, // and cannot use a try-finally as it would break scope for assignments - jl_measure_compile_time[ptls->tid] = 0; + jl_atomic_fetch_add(&jl_measure_compile_time, -1); JL_GC_PUSH1(&exception); jl_gc_unsafe_enter(ptls); if (exception) { diff --git a/src/threading.c b/src/threading.c index 235bb9f870ba19..65ed0171ff317b 100644 --- a/src/threading.c +++ b/src/threading.c @@ -287,8 +287,8 @@ void jl_pgcstack_getkey(jl_get_pgcstack_func **f, jl_pgcstack_key_t *k) #endif jl_ptls_t *jl_all_tls_states JL_GLOBALLY_ROOTED; -uint8_t *jl_measure_compile_time = NULL; -uint64_t *jl_cumulative_compile_time = NULL; +volatile uint8_t jl_measure_compile_time = 0; +volatile uint64_t jl_cumulative_compile_time = 0; // return calling thread's ID // Also update the suspended_threads list in signals-mach when changing the @@ -467,8 +467,6 @@ void jl_init_threading(void) } if (jl_n_threads <= 0) jl_n_threads = 1; - jl_measure_compile_time = (uint8_t*)calloc(jl_n_threads, sizeof(*jl_measure_compile_time)); - jl_cumulative_compile_time = (uint64_t*)calloc(jl_n_threads, sizeof(*jl_cumulative_compile_time)); #ifndef __clang_analyzer__ jl_all_tls_states = (jl_ptls_t*)calloc(jl_n_threads, sizeof(void*)); #endif diff --git a/src/uprobes.d b/src/uprobes.d new file mode 100644 index 00000000000000..ef4c59344bc7d5 --- /dev/null +++ b/src/uprobes.d @@ -0,0 +1,18 @@ +/* Julia DTrace provider */ + +provider julia { + probe gc__begin(int collection) + probe gc__stop_the_world() + probe gc__mark__begin() + probe gc__mark__end(int64 scanned_bytes, int64 perm_scanned_bytes) + probe gc__sweep__begin(int full) + probe gc__sweep__end() + probe gc__end() + probe gc__finalizer() +} + +#pragma D attributes Evolving/Evolving/Common provider julia provider +#pragma D attributes Evolving/Evolving/Common provider julia module +#pragma D attributes Evolving/Evolving/Common provider julia function +#pragma D attributes Evolving/Evolving/Common provider julia name +#pragma D attributes Evolving/Evolving/Common provider julia argst diff --git a/stdlib/Distributed/src/cluster.jl b/stdlib/Distributed/src/cluster.jl index 52400a129efb4d..ebe4cac0f3bbe4 100644 --- a/stdlib/Distributed/src/cluster.jl +++ b/stdlib/Distributed/src/cluster.jl @@ -95,14 +95,13 @@ end @enum WorkerState W_CREATED W_CONNECTED W_TERMINATING W_TERMINATED mutable struct Worker id::Int - msg_lock::Threads.ReentrantLock # Lock for del_msgs, add_msgs, and gcflag del_msgs::Array{Any,1} add_msgs::Array{Any,1} gcflag::Bool state::WorkerState - c_state::Threads.Condition # wait for state changes, lock for state - ct_time::Float64 # creation time - conn_func::Any # used to setup connections lazily + c_state::Condition # wait for state changes + ct_time::Float64 # creation time + conn_func::Any # used to setup connections lazily r_stream::IO w_stream::IO @@ -134,7 +133,7 @@ mutable struct Worker if haskey(map_pid_wrkr, id) return map_pid_wrkr[id] end - w=new(id, Threads.ReentrantLock(), [], [], false, W_CREATED, Threads.Condition(), time(), conn_func) + w=new(id, [], [], false, W_CREATED, Condition(), time(), conn_func) w.initialized = Event() register_worker(w) w @@ -144,16 +143,12 @@ mutable struct Worker end function set_worker_state(w, state) - lock(w.c_state) do - w.state = state - notify(w.c_state; all=true) - end + w.state = state + notify(w.c_state; all=true) end function check_worker_state(w::Worker) - lock(w.c_state) if w.state === W_CREATED - unlock(w.c_state) if !isclusterlazy() if PGRP.topology === :all_to_all # Since higher pids connect with lower pids, the remote worker @@ -173,8 +168,6 @@ function check_worker_state(w::Worker) errormonitor(t) wait_for_conn(w) end - else - unlock(w.c_state) end end @@ -193,25 +186,13 @@ function exec_conn_func(w::Worker) end function wait_for_conn(w) - lock(w.c_state) if w.state === W_CREATED - unlock(w.c_state) timeout = worker_timeout() - (time() - w.ct_time) timeout <= 0 && error("peer $(w.id) has not connected to $(myid())") - T = Threads.@spawn begin - sleep($timeout) - lock(w.c_state) do - notify(w.c_state; all=true) - end - end - errormonitor(T) - lock(w.c_state) do - wait(w.c_state) - w.state === W_CREATED && error("peer $(w.id) didn't connect to $(myid()) within $timeout seconds") - end - else - unlock(w.c_state) + @async (sleep(timeout); notify(w.c_state; all=true)) + wait(w.c_state) + w.state === W_CREATED && error("peer $(w.id) didn't connect to $(myid()) within $timeout seconds") end nothing end @@ -490,10 +471,6 @@ function addprocs_locked(manager::ClusterManager; kwargs...) # The `launch` method should add an object of type WorkerConfig for every # worker launched. It provides information required on how to connect # to it. - - # FIXME: launched should be a Channel, launch_ntfy should be a Threads.Condition - # but both are part of the public interface. This means we currently can't use - # `Threads.@spawn` in the code below. launched = WorkerConfig[] launch_ntfy = Condition() @@ -506,10 +483,7 @@ function addprocs_locked(manager::ClusterManager; kwargs...) while true if isempty(launched) istaskdone(t_launch) && break - @async begin - sleep(1) - notify(launch_ntfy) - end + @async (sleep(1); notify(launch_ntfy)) wait(launch_ntfy) end @@ -662,12 +636,7 @@ function create_worker(manager, wconfig) # require the value of config.connect_at which is set only upon connection completion for jw in PGRP.workers if (jw.id != 1) && (jw.id < w.id) - # wait for wl to join - lock(jw.c_state) do - if jw.state === W_CREATED - wait(jw.c_state) - end - end + (jw.state === W_CREATED) && wait(jw.c_state) push!(join_list, jw) end end @@ -690,12 +659,7 @@ function create_worker(manager, wconfig) end for wl in wlist - lock(wl.c_state) do - if wl.state === W_CREATED - # wait for wl to join - wait(wl.c_state) - end - end + (wl.state === W_CREATED) && wait(wl.c_state) push!(join_list, wl) end end @@ -712,11 +676,7 @@ function create_worker(manager, wconfig) @async manage(w.manager, w.id, w.config, :register) # wait for rr_ntfy_join with timeout timedout = false - @async begin - sleep($timeout) - timedout = true - put!(rr_ntfy_join, 1) - end + @async (sleep($timeout); timedout = true; put!(rr_ntfy_join, 1)) wait(rr_ntfy_join) if timedout error("worker did not connect within $timeout seconds") diff --git a/stdlib/Distributed/src/macros.jl b/stdlib/Distributed/src/macros.jl index 24a24f4c08ed4f..6603d627c34092 100644 --- a/stdlib/Distributed/src/macros.jl +++ b/stdlib/Distributed/src/macros.jl @@ -1,10 +1,14 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -let nextidx = Threads.Atomic{Int}(0) +let nextidx = 0 global nextproc function nextproc() - idx = Threads.atomic_add!(nextidx, 1) - return workers()[(idx % nworkers()) + 1] + p = -1 + if p == -1 + p = workers()[(nextidx % nworkers()) + 1] + nextidx += 1 + end + p end end diff --git a/stdlib/Distributed/src/managers.jl b/stdlib/Distributed/src/managers.jl index 5b4f016c63a787..08686fc2a0b87d 100644 --- a/stdlib/Distributed/src/managers.jl +++ b/stdlib/Distributed/src/managers.jl @@ -163,7 +163,7 @@ function launch(manager::SSHManager, params::Dict, launched::Array, launch_ntfy: # Wait for all launches to complete. @sync for (i, (machine, cnt)) in enumerate(manager.machines) let machine=machine, cnt=cnt - @async try + @async try launch_on_machine(manager, $machine, $cnt, params, launched, launch_ntfy) catch e print(stderr, "exception launching on machine $(machine) : $(e)\n") diff --git a/stdlib/Distributed/src/messages.jl b/stdlib/Distributed/src/messages.jl index fcba709b4db4af..47f70e044a2c0e 100644 --- a/stdlib/Distributed/src/messages.jl +++ b/stdlib/Distributed/src/messages.jl @@ -126,20 +126,22 @@ function flush_gc_msgs(w::Worker) if !isdefined(w, :w_stream) return end - lock(w.msg_lock) do - w.gcflag || return # early exit if someone else got to this - w.gcflag = false - msgs = w.add_msgs - w.add_msgs = Any[] - if !isempty(msgs) - remote_do(add_clients, w, msgs) - end + w.gcflag = false + new_array = Any[] + msgs = w.add_msgs + w.add_msgs = new_array + if !isempty(msgs) + remote_do(add_clients, w, msgs) + end - msgs = w.del_msgs - w.del_msgs = Any[] - if !isempty(msgs) - remote_do(del_clients, w, msgs) - end + # del_msgs gets populated by finalizers, so be very careful here about ordering of allocations + # XXX: threading requires this to be atomic + new_array = Any[] + msgs = w.del_msgs + w.del_msgs = new_array + if !isempty(msgs) + #print("sending delete of $msgs\n") + remote_do(del_clients, w, msgs) end end diff --git a/stdlib/Distributed/src/process_messages.jl b/stdlib/Distributed/src/process_messages.jl index 8d5dac5af571e9..732b972858dc97 100644 --- a/stdlib/Distributed/src/process_messages.jl +++ b/stdlib/Distributed/src/process_messages.jl @@ -230,8 +230,8 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) deregister_worker(wpid) end - isopen(r_stream) && close(r_stream) - isopen(w_stream) && close(w_stream) + close(r_stream) + close(w_stream) if (myid() == 1) && (wpid > 1) if oldstate != W_TERMINATING diff --git a/stdlib/Distributed/src/remotecall.jl b/stdlib/Distributed/src/remotecall.jl index 5ac397656ce449..088b7416f44886 100644 --- a/stdlib/Distributed/src/remotecall.jl +++ b/stdlib/Distributed/src/remotecall.jl @@ -247,42 +247,22 @@ function del_clients(pairs::Vector) end end -# The task below is coalescing the `flush_gc_msgs` call -# across multiple producers, see `send_del_client`, -# and `send_add_client`. -# XXX: Is this worth the additional complexity? -# `flush_gc_msgs` has to iterate over all connected workers. -const any_gc_flag = Threads.Condition() +const any_gc_flag = Condition() function start_gc_msgs_task() - errormonitor( - Threads.@spawn begin - while true - lock(any_gc_flag) do - wait(any_gc_flag) - flush_gc_msgs() # handles throws internally - end - end - end - ) + errormonitor(@async while true + wait(any_gc_flag) + flush_gc_msgs() + end) end -# Function can be called within a finalizer function send_del_client(rr) if rr.where == myid() del_client(rr) elseif id_in_procs(rr.where) # process only if a valid worker w = worker_from_id(rr.where)::Worker - msg = (remoteref_id(rr), myid()) - # We cannot acquire locks from finalizers - Threads.@spawn begin - lock(w.msg_lock) do - push!(w.del_msgs, msg) - w.gcflag = true - end - lock(any_gc_flag) do - notify(any_gc_flag) - end - end + push!(w.del_msgs, (remoteref_id(rr), myid())) + w.gcflag = true + notify(any_gc_flag) end end @@ -308,13 +288,9 @@ function send_add_client(rr::AbstractRemoteRef, i) # to the processor that owns the remote ref. it will add_client # itself inside deserialize(). w = worker_from_id(rr.where) - lock(w.msg_lock) do - push!(w.add_msgs, (remoteref_id(rr), i)) - w.gcflag = true - end - lock(any_gc_flag) do - notify(any_gc_flag) - end + push!(w.add_msgs, (remoteref_id(rr), i)) + w.gcflag = true + notify(any_gc_flag) end end diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index 3b99afac8cc15c..749c18f6b61f05 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1696,5 +1696,4 @@ include("splitrange.jl") # Run topology tests last after removing all workers, since a given # cluster at any time only supports a single topology. rmprocs(workers()) -include("threads.jl") include("topology.jl") diff --git a/stdlib/Distributed/test/threads.jl b/stdlib/Distributed/test/threads.jl deleted file mode 100644 index 57d99b7ea056c3..00000000000000 --- a/stdlib/Distributed/test/threads.jl +++ /dev/null @@ -1,63 +0,0 @@ -using Test -using Distributed, Base.Threads -using Base.Iterators: product - -exeflags = ("--startup-file=no", - "--check-bounds=yes", - "--depwarn=error", - "--threads=2") - -function call_on(f, wid, tid) - remotecall(wid) do - t = Task(f) - ccall(:jl_set_task_tid, Cvoid, (Any, Cint), t, tid - 1) - schedule(t) - @assert threadid(t) == tid - t - end -end - -# Run function on process holding the data to only serialize the result of f. -# This becomes useful for things that cannot be serialized (e.g. running tasks) -# or that would be unnecessarily big if serialized. -fetch_from_owner(f, rr) = remotecall_fetch(f ∘ fetch, rr.where, rr) - -isdone(rr) = fetch_from_owner(istaskdone, rr) -isfailed(rr) = fetch_from_owner(istaskfailed, rr) - -@testset "RemoteChannel allows put!/take! from thread other than 1" begin - ws = ts = product(1:2, 1:2) - @testset "from worker $w1 to $w2 via 1" for (w1, w2) in ws - @testset "from thread $w1.$t1 to $w2.$t2" for (t1, t2) in ts - # We want (the default) lazyness, so that we wait for `Worker.c_state`! - procs_added = addprocs(2; exeflags, lazy=true) - @everywhere procs_added using Base.Threads - - p1 = procs_added[w1] - p2 = procs_added[w2] - chan_id = first(procs_added) - chan = RemoteChannel(chan_id) - send = call_on(p1, t1) do - put!(chan, nothing) - end - recv = call_on(p2, t2) do - take!(chan) - end - - # Wait on the spawned tasks on the owner - @sync begin - Threads.@spawn fetch_from_owner(wait, recv) - Threads.@spawn fetch_from_owner(wait, send) - end - - # Check the tasks - @test isdone(send) - @test isdone(recv) - - @test !isfailed(send) - @test !isfailed(recv) - - rmprocs(procs_added) - end - end -end diff --git a/stdlib/LibUV_jll/Project.toml b/stdlib/LibUV_jll/Project.toml index 241a2a16edb615..6a5e2c8ca891be 100644 --- a/stdlib/LibUV_jll/Project.toml +++ b/stdlib/LibUV_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibUV_jll" uuid = "183b4373-6708-53ba-ad28-60e28bb38547" -version = "2.0.1+2" +version = "2.0.1+3" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/Logging/docs/src/index.md b/stdlib/Logging/docs/src/index.md index f5367d05b14a52..7a6fbbbdd20810 100644 --- a/stdlib/Logging/docs/src/index.md +++ b/stdlib/Logging/docs/src/index.md @@ -217,6 +217,9 @@ julia> foo() ``` +Use a comma separator to enable debug for multiple +modules: `JULIA_DEBUG=loading,Main`. + ## Examples ### Example: Writing log events to a file diff --git a/stdlib/Sockets/src/Sockets.jl b/stdlib/Sockets/src/Sockets.jl index 6952aa9bd8a0fd..fb46b9255e6f03 100644 --- a/stdlib/Sockets/src/Sockets.jl +++ b/stdlib/Sockets/src/Sockets.jl @@ -139,9 +139,6 @@ function TCPServer(; delay=true) return tcp end -isreadable(io::TCPSocket) = isopen(io) || bytesavailable(io) > 0 -iswritable(io::TCPSocket) = isopen(io) && io.status != StatusClosing - """ accept(server[, client]) @@ -578,11 +575,11 @@ Enables or disables Nagle's algorithm on a given TCP server or socket. """ function nagle(sock::Union{TCPServer, TCPSocket}, enable::Bool) # disable or enable Nagle's algorithm on all OSes - Sockets.iolock_begin() - Sockets.check_open(sock) + iolock_begin() + check_open(sock) err = ccall(:uv_tcp_nodelay, Cint, (Ptr{Cvoid}, Cint), sock.handle, Cint(!enable)) # TODO: check err - Sockets.iolock_end() + iolock_end() return err end @@ -592,15 +589,15 @@ end On Linux systems, the TCP_QUICKACK is disabled or enabled on `socket`. """ function quickack(sock::Union{TCPServer, TCPSocket}, enable::Bool) - Sockets.iolock_begin() - Sockets.check_open(sock) + iolock_begin() + check_open(sock) @static if Sys.islinux() # tcp_quickack is a linux only option if ccall(:jl_tcp_quickack, Cint, (Ptr{Cvoid}, Cint), sock.handle, Cint(enable)) < 0 @warn "Networking unoptimized ( Error enabling TCP_QUICKACK : $(Libc.strerror(Libc.errno())) )" maxlog=1 end end - Sockets.iolock_end() + iolock_end() nothing end diff --git a/stdlib/Sockets/test/runtests.jl b/stdlib/Sockets/test/runtests.jl index b00eeeee2d068c..552ca3eac528ee 100644 --- a/stdlib/Sockets/test/runtests.jl +++ b/stdlib/Sockets/test/runtests.jl @@ -526,17 +526,42 @@ end r = @async close(s) @test_throws Base._UVError("connect", Base.UV_ECANCELED) Sockets.wait_connected(s) fetch(r) + close(srv) end end @testset "iswritable" begin let addr = Sockets.InetAddr(ip"127.0.0.1", 4445) srv = listen(addr) - s = Sockets.TCPSocket() - Sockets.connect!(s, addr) - @test iswritable(s) - close(s) - @test !iswritable(s) + let s = Sockets.TCPSocket() + Sockets.connect!(s, addr) + @test iswritable(s) broken=Sys.iswindows() + close(s) + @test !iswritable(s) + end + let s = Sockets.connect(addr) + @test iswritable(s) + shutdown(s) + @test !iswritable(s) + close(s) + end + close(srv) + srv = listen(addr) + let s = Sockets.connect(addr) + let c = accept(srv) + Base.errormonitor(@async try; write(c, c); finally; close(c); end) + end + @test iswritable(s) + write(s, "hello world\n") + shutdown(s) + @test !iswritable(s) + @test isreadable(s) + @test read(s, String) == "hello world\n" + @test !isreadable(s) + @test !isopen(s) + close(s) + end + close(srv) end end diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 80972a7c654484..5514b6dc03f7fa 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -9,7 +9,7 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) @testset "Read/write empty IOBuffer" begin io = IOBuffer() @test eof(io) - @test_throws EOFError read(io,UInt8) + @test_throws EOFError read(io, UInt8) @test write(io,"abc") === 3 @test isreadable(io) @test iswritable(io) @@ -18,7 +18,7 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) @test position(io) == 3 @test eof(io) seek(io, 0) - @test read(io,UInt8) == convert(UInt8, 'a') + @test read(io, UInt8) == convert(UInt8, 'a') a = Vector{UInt8}(undef, 2) @test read!(io, a) == a @test a == UInt8['b','c'] @@ -34,22 +34,24 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) truncate(io, 10) @test position(io) == 0 @test all(io.data .== 0) - @test write(io,Int16[1,2,3,4,5,6]) === 12 + @test write(io, Int16[1, 2, 3, 4, 5, 6]) === 12 seek(io, 2) truncate(io, 10) @test ioslength(io) == 10 io.readable = false - @test_throws ArgumentError read!(io,UInt8[0]) + @test_throws ArgumentError read!(io, UInt8[0]) truncate(io, 0) @test write(io,"boston\ncambridge\n") > 0 @test String(take!(io)) == "boston\ncambridge\n" @test String(take!(io)) == "" @test write(io, ComplexF64(0)) === 16 @test write(io, Rational{Int64}(1//2)) === 16 - close(io) - @test_throws ArgumentError write(io,UInt8[0]) - @test_throws ArgumentError seek(io,0) + @test shutdown(io) === nothing + @test_throws ArgumentError write(io, UInt8[0]) @test eof(io) + @test close(io) === nothing + @test_throws ArgumentError write(io, UInt8[0]) + @test_throws ArgumentError seek(io, 0) end @testset "Read/write readonly IOBuffer" begin @@ -237,7 +239,7 @@ end @test isreadable(bstream) @test iswritable(bstream) @test bytesavailable(bstream) == 0 - @test sprint(show, bstream) == "BufferStream() bytes waiting:$(bytesavailable(bstream.buffer)), isopen:true" + @test sprint(show, bstream) == "BufferStream(bytes waiting=$(bytesavailable(bstream.buffer)), isopen=true)" a = rand(UInt8,10) write(bstream,a) @test !eof(bstream) @@ -251,9 +253,10 @@ end @test !eof(bstream) read!(bstream,c) @test c == a[3:10] - @test close(bstream) === nothing + @test shutdown(bstream) === nothing @test eof(bstream) @test bytesavailable(bstream) == 0 + @test close(bstream) === nothing flag = Ref{Bool}(false) event = Base.Event() bstream = Base.BufferStream() diff --git a/test/misc.jl b/test/misc.jl index e765dc9279b86d..d15b5f64426d99 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -263,6 +263,23 @@ function timev_macro_scope() end @test timev_macro_scope() == 1 +before = Base.cumulative_compile_time_ns_before(); + +# exercise concurrent calls to `@time` for reentrant compilation time measurement. +t1 = @async @time begin + sleep(2) + @eval module M ; f(x,y) = x+y ; end + @eval M.f(2,3) +end +t2 = @async begin + sleep(1) + @time 2 + 2 +end + +after = Base.cumulative_compile_time_ns_after(); +@test after >= before; +@test Base.cumulative_compile_time_ns() >= after; + # interactive utilities struct ambigconvert; end # inject a problematic `convert` method to ensure it still works diff --git a/test/ranges.jl b/test/ranges.jl index d4ff83b81fe2de..1c31585ece45e0 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -544,6 +544,19 @@ end end end +# A number type with the overflow behavior of `UInt8`. Conversion to `Integer` returns an +# `Int32`, i.e., a type with different `typemin`/`typemax`. See #41479 +struct OverflowingReal <: Real + val::UInt8 +end +OverflowingReal(x::OverflowingReal) = x +Base.:<=(x::OverflowingReal, y::OverflowingReal) = x.val <= y.val +Base.:+(x::OverflowingReal, y::OverflowingReal) = OverflowingReal(x.val + y.val) +Base.:-(x::OverflowingReal, y::OverflowingReal) = OverflowingReal(x.val - y.val) +Base.round(x::OverflowingReal, ::RoundingMode) = x +Base.Integer(x::OverflowingReal) = Int32(x.val) +@test length(OverflowingReal(1):OverflowingReal(0)) == 0 + @testset "loops involving typemin/typemax" begin n = 0 s = 0 @@ -1095,7 +1108,7 @@ end @testset "issue 10950" begin r = 1//2:3 @test length(r) == 3 - @test_throws MethodError checked_length(r) == 3 # this would work if checked_sub is defined on Rational + @test checked_length(r) == 3 i = 1 for x in r @test x == i//2 diff --git a/test/spawn.jl b/test/spawn.jl index 95915a5ad804b2..9ec7c6842bedbf 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -765,7 +765,15 @@ let text = "input-test-text" @test read(proc, String) == string(length(text), '\n') @test success(proc) @test String(take!(b)) == text + + out = Base.BufferStream() + proc = run(catcmd, IOBuffer(text), out, wait=false) + @test proc.out === out + @test read(out, String) == text + @test success(proc) end + + @test repr(Base.CmdRedirect(``, devnull, 0, false)) == "pipeline(``, stdin>Base.DevNull())" @test repr(Base.CmdRedirect(``, devnull, 1, true)) == "pipeline(``, stdout