From 148493d3da245e06f9e426e275cfe00b53ec45aa Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 26 Mar 2025 21:18:33 -0400 Subject: [PATCH 001/662] Undo explicit extra bindings hack in Logging (#57891) We now have `public` to indicate public symbols, which addresses the original reason this hack was introduced. Semantically reverts #32473. --- stdlib/Logging/src/Logging.jl | 99 ++++++++++------------------------- 1 file changed, 28 insertions(+), 71 deletions(-) diff --git a/stdlib/Logging/src/Logging.jl b/stdlib/Logging/src/Logging.jl index 192885f2f94b7..ab02073f7b40a 100644 --- a/stdlib/Logging/src/Logging.jl +++ b/stdlib/Logging/src/Logging.jl @@ -8,70 +8,33 @@ and available by default. """ module Logging -# Import the CoreLogging implementation into Logging as new const bindings. -# Doing it this way (rather than with import) makes these symbols accessible to -# tab completion. -for sym in [ - :LogLevel, - :AbstractLogger, - :NullLogger, - :handle_message, :shouldlog, :min_enabled_level, :catch_exceptions, - Symbol("@debug"), - Symbol("@info"), - Symbol("@warn"), - Symbol("@error"), - Symbol("@logmsg"), - :with_logger, - :current_logger, - :global_logger, - :disable_logging, - :SimpleLogger] - @eval const $sym = Base.CoreLogging.$sym -end - -# LogLevel aliases (re-)documented here (JuliaLang/julia#40978) -""" - Debug - -Alias for [`LogLevel(-1000)`](@ref LogLevel). -""" -const Debug = Base.CoreLogging.Debug -""" - Info - -Alias for [`LogLevel(0)`](@ref LogLevel). -""" -const Info = Base.CoreLogging.Info -""" - Warn - -Alias for [`LogLevel(1000)`](@ref LogLevel). -""" -const Warn = Base.CoreLogging.Warn -""" - Error - -Alias for [`LogLevel(2000)`](@ref LogLevel). -""" -const Error = Base.CoreLogging.Error -""" - BelowMinLevel - -Alias for [`LogLevel(-1_000_001)`](@ref LogLevel). -""" -const BelowMinLevel = Base.CoreLogging.BelowMinLevel -""" - AboveMaxLevel - -Alias for [`LogLevel(1_000_001)`](@ref LogLevel). -""" -const AboveMaxLevel = Base.CoreLogging.AboveMaxLevel - -using Base.CoreLogging: - closed_stream, ConsoleLogger, default_metafmt - -# Some packages use `Logging.default_logcolor` -const default_logcolor = Base.CoreLogging.default_logcolor +import Base.CoreLogging: + LogLevel, + AbstractLogger, + NullLogger, + handle_message, shouldlog, min_enabled_level, catch_exceptions, + var"@debug", + var"@info", + var"@warn", + var"@error", + var"@logmsg", + with_logger, + current_logger, + global_logger, + disable_logging, + SimpleLogger, + Debug, + Info, + Warn, + Error, + BelowMinLevel, + AboveMaxLevel, + default_logcolor, + closed_stream, + ConsoleLogger, + default_metafmt, + # Some packages use `Logging.default_logcolor` + default_logcolor export AbstractLogger, @@ -95,12 +58,6 @@ export Error, AboveMaxLevel -# The following are also part of the public API, but not exported: -# -# 1. Log levels: -# BelowMinLevel, Debug, Info, Warn, Error, AboveMaxLevel, -# -# 2. AbstractLogger message related functions: -# handle_message, shouldlog, min_enabled_level, catch_exceptions, +public handle_message, shouldlog, min_enabled_level, catch_exceptions end From 626d541de6129179ff3934d6a4f95ef05c4c232c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:57:51 +0900 Subject: [PATCH 002/662] inference: fix exct modeling of `setglobal!` (#57896) --- Compiler/src/abstractinterpretation.jl | 7 +++---- Compiler/test/inference.jl | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index f2b11996f672a..34bf08731b239 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2565,7 +2565,6 @@ function abstract_eval_setglobalonce!(interp::AbstractInterpreter, sv::AbsIntSta end end - function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) if length(argtypes) in (5, 6, 7) (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] @@ -3671,7 +3670,7 @@ end function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, g::GlobalRef, @nospecialize(newty)) if saw_latestworld - return Pair{Any,Any}(newty, Union{ErrorException, TypeError}) + return Pair{Any,Any}(newty, ErrorException) end (valid_worlds, ret) = scan_partitions((interp, _, partition)->global_assignment_binding_rt_exct(interp, partition, newty), interp, g, sv.world) update_valid_age!(sv, valid_worlds) @@ -3688,10 +3687,10 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio ty = kind == PARTITION_KIND_DECLARED ? Any : partition_restriction(partition) wnewty = widenconst(newty) if !hasintersect(wnewty, ty) - return Pair{Any,Any}(Bottom, TypeError) + return Pair{Any,Any}(Bottom, ErrorException) elseif !(wnewty <: ty) retty = tmeet(typeinf_lattice(interp), newty, ty) - return Pair{Any,Any}(retty, TypeError) + return Pair{Any,Any}(retty, ErrorException) end return Pair{Any,Any}(newty, Bottom) end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index c6649afdb1a51..c94059f59f6e7 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6408,4 +6408,9 @@ let src = code_typed1((Base.RefValue{String}, String)) do x, val @test isa(retval, Core.SSAValue) end +global invalid_setglobal!_exct_modeling::Int +@test Base.infer_exception_type((Float64,)) do x + setglobal!(@__MODULE__, :invalid_setglobal!_exct_modeling, x) +end == ErrorException + end # module inference From ed3fccc0be0f3cf48ba8f0910e5a5faa58280787 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:12:20 +0100 Subject: [PATCH 003/662] `Compiler`: `abstract_eval_invoke_inst`: type assert `Expr` (#57860) Should make the code less vulnerable to invalidation. --- Compiler/src/ssair/irinterp.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compiler/src/ssair/irinterp.jl b/Compiler/src/ssair/irinterp.jl index 084f28f0aa523..3d72da72625be 100644 --- a/Compiler/src/ssair/irinterp.jl +++ b/Compiler/src/ssair/irinterp.jl @@ -32,7 +32,7 @@ function concrete_eval_invoke(interp::AbstractInterpreter, ci::CodeInstance, arg end function abstract_eval_invoke_inst(interp::AbstractInterpreter, inst::Instruction, irsv::IRInterpretationState) - stmt = inst[:stmt] + stmt = inst[:stmt]::Expr ci = stmt.args[1] if ci isa MethodInstance world = frame_world(irsv) From 0d4d6d927d531c82e454e5a08c4bf6802cf69001 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Thu, 27 Mar 2025 11:10:05 -0400 Subject: [PATCH 004/662] Scheduler: Use a "scheduler" task for thread sleep (#57544) A Julia thread runs Julia's scheduler in the context of the switching task. If no task is found to switch to, the thread will sleep while holding onto the (possibly completed) task, preventing the task from being garbage collected. This recent [Discourse post](https://discourse.julialang.org/t/weird-behaviour-of-gc-with-multithreaded-array-access/125433) illustrates precisely this problem. A solution to this would be for an idle Julia thread to switch to a "scheduler" task, thereby freeing the old task. This PR uses `OncePerThread` to create a "scheduler" task (that does nothing but run `wait()` in a loop) and switches to that task when the thread finds itself idle. Other approaches considered and discarded in favor of this one: https://github.com/JuliaLang/julia/pull/57465 and https://github.com/JuliaLang/julia/pull/57543. --- base/task.jl | 50 ++++++++++++++++++++++++--------- stdlib/Sockets/test/runtests.jl | 4 +-- test/channels.jl | 5 ++-- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/base/task.jl b/base/task.jl index c6aec03191144..15ac907c18988 100644 --- a/base/task.jl +++ b/base/task.jl @@ -1145,6 +1145,16 @@ function throwto(t::Task, @nospecialize exc) return try_yieldto(identity) end +@inline function wait_forever() + while true + wait() + end +end + +const get_sched_task = OncePerThread{Task}() do + @task wait_forever() +end + function ensure_rescheduled(othertask::Task) ct = current_task() W = workqueue_for(Threads.threadid()) @@ -1181,25 +1191,39 @@ end checktaskempty = Partr.multiq_check_empty -@noinline function poptask(W::StickyWorkqueue) - task = trypoptask(W) - if !(task isa Task) - task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), trypoptask, W, checktaskempty) - end - set_next_task(task) - nothing -end - function wait() ct = current_task() # [task] user_time -yield-or-done-> wait_time record_running_time!(ct) + # let GC run GC.safepoint() - W = workqueue_for(Threads.threadid()) - poptask(W) - result = try_yieldto(ensure_rescheduled) + # check for libuv events process_events() - # return when we come out of the queue + + # get the next task to run + result = nothing + have_result = false + W = workqueue_for(Threads.threadid()) + task = trypoptask(W) + if !(task isa Task) + # No tasks to run; switch to the scheduler task to run the + # thread sleep logic. + sched_task = get_sched_task() + if ct !== sched_task + result = yieldto(sched_task) + have_result = true + else + task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), + trypoptask, W, checktaskempty) + end + end + # We may have already switched tasks (via the scheduler task), so + # only switch if we haven't. + if !have_result + @assert task isa Task + set_next_task(task) + result = try_yieldto(ensure_rescheduled) + end return result end diff --git a/stdlib/Sockets/test/runtests.jl b/stdlib/Sockets/test/runtests.jl index 26f95d4ce1819..5a822315ba2cd 100644 --- a/stdlib/Sockets/test/runtests.jl +++ b/stdlib/Sockets/test/runtests.jl @@ -547,14 +547,12 @@ end fetch(r) end - let addr = Sockets.InetAddr(ip"127.0.0.1", 4444) - srv = listen(addr) + let addr = Sockets.InetAddr(ip"192.0.2.5", 4444) s = Sockets.TCPSocket() Sockets.connect!(s, addr) r = @async close(s) @test_throws Base._UVError("connect", Base.UV_ECANCELED) Sockets.wait_connected(s) fetch(r) - close(srv) end end diff --git a/test/channels.jl b/test/channels.jl index f646b41cfa1a0..721eb478bd13a 100644 --- a/test/channels.jl +++ b/test/channels.jl @@ -463,15 +463,14 @@ end cb = first(async.cond.waitq) @test isopen(async) ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) - ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) - @test isempty(Base.Workqueue) Base.process_events() # schedule event Sys.iswindows() && Base.process_events() # schedule event (windows?) - @test length(Base.Workqueue) == 1 ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) @test tc[] == 0 yield() # consume event @test tc[] == 1 + ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) + Base.process_events() Sys.iswindows() && Base.process_events() # schedule event (windows?) yield() # consume event @test tc[] == 2 From 608a06c40deba119a6c03612a512eac54ca971b8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 12 Mar 2025 16:35:25 +0000 Subject: [PATCH 005/662] inference: add internal SOURCE_MODE_GET_SOURCE mode We were previously using typeinf_code for this, but that has slightly different semantics for recursion detection and handling of some edge cases related to caching (e.g. seen in #57634). Overall, we would like to avoid those sorts of spurious differences in compilation. Instead (re)introduce a `source_mode` called `SOURCE_MODE_GET_SOURCE` which can be used internally (requires a NativeInterpreter) here to avoid needing to duplicate the existing `typeinf_ext` code to implement this use case. While we are at it, also remove the code duplication for `typeinf_type`. --- Compiler/src/typeinfer.jl | 167 ++++++++++++++++++++---------------- Compiler/src/verifytrim.jl | 17 +++- Compiler/test/verifytrim.jl | 2 +- src/aotcompile.cpp | 14 +++ src/gf.c | 23 +++-- src/jitlayers.cpp | 4 +- src/julia_internal.h | 3 +- test/trimming/trimming.jl | 2 +- 8 files changed, 141 insertions(+), 91 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 9a5acece139a3..5bdd57938e1fb 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -156,7 +156,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation # when compiling the compiler to inject everything eagerly # where codegen can start finding and using it right away mi = result.linfo - if mi.def isa Method && isa_compileable_sig(mi) + if mi.def isa Method && isa_compileable_sig(mi) && is_cached(caller) ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, uncompressed) end end @@ -1113,10 +1113,10 @@ end """ SOURCE_MODE_NOT_REQUIRED -Indicates to inference that the source is not required and the only fields -of the resulting `CodeInstance` that the caller is interested in are types -and effects. Inference is still free to create a CodeInstance with source, -but is not required to do so. +Indicates to inference that the source is not required and the only fields of +the resulting `CodeInstance` that the caller is interested in are return or +exception types and IPO effects. Inference is still free to create source for +it or add it to the JIT even, but is not required or expected to do so. """ const SOURCE_MODE_NOT_REQUIRED = 0x0 @@ -1124,28 +1124,51 @@ const SOURCE_MODE_NOT_REQUIRED = 0x0 SOURCE_MODE_ABI Indicates to inference that it should return a CodeInstance that can -either be `->invoke`'d (because it has already been compiled or because -it has constabi) or one that can be made so by compiling its `->inferred` -field. - -N.B.: The `->inferred` field is volatile and the compiler may delete it. +be `->invoke`'d (because it has already been compiled). """ const SOURCE_MODE_ABI = 0x1 """ - ci_has_abi(code::CodeInstance) + SOURCE_MODE_GET_SOURCE + +Indicates to inference that it should return a CodeInstance after it has +prepared interp to be able to provide source code for it. +""" +const SOURCE_MODE_GET_SOURCE = 0xf -Determine whether this CodeInstance is something that could be invoked if we gave it -to the runtime system (either because it already has an ->invoke ptr, or -because it has source that could be compiled). Note that this information may -be stale by the time the user see it, so the user will need to perform their -own checks if they actually need the abi from it. """ -function ci_has_abi(code::CodeInstance) + ci_has_abi(interp::AbstractInterpreter, code::CodeInstance) + +Determine whether this CodeInstance is something that could be invoked if +interp gave it to the runtime system (either because it already has an ->invoke +ptr, or because interp has source that could be compiled). +""" +function ci_has_abi(interp::AbstractInterpreter, code::CodeInstance) (@atomic :acquire code.invoke) !== C_NULL && return true + return ci_has_source(interp, code) +end + +""" + ci_has_source(interp::AbstractInterpreter, code::CodeInstance) + +Determine whether this CodeInstance is something that could be compiled from +source that interp has. +""" +function ci_has_source(interp::AbstractInterpreter, code::CodeInstance) + codegen = codegen_cache(interp) + codegen === nothing && return false + use_const_api(code) && return true + haskey(codegen, code) && return true inf = @atomic :monotonic code.inferred - if code.owner === nothing ? (isa(inf, CodeInfo) || isa(inf, String)) : inf !== nothing - # interp.codegen[code] = maybe_uncompress(code, inf) # TODO: the correct way to ensure this information doesn't become stale would be to push it into the stable codegen cache + if isa(inf, String) + inf = _uncompressed_ir(code, inf) + end + if code.owner === nothing + if isa(inf, CodeInfo) + codegen[code] = inf + return true + end + elseif inf !== nothing return true end return false @@ -1155,9 +1178,10 @@ function ci_has_invoke(code::CodeInstance) return (@atomic :monotonic code.invoke) !== C_NULL end -function ci_meets_requirement(code::CodeInstance, source_mode::UInt8) +function ci_meets_requirement(interp::AbstractInterpreter, code::CodeInstance, source_mode::UInt8) source_mode == SOURCE_MODE_NOT_REQUIRED && return true - source_mode == SOURCE_MODE_ABI && return ci_has_abi(code) + source_mode == SOURCE_MODE_ABI && return ci_has_abi(interp, code) + source_mode == SOURCE_MODE_GET_SOURCE && return ci_has_source(interp, code) return false end @@ -1167,7 +1191,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod let code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # see if this code already exists in the cache - if ci_meets_requirement(code, source_mode) + if ci_meets_requirement(interp, code, source_mode) ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) return code end @@ -1179,7 +1203,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod let code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # see if this code already exists in the cache - if ci_meets_requirement(code, source_mode) + if ci_meets_requirement(interp, code, source_mode) engine_reject(interp, ci) ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) return code @@ -1210,18 +1234,11 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) ci = result.ci # reload from result in case it changed + codegen = codegen_cache(interp) @assert frame.cache_mode != CACHE_MODE_NULL - @assert is_result_constabi_eligible(result) || begin - codegen = codegen_cache(interp) - codegen === nothing || haskey(codegen, ci) - end + @assert is_result_constabi_eligible(result) || codegen === nothing || haskey(codegen, ci) @assert is_result_constabi_eligible(result) == use_const_api(ci) @assert isdefined(ci, :inferred) "interpreter did not fulfill our expectations" - if !is_cached(frame) && source_mode == SOURCE_MODE_ABI - # XXX: jl_type_infer somewhat ambiguously assumes this must be cached - # XXX: this should be using the CI from the cache, if possible instead: haskey(cache, mi) && (ci = cache[mi]) - code_cache(interp)[mi] = ci - end return ci end @@ -1235,35 +1252,9 @@ end typeinf_type(interp::AbstractInterpreter, match::MethodMatch) = typeinf_type(interp, specialize_method(match)) function typeinf_type(interp::AbstractInterpreter, mi::MethodInstance) - # n.b.: this could be replaced with @something(typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED), return nothing).rettype - start_time = ccall(:jl_typeinf_timing_begin, UInt64, ()) - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - # see if this rettype already exists in the cache - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code.rettype - end - end - ci = engine_reserve(interp, mi) - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - engine_reject(interp, ci) - # see if this rettype already exists in the cache - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code.rettype - end - end - result = InferenceResult(mi, typeinf_lattice(interp)) - result.ci = ci - frame = InferenceState(result, #=cache_mode=#:global, interp) - if frame === nothing - engine_reject(interp, ci) - return nothing - end - typeinf(interp, frame) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - is_inferred(result) || return nothing - return widenconst(ignorelimited(result.result)) + ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED) + ci isa CodeInstance || return nothing + return ci.rettype end # collect a list of all code that is needed along with CodeInstance to codegen it fully @@ -1300,18 +1291,31 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn src = _uncompressed_ir(callee, src) end if !isa(src, CodeInfo) - newcallee = typeinf_ext(interp, callee.def, source_mode) + newcallee = typeinf_ext(interp, callee.def, source_mode) # always SOURCE_MODE_ABI if newcallee isa CodeInstance callee === ci && (ci = newcallee) # ci stopped meeting the requirements after typeinf_ext last checked, try again with newcallee push!(tocompile, newcallee) - #else - # println("warning: could not get source code for ", callee.def) + end + if newcallee !== callee + push!(inspected, callee) end continue end end push!(inspected, callee) collectinvokes!(tocompile, src) + mi = get_ci_mi(callee) + if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(interp))::CodeInstance + if cached === callee + # make sure callee is gc-rooted and cached, as required by jl_add_codeinst_to_jit + code_cache(interp)[mi] = callee + else + # use an existing CI from the cache, if there is available one that is compatible + callee === ci && (ci = cached) + callee = cached + end + end ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), callee, src) end return ci @@ -1355,7 +1359,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m # and this is either the primary world, or not applicable in the primary world # then we want to compile and emit this if item.def.primary_world <= this_world <= item.def.deleted_world - ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED) + ci = typeinf_ext(interp, item, SOURCE_MODE_GET_SOURCE) ci isa CodeInstance && push!(tocompile, ci) end elseif item isa SimpleVector && latest @@ -1366,7 +1370,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m sig, this_world, #= mt_cache =# 0) if ptr !== C_NULL mi = unsafe_pointer_to_objref(ptr)::MethodInstance - ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED) + ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) ci isa CodeInstance && push!(tocompile, ci) end # additionally enqueue the ccallable entrypoint / adapter, which implicitly @@ -1377,26 +1381,37 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m while !isempty(tocompile) callee = pop!(tocompile) callee in inspected && continue - push!(inspected, callee) # now make sure everything has source code, if desired mi = get_ci_mi(callee) def = mi.def if use_const_api(callee) src = codeinfo_for_const(interp, mi, callee.rettype_const) - elseif haskey(interp.codegen, callee) - src = interp.codegen[callee] - elseif isa(def, Method) && !InferenceParams(interp).force_enable_inference && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 - src = retrieve_code_info(mi, get_inference_world(interp)) else - # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here - src = typeinf_code(interp, mi, true) + src = get(interp.codegen, callee, nothing) + if src === nothing + newcallee = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) + if newcallee isa CodeInstance + @assert use_const_api(newcallee) || haskey(interp.codegen, newcallee) + push!(tocompile, newcallee) + end + if newcallee !== callee + push!(inspected, callee) + end + continue + end end + push!(inspected, callee) if src isa CodeInfo collectinvokes!(tocompile, src) - # It is somewhat ambiguous if typeinf_ext might have callee in the caches, - # but for the purpose of native compile, we always want them put there. + # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - code_cache(interp)[mi] = callee + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, this_world)::CodeInstance + if cached === callee + code_cache(interp)[mi] = callee + else + # Use an existing CI from the cache, if there is available one that is compatible + callee = cached + end end push!(codeinfos, callee) push!(codeinfos, src) diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 735fb83d6f6bb..9b49c4b4500c1 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -110,7 +110,7 @@ end function verify_print_error(io::IOContext{IO}, desc::CallMissing, parents::ParentMap) (; codeinst, codeinfo, sptypes, stmtidx, desc) = desc frames = verify_create_stackframes(codeinst, stmtidx, parents) - print(io, desc, " from ") + print(io, desc, " from statement ") verify_print_stmt(io, codeinfo, sptypes, stmtidx) Base.show_backtrace(io, frames) print(io, "\n\n") @@ -181,6 +181,11 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec if edge isa CodeInstance haskey(parents, edge) || (parents[edge] = (codeinst, i)) edge in inspected && continue + edge_mi = get_ci_mi(edge) + if edge_mi === edge.def + ci = get(caches, edge_mi, nothing) + ci isa CodeInstance && continue # assume that only this_world matters for trim + end end # TODO: check for calls to Base.atexit? elseif isexpr(stmt, :call) @@ -287,7 +292,7 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) # TODO: should we find a way to indicate to the user that this gets called via ccallable? # parent[ci] = something asrt = ci.rettype - ci in inspected + true else false end @@ -326,6 +331,14 @@ function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) verify_print_error(io, desc, parents) end + ## TODO: compute and display the minimum and/or full call graph instead of merely the first parent stacktrace? + #for i = 1:length(codeinfos) + # item = codeinfos[i] + # if item isa CodeInstance + # println(item, "::", item.rettype) + # end + #end + let severity = 0 if counts[1] > 0 || counts[2] > 0 print("Trim verify finished with ") diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index 34ca0f89f26fb..2462b50b0559a 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -33,7 +33,7 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Bas @test occursin("finalizer", desc.desc) repr = sprint(verify_print_error, desc, parents) @test occursin( - r"""^unresolved finalizer registered from \(Core.finalizer\)\(Base.final_shred!, %new\(\)::Base.SecretBuffer\)::Nothing + r"""^unresolved finalizer registered from statement \(Core.finalizer\)\(Base.final_shred!, %new\(\)::Base.SecretBuffer\)::Nothing Stacktrace: \[1\] finalizer\(f::typeof\(Base.final_shred!\), o::Base.SecretBuffer\) @ Base gcutils.jl:(\d+) \[inlined\] diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5cdecda9d8582..d687f44808409 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -675,6 +675,20 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[0] = (jl_value_t*)codeinfos; void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, external_linkage); + // examine everything just emitted and save it to the caches + if (!external_linkage) { + for (size_t i = 0, l = jl_array_nrows(codeinfos); i < l; i++) { + jl_value_t *item = jl_array_ptr_ref(codeinfos, i); + if (jl_is_code_instance(item)) { + // now add it to our compilation results + jl_code_instance_t *codeinst = (jl_code_instance_t*)item; + jl_code_info_t *src = (jl_code_info_t*)jl_array_ptr_ref(codeinfos, ++i); + assert(jl_is_code_info(src)); + jl_add_codeinst_to_cache(codeinst, src); + } + } + } + // move everything inside, now that we've merged everything // (before adding the exported headers) ((jl_native_code_desc_t*)data)->M.withModuleDo([&](Module &M) { diff --git a/src/gf.c b/src/gf.c index 6252d78acd4c8..714bf0b285cc6 100644 --- a/src/gf.c +++ b/src/gf.c @@ -585,8 +585,8 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, return 0; } -// look for something with an egal ABI and properties that is already in the JIT (compiled=true) or simply in the cache (compiled=false) -JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, int compiled) JL_NOTSAFEPOINT +// look for something with an egal ABI and properties that is already in the JIT for a whole edge (target_world=0) or can be added to the JIT with new source just for target_world. +JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT { jl_value_t *def = ci->def; jl_method_instance_t *mi = jl_get_ci_mi(ci); @@ -598,9 +598,9 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPA while (codeinst) { if (codeinst != ci && jl_atomic_load_relaxed(&codeinst->inferred) != NULL && - (!compiled || jl_atomic_load_relaxed(&codeinst->invoke) != NULL) && - jl_atomic_load_relaxed(&codeinst->min_world) <= min_world && - jl_atomic_load_relaxed(&codeinst->max_world) >= max_world && + (target_world ? 1 : jl_atomic_load_relaxed(&codeinst->invoke) != NULL) && + jl_atomic_load_relaxed(&codeinst->min_world) <= (target_world ? target_world : min_world) && + jl_atomic_load_relaxed(&codeinst->max_world) >= (target_world ? target_world : max_world) && jl_egal(codeinst->def, def) && jl_egal(codeinst->owner, owner) && jl_egal(codeinst->rettype, rettype)) { @@ -608,7 +608,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPA } codeinst = jl_atomic_load_relaxed(&codeinst->next); } - return (jl_code_instance_t*)jl_nothing; + return ci; } @@ -2795,10 +2795,9 @@ void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_c jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_PROPAGATES_ROOT); -JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src) +JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src) { assert(jl_is_code_info(src)); - jl_emit_codeinst_to_jit(codeinst, src); jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (jl_generating_output() && jl_is_method(mi->def.method) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { jl_value_t *compressed = jl_compress_ir(mi->def.method, src); @@ -2814,6 +2813,14 @@ JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_i } } + +JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src) +{ + assert(jl_is_code_info(src)); + jl_emit_codeinst_to_jit(codeinst, src); + jl_add_codeinst_to_cache(codeinst, src); +} + jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) { // quick check if we already have a compiled result diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 1358c923d138a..bf49b7010b97b 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -386,8 +386,8 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t } if (preal_decl.empty()) { // there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead - jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 1); - if ((jl_value_t*)compiled_ci != jl_nothing) { + jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 0); + if (compiled_ci != codeinst) { codeinst = compiled_ci; uint8_t specsigflags; void *fptr; diff --git a/src/julia_internal.h b/src/julia_internal.h index 3bd2ac4f3ce35..cf9ed6f08f4af 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -685,6 +685,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROP JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile); JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src); +JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( @@ -694,7 +695,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/); -JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, int compiled) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT; STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index a752c69460ad4..4a0bb14ceebe2 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -4,7 +4,7 @@ let exe_suffix = splitext(Base.julia_exename())[2] hello_exe = joinpath(@__DIR__, "hello" * exe_suffix) @test readchomp(`$hello_exe`) == "Hello, world!" - @test filesize(hello_exe) < 2000000 + @test filesize(hello_exe) < 20_000_000 basic_jll_exe = joinpath(@__DIR__, "basic_jll" * exe_suffix) lines = split(readchomp(`$basic_jll_exe`), "\n") From 14bbb4af909d10f5a27b0c71b5477f7b66bbec13 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 12 Mar 2025 21:45:03 +0000 Subject: [PATCH 006/662] [Compiler] use a slightly better (hopefully faster) bootstrap call --- Compiler/src/bootstrap.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Compiler/src/bootstrap.jl b/Compiler/src/bootstrap.jl index 2671ea114e818..a847d1fb835c7 100644 --- a/Compiler/src/bootstrap.jl +++ b/Compiler/src/bootstrap.jl @@ -67,17 +67,10 @@ function bootstrap!() end mi = specialize_method(m.method, Tuple{params...}, m.sparams) #isa_compileable_sig(mi) || println(stderr, "WARNING: inferring `", mi, "` which isn't expected to be called.") - push!(methods, mi) + typeinf_ext_toplevel(mi, world, isa_compileable_sig(mi) ? SOURCE_MODE_ABI : SOURCE_MODE_NOT_REQUIRED) end end end - codeinfos = typeinf_ext_toplevel(methods, [world], TRIM_NO) - for i = 1:2:length(codeinfos) - ci = codeinfos[i]::CodeInstance - src = codeinfos[i + 1]::CodeInfo - isa_compileable_sig(ci.def) || continue # println(stderr, "WARNING: compiling `", ci.def, "` which isn't expected to be called.") - ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, src) - end endtime = time() println("Base.Compiler ──── ", sub_float(endtime,starttime), " seconds") end From d9441ac9bc055562ba9009357b465e69c16ade68 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 27 Mar 2025 19:29:55 +0100 Subject: [PATCH 007/662] `Compiler`: `walk_to_defs`, `collect_leaves`: specialize for `predecessors` (#57859) Should make the code less vulnerable to invalidation. --- Compiler/src/ssair/passes.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Compiler/src/ssair/passes.jl b/Compiler/src/ssair/passes.jl index 4251eecb23631..f2e41d1142e65 100644 --- a/Compiler/src/ssair/passes.jl +++ b/Compiler/src/ssair/passes.jl @@ -183,7 +183,7 @@ function find_def_for_use( end function collect_leaves(compact::IncrementalCompact, @nospecialize(val), @nospecialize(typeconstraint), 𝕃ₒ::AbstractLattice, - predecessors = ((@nospecialize(def), compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) + predecessors::Pre = ((@nospecialize(def), compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) where {Pre} if isa(val, Union{OldSSAValue, SSAValue}) val, typeconstraint = simple_walk_constraint(compact, val, typeconstraint) end @@ -271,7 +271,7 @@ Starting at `val` walk use-def chains to get all the leaves feeding into this `v `predecessors(def, compact)` is a callback which should return the set of possible predecessors for a "phi-like" node (PhiNode or Core.ifelse) or `nothing` otherwise. """ -function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospecialize(typeconstraint), predecessors, 𝕃ₒ::AbstractLattice) +function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospecialize(typeconstraint), predecessors::Pre, 𝕃ₒ::AbstractLattice) where {Pre} visited_philikes = AnySSAValue[] isa(defssa, AnySSAValue) || return Any[defssa], visited_philikes def = compact[defssa][:stmt] From d6fdbf5212aaf270749138db2e285141e4de1dc6 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Fri, 28 Mar 2025 01:56:26 -0300 Subject: [PATCH 008/662] only update fragmentation data for pages that are not lazily freed (#57907) We are including data from lazily freed pages into this metric, which makes it inaccurate. Thanks @qinsoon for spotting it. --- src/gc-stock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gc-stock.c b/src/gc-stock.c index 0bc4ceca52257..01453a30b2a4b 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -948,6 +948,7 @@ static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_ done: if (re_use_page) { + gc_update_page_fragmentation_data(pg); push_lf_back(allocd, pg); } else { @@ -956,7 +957,6 @@ static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_ push_lf_back(&global_page_pool_lazily_freed, pg); } gc_page_profile_write_to_file(s); - gc_update_page_fragmentation_data(pg); gc_time_count_page(freedall, pg_skpd); jl_ptls_t ptls = jl_current_task->ptls; // Note that we aggregate the `pool_live_bytes` over all threads before returning this From 8e03cb11c8460b14ccb1553cb11343c3f39a6774 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Fri, 28 Mar 2025 12:26:13 +0100 Subject: [PATCH 009/662] `Random`: `show` method for `MersenneTwister`: invalidation resistance (#57913) Avoid using or constructing the vector with nonconcrete element type. Should make the sysimage more resistant to method invalidation. --- stdlib/Random/src/RNGs.jl | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 0ae479e129b3b..48d474f4ba41f 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -147,21 +147,26 @@ function show(io::IO, rng::MersenneTwister) end print(io, MersenneTwister, "(", repr(rng.seed), ", (") # state - adv = Integer[rng.adv_jump, rng.adv] + sep = ", " + show(io, rng.adv_jump) + print(io, sep) + show(io, rng.adv) if rng.adv_vals != -1 || rng.adv_ints != -1 - if rng.adv_vals == -1 - @assert rng.idxF == MT_CACHE_F - push!(adv, 0, 0) # "(0, 0)" is nicer on the eyes than (-1, 1002) - else - push!(adv, rng.adv_vals, rng.idxF) - end + # "(0, 0)" is nicer on the eyes than (-1, 1002) + s = rng.adv_vals != -1 + print(io, sep) + show(io, s ? rng.adv_vals : zero(rng.adv_vals)) + print(io, sep) + show(io, s ? rng.idxF : zero(rng.idxF)) end if rng.adv_ints != -1 idxI = (length(rng.ints)*16 - rng.idxI) / 8 # 8 represents one Int64 idxI = Int(idxI) # idxI should always be an integer when using public APIs - push!(adv, rng.adv_ints, idxI) + print(io, sep) + show(io, rng.adv_ints) + print(io, sep) + show(io, idxI) end - join(io, adv, ", ") print(io, "))") end From aab74908dc6b027fc4edd134bfeeaa3af9f141d2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Nov 2024 10:18:33 -0500 Subject: [PATCH 010/662] codegen: add a pass for late conversion of known modify ops to call atomicrmw The ExpandAtomicModify can recognize our pseudo-intrinsic julia.atomicmodify and convert it into some of known atomicrmw expressions, or simplify it with more inlining, as applicable. This ensures that now our `@atomic` modify is as fast as `Threads.Atomic` for the cases we implement now. --- src/Makefile | 8 +- src/aotcompile.cpp | 199 +++++++++---- src/cgutils.cpp | 82 ++++-- src/codegen.cpp | 193 ++++++++++-- src/jitlayers.cpp | 454 +++++++++++----------------- src/jitlayers.h | 17 +- src/julia.expmap.in | 1 - src/llvm-expand-atomic-modify.cpp | 473 ++++++++++++++++++++++++++++++ src/llvm-julia-passes.inc | 1 + src/passes.h | 5 + src/pipeline.cpp | 4 +- test/llvmpasses/atomic-modify.ll | 288 ++++++++++++++++++ 12 files changed, 1346 insertions(+), 379 deletions(-) create mode 100644 src/llvm-expand-atomic-modify.cpp create mode 100644 test/llvmpasses/atomic-modify.ll diff --git a/src/Makefile b/src/Makefile index c605d6c70573b..6a6f604f3c5fc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -79,7 +79,8 @@ endif CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop \ llvm-pass-helpers llvm-ptls llvm-propagate-addrspaces null_sysimage \ llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \ - llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api \ + llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures llvm-expand-atomic-modify \ + pipeline llvm_api \ $(GC_CODEGEN_SRCS) FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) CG_LLVM_LIBS := all @@ -338,7 +339,7 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) # additional dependency links -$(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.h llvm-julia-passes.inc) $(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/processor.h $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/idset.c $(SRCDIR)/builtin_proto.h @@ -378,7 +379,8 @@ $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $ $(BUILDDIR)/staticdata.o $(BUILDDIR)/staticdata.dbg.obj: $(SRCDIR)/staticdata_utils.c $(SRCDIR)/precompile_utils.c $(SRCDIR)/processor.h $(SRCDIR)/builtin_proto.h $(BUILDDIR)/toplevel.o $(BUILDDIR)/toplevel.dbg.obj: $(SRCDIR)/builtin_proto.h $(BUILDDIR)/ircode.o $(BUILDDIR)/ircode.dbg.obj: $(SRCDIR)/serialize.h $(SRCDIR)/common_symbols1.inc $(SRCDIR)/common_symbols2.inc -$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(SRCDIR)/passes.h $(SRCDIR)/jitlayers.h +$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(addprefix $(SRCDIR)/,passes.h jitlayers.h llvm-julia-passes.inc) +$(BUILDDIR)/llvm_api.o $(BUILDDIR)/llvm_api.dbg.obj: $(SRCDIR)/llvm-julia-passes.inc $(addprefix $(BUILDDIR)/,threading.o threading.dbg.obj gc-common.o gc-stock.o gc.dbg.obj init.c init.dbg.obj task.o task.dbg.obj): $(addprefix $(SRCDIR)/,threading.h) $(addprefix $(BUILDDIR)/,APInt-C.o APInt-C.dbg.obj runtime_intrinsics.o runtime_intrinsics.dbg.obj): $(SRCDIR)/APInt-C.h diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index d687f44808409..7c9d8aec1a1c9 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -330,7 +330,11 @@ class egal_set { }; } using ::egal_set; -typedef DenseMap> jl_compiled_functions_t; +struct jl_compiled_function_t { + orc::ThreadSafeModule TSM; + jl_llvm_functions_t decls; +}; +typedef DenseMap jl_compiled_functions_t; static void record_method_roots(egal_set &method_roots, jl_method_instance_t *mi) { @@ -376,7 +380,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo std::string OldName(GV->getName()); StringRef NewName(mref->second->getName()); for (auto &def : compiled_functions) { - orc::ThreadSafeModule &TSM = std::get<0>(def.second); + orc::ThreadSafeModule &TSM = def.second.TSM; Module &M = *TSM.getModuleUnlocked(); if (GlobalValue *GV2 = M.getNamedValue(OldName)) { if (GV2 == GV) @@ -402,7 +406,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_roots, jl_compiled_functions_t &compiled_functions) { - decltype(params.workqueue) workqueue; + jl_workqueue_t workqueue; std::swap(params.workqueue, workqueue); jl_code_instance_t *codeinst = NULL; JL_GC_PUSH1(&codeinst); @@ -418,7 +422,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root { auto it = compiled_functions.find(codeinst); if (it != compiled_functions.end()) { - auto &decls = it->second.second; + auto &decls = it->second.decls; invokeName = decls.functionObject; if (decls.functionObject == "jl_fptr_args") { preal_decl = decls.specFunctionObject; @@ -442,8 +446,11 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root } if (preal_decl.empty()) { pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); - if (!proto.specsig) + if (!proto.specsig) { proto.decl->replaceAllUsesWith(pinvoke); + proto.decl->eraseFromParent(); + proto.decl = pinvoke; + } } if (proto.specsig && !preal_specsig) { // get or build an fptr1 that can invoke codeinst @@ -462,9 +469,12 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root } if (!preal_decl.empty()) { // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(preal_decl)) { - if (proto.decl != specfun) + if (Function *specfun = cast_or_null(mod->getNamedValue(preal_decl))) { + if (proto.decl != specfun) { proto.decl->replaceAllUsesWith(specfun); + proto.decl->eraseFromParent(); + proto.decl = specfun; + } } else { proto.decl->setName(preal_decl); @@ -482,9 +492,12 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root assert(ocinvokeDecl != "jl_fptr_const_return"); assert(ocinvokeDecl != "jl_fptr_sparam"); // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) { - if (proto.oc != specfun) + if (Function *specfun = cast_or_null(mod->getNamedValue(ocinvokeDecl))) { + if (proto.oc != specfun) { proto.oc->replaceAllUsesWith(specfun); + proto.oc->eraseFromParent(); + proto.oc = specfun; + } } else { proto.oc->setName(ocinvokeDecl); @@ -496,6 +509,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root JL_GC_POP(); } + /// Link the function in the source module into the destination module if /// needed, setting up mapping information. /// Similar to orc::cloneFunctionDecl, but more complete for greater correctness @@ -577,8 +591,8 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct codeinst = it->second; JL_GC_PROMISE_ROOTED(codeinst); auto defs = compiled_functions.find(codeinst); - defM = std::get<0>(defs->second).getModuleUnlocked(); - const jl_llvm_functions_t &decls = std::get<1>(defs->second); + defM = defs->second.TSM.getModuleUnlocked(); + const jl_llvm_functions_t &decls = defs->second.decls; func = decls.functionObject; StringRef specfunc = decls.specFunctionObject; jl_value_t *astrt = codeinst->rettype; @@ -624,6 +638,25 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } } +// destructively move the contents of src into dest +// this assumes that the targets of the two modules are the same +// including the DataLayout and ModuleFlags (for example) +// and that there is no module-level assembly +// Comdat is also removed, since this needs to be re-added later +static void jl_merge_module(Linker &L, orc::ThreadSafeModule srcTSM) JL_NOTSAFEPOINT +{ + srcTSM.consumingModuleDo([&L](std::unique_ptr src) JL_NOTSAFEPOINT { + bool error = L.linkInModule(std::move(src)); + assert(!error && "linking llvmcall modules failed"); + (void)error; + }); +} + +static bool canPartition(const Function &F) +{ + return !F.hasFnAttribute(Attribute::AlwaysInline) && + !F.hasFnAttribute(Attribute::InlineHint); +} // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup @@ -743,7 +776,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm orc::ThreadSafeModule backing; if (!llvmmod) { ctx = jl_ExecutionEngine->makeContext(); - backing = jl_create_ts_module("text", ctx); + backing = jl_create_ts_module("text", ctx, jl_ExecutionEngine->getDataLayout(), jl_ExecutionEngine->getTargetTriple()); } orc::ThreadSafeModule &clone = llvmmod ? *unwrap(llvmmod) : backing; auto ctxt = clone.getContext(); @@ -760,6 +793,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm assert(params.imaging_mode); // `_imaging_mode` controls if broken features like code-coverage are disabled params.external_linkage = external_linkage; params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); + bool safepoint_on_entry = params.safepoint_on_entry; JL_GC_PUSH3(¶ms.temporary_roots, &method_roots.list, &method_roots.keyset); jl_compiled_functions_t compiled_functions; size_t i, l; @@ -774,17 +808,8 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm assert(jl_is_code_info(src)); if (compiled_functions.count(codeinst)) continue; // skip any duplicates that accidentally made there way in here (or make this an error?) - if (external_linkage) { - uint8_t specsigflags; - jl_callptr_t invoke; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - if (invoke != NULL && (specsigflags & 0b100)) { - // this codeinst is already available externally - // TODO: for performance, avoid generating the src code when we know it would reach here anyways - continue; - } - } + if (jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + params.safepoint_on_entry = false; // ensure we don't block ExpandAtomicModifyPass from inlining this code if applicable orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, clone.getModuleUnlocked()->getDataLayout(), Triple(clone.getModuleUnlocked()->getTargetTriple())); @@ -793,6 +818,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm decls.functionObject = "jl_fptr_const_return"; else decls = jl_emit_codeinst(result_m, codeinst, src, params); + params.safepoint_on_entry = safepoint_on_entry; record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; @@ -823,7 +849,6 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm size_t idx = 0; for (auto &global : params.global_targets) { gvars[idx] = global.second->getName().str(); - global.second->setInitializer(literal_static_pointer_val(global.first, global.second->getValueType())); assert(gvars_set.insert(global.second).second && "Duplicate gvar in params!"); assert(gvars_names.insert(gvars[idx]).second && "Duplicate gvar name in params!"); data->jl_value_to_llvm[idx] = global.first; @@ -854,11 +879,27 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm { Linker L(*clone.getModuleUnlocked()); for (auto &def : compiled_functions) { - jl_merge_module(clone, std::move(std::get<0>(def.second))); jl_code_instance_t *this_code = def.first; - jl_llvm_functions_t decls = std::get<1>(def.second); + JL_GC_PROMISE_ROOTED(this_code); + jl_llvm_functions_t &decls = def.second.decls; StringRef func = decls.functionObject; StringRef cfunc = decls.specFunctionObject; + orc::ThreadSafeModule &M = def.second.TSM; + if (external_linkage) { + uint8_t specsigflags; + jl_callptr_t invoke; + void *fptr; + jl_read_codeinst_invoke(this_code, &specsigflags, &invoke, &fptr, 0); + if (invoke != NULL && (specsigflags & 0b100)) { + // this codeinst is already available externally: keep it only if canPartition demands it for local use + // TODO: for performance, avoid generating the src code when we know it would reach here anyways? + if (M.withModuleDo([&](Module &M) { return !canPartition(*cast(M.getNamedValue(cfunc))); })) { + jl_merge_module(L, std::move(M)); + } + continue; + } + } + jl_merge_module(L, std::move(M)); uint32_t func_id = 0; uint32_t cfunc_id = 0; if (func == "jl_fptr_args") { @@ -885,6 +926,52 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm } data->jl_fvar_map[this_code] = std::make_tuple(func_id, cfunc_id); } + bool Changed = true; + while (Changed) { + Changed = false; + // make sure everything referenced got included though, since some functions aren't + // correctly implemented by staticdata for external use, and so codegen won't emit + // an external reference but expects a private copy here instead + for (auto &def : compiled_functions) { + orc::ThreadSafeModule &M = def.second.TSM; + if (!M) + continue; + jl_llvm_functions_t &decls = def.second.decls; + StringRef func = decls.functionObject; + StringRef cfunc = decls.specFunctionObject; + if (func != "jl_fptr_args" && + func != "jl_fptr_sparam" && + func != "jl_f_opaque_closure_call" && + clone.getModuleUnlocked()->getNamedValue(func)) { + jl_merge_module(L, std::move(M)); + Changed = true; + continue; + } + if (!cfunc.empty() && clone.getModuleUnlocked()->getNamedValue(cfunc)) { + Changed = true; + jl_merge_module(L, std::move(M)); + } + } + } +#ifndef NDEBUG + // make sure we didn't forget anything that we promised to include in here + for (auto &def : compiled_functions) { + jl_llvm_functions_t &decls = def.second.decls; + StringRef func = decls.functionObject; + StringRef cfunc = decls.specFunctionObject; + if (func != "jl_fptr_args" && + func != "jl_fptr_sparam" && + func != "jl_f_opaque_closure_call") { + GlobalValue *F = clone.getModuleUnlocked()->getNamedValue(func); + assert(!F || !F->isDeclaration()); + } + if (!cfunc.empty()) { + GlobalValue *F = clone.getModuleUnlocked()->getNamedValue(cfunc); + assert(!F || !F->isDeclaration()); + } + } +#endif + compiled_functions.clear(); if (params._shared_module) { bool error = L.linkInModule(std::move(params._shared_module)); assert(!error && "Error linking in shared module"); @@ -894,15 +981,35 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm // now get references to the globals in the merged module // and set them to be internalized and initialized at startup + // filter out any gvars that got optimized away + idx = 0; + size_t newoffset = 0; + size_t newidx = 0; for (auto &global : gvars) { //Safe b/c context is locked by params - GlobalVariable *G = cast(clone.getModuleUnlocked()->getNamedValue(global)); - assert(G->hasInitializer()); - G->setLinkage(GlobalValue::InternalLinkage); - G->setDSOLocal(true); - data->jl_sysimg_gvars.push_back(G); + GlobalVariable *G = cast_or_null(clone.getModuleUnlocked()->getNamedValue(global)); + if (G != nullptr) { + assert(!G->hasInitializer()); + G->setInitializer(Constant::getNullValue(G->getValueType())); + G->setLinkage(GlobalValue::InternalLinkage); + G->setDSOLocal(true); + assert(newidx == data->jl_sysimg_gvars.size()); + if (idx < offset) { + data->jl_value_to_llvm[newidx] = data->jl_value_to_llvm[idx]; + newoffset = newidx + 1; + } + else { + data->jl_external_to_llvm[newidx - newoffset] = data->jl_external_to_llvm[idx - offset]; + } + data->jl_sysimg_gvars.push_back(G); + newidx++; + } + idx++; } - CreateNativeGlobals += gvars.size(); + data->jl_value_to_llvm.resize(newoffset); + data->jl_external_to_llvm.resize(newidx - newoffset); + gvars.clear(); + CreateNativeGlobals += idx; data->M = std::move(clone); return (void*)data; @@ -1126,11 +1233,6 @@ struct Partition { size_t weight; }; -static bool canPartition(const Function &F) -{ - return !F.hasFnAttribute(Attribute::AlwaysInline); -} - static inline bool verify_partitioning(const SmallVectorImpl &partitions, const Module &M, DenseMap &fvars, DenseMap &gvars) { bool bad = false; #ifndef JL_NDEBUG @@ -1583,7 +1685,8 @@ static void materializePreserved(Module &M, Partition &partition) { // This just avoids a hashtable lookup. GV->setLinkage(GlobalValue::InternalLinkage); assert(GV->hasDefaultVisibility()); - } else { + } + else { Preserve.insert(GV); } } @@ -2094,11 +2197,6 @@ void jl_dump_native_impl(void *native_code, addComdat(&GA, TheTriple); } - // Wipe the global initializers, we'll reset them at load time - for (auto gv : data->jl_sysimg_gvars) { - cast(gv)->setInitializer(Constant::getNullValue(gv->getValueType())); - } - // add metadata information if (imaging_mode) { multiversioning_preannotate(dataM); @@ -2359,17 +2457,16 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ dump->TSM = nullptr; if (src && jl_is_code_info(src)) { auto ctx = jl_ExecutionEngine->makeContext(); - orc::ThreadSafeModule m = jl_create_ts_module(name_from_method_instance(mi), ctx); + const auto &DL = jl_ExecutionEngine->getDataLayout(); + const auto &TT = jl_ExecutionEngine->getTargetTriple(); + orc::ThreadSafeModule m = jl_create_ts_module(name_from_method_instance(mi), ctx, DL, TT); Function *F = nullptr; { uint64_t compiler_start_time = 0; uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled); if (measure_compile_time_enabled) compiler_start_time = jl_hrtime(); - auto target_info = m.withModuleDo([&](Module &M) { - return std::make_pair(M.getDataLayout(), Triple(M.getTargetTriple())); - }); - jl_codegen_params_t output(ctx, std::move(target_info.first), std::move(target_info.second)); + jl_codegen_params_t output(ctx, DL, TT); output.params = ¶ms; output.imaging_mode = jl_options.image_codegen; output.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); @@ -2389,7 +2486,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED); if (codeinst == nullptr || compiled_functions.count(codeinst)) continue; - orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx); + orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx, DL, TT); jl_llvm_functions_t decls; if (jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) decls.functionObject = "jl_fptr_const_return"; @@ -2398,6 +2495,8 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ compiled_functions[codeinst] = {std::move(decl_m), std::move(decls)}; } generate_cfunc_thunks(output, compiled_functions); + emit_always_inline(m, output); + output.workqueue.clear(); compiled_functions.clear(); output.temporary_roots = nullptr; JL_GC_POP(); // GC the global_targets array contents now since reflection doesn't need it @@ -2412,7 +2511,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ } else { auto p = literal_static_pointer_val(global.first, global.second->getValueType()); - Type *elty = PointerType::get(output.getContext(), 0); + Type *elty = PointerType::get(p->getContext(), 0); // For pretty printing, when LLVM inlines the global initializer into its loads auto alias = GlobalAlias::create(elty, 0, GlobalValue::PrivateLinkage, global.second->getName() + ".jit", p, global.second->getParent()); global.second->setInitializer(ConstantExpr::getBitCast(alias, global.second->getValueType())); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d9b7b98e40ef4..64d6f6eb54de8 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2265,8 +2265,10 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j return mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); } +static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, const jl_cgval_t &modifyop, jl_value_t *jltype, Type *elty, jl_cgval_t rhs, const Twine &fname, bool gcstack_arg); + static jl_cgval_t typed_store(jl_codectx_t &ctx, - Value *ptr, jl_cgval_t rhs, jl_cgval_t cmp, + Value *ptr, jl_cgval_t rhs, jl_cgval_t cmpop, jl_value_t *jltype, MDNode *tbaa, MDNode *aliasscope, Value *parent, // for the write barrier, NULL if no barrier needed bool isboxed, AtomicOrdering Order, AtomicOrdering FailOrder, unsigned alignment, @@ -2275,10 +2277,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *var) { auto newval = [&](const jl_cgval_t &lhs) { - const jl_cgval_t argv[3] = { cmp, lhs, rhs }; + const jl_cgval_t argv[3] = { cmpop, lhs, rhs }; jl_cgval_t ret; if (modifyop) { - ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, true); } else { Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); @@ -2302,7 +2304,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, return rhs; } else if (isreplacefield) { - Value *Success = emit_f_is(ctx, cmp, ghostValue(ctx, jltype)); + Value *Success = emit_f_is(ctx, cmpop, ghostValue(ctx, jltype)); Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); const jl_cgval_t argv[2] = {ghostValue(ctx, jltype), mark_julia_type(ctx, Success, false, jl_bool_type)}; jl_datatype_t *rettyp = jl_apply_cmpswap_type(jltype); @@ -2403,6 +2405,46 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ai.decorateInst(store); instr = store; } + else if (ismodifyfield && modifyop && !needlock && Order != AtomicOrdering::NotAtomic && !isboxed && realelty == elty && !intcast && elty->isIntegerTy() && !jl_type_hasptr(jltype)) { + // emit this only if we have a possibility of optimizing it + if (Order == AtomicOrdering::Unordered) + Order = AtomicOrdering::Monotonic; + if (jl_is_pointerfree(rhs.typ) && !rhs.isghost && (rhs.constant || rhs.isboxed || rhs.ispointer())) { + // if this value can be loaded from memory, do that now so that it is sequenced before the atomicmodify + // and the IR is less dependent on what was emitted before now to create this rhs. + // Inlining should do okay to clean this up later if there are parts we don't need. + rhs = jl_cgval_t(emit_unbox(ctx, julia_type_to_llvm(ctx, rhs.typ), rhs, rhs.typ), rhs.typ, NULL); + } + bool gcstack_arg = JL_FEAT_TEST(ctx,gcstack_arg); + Function *op = emit_modifyhelper(ctx, cmpop, *modifyop, jltype, elty, rhs, fname, gcstack_arg); + std::string intr_name = "julia.atomicmodify.i"; + intr_name += utostr(cast(elty)->getBitWidth()); + intr_name += ".p"; + intr_name += utostr(ptr->getType()->getPointerAddressSpace()); + FunctionCallee intr = jl_Module->getOrInsertFunction(intr_name, + FunctionType::get(StructType::get(elty, elty), {ptr->getType(), ctx.builder.getPtrTy(), ctx.builder.getInt8Ty(), ctx.builder.getInt8Ty()}, true), + AttributeList::get(elty->getContext(), + Attributes(elty->getContext(), {Attribute::NoMerge}), // prevent llvm from merging calls to different functions + AttributeSet(), + None)); + SmallVector Args = {ptr, op, ctx.builder.getInt8((unsigned)Order), ctx.builder.getInt8(SyncScope::System)}; + if (rhs.V) + Args.push_back(rhs.V); + if (rhs.Vboxed) + Args.push_back(rhs.Vboxed); + if (rhs.TIndex) + Args.push_back(rhs.TIndex); + Args.append(rhs.inline_roots); + if (gcstack_arg) + Args.push_back(ctx.pgcstack); + auto oldnew = ctx.builder.CreateCall(intr, Args); + oldnew->addParamAttr(0, Attribute::getWithAlignment(oldnew->getContext(), Align(alignment))); + //jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + //ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + //ai.decorateInst(oldnew); + oldval = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 0), isboxed, jltype); + rhs = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 1), isboxed, jltype); + } else { // replacefield, modifyfield, swapfield, setfieldonce (isboxed && atomic) DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); @@ -2416,7 +2458,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, assert(jl_is_concrete_type(jltype)); needloop = ((jl_datatype_t*)jltype)->layout->flags.haspadding || !((jl_datatype_t*)jltype)->layout->flags.isbitsegal; - Value *SameType = emit_isa(ctx, cmp, jltype, Twine()).first; + Value *SameType = emit_isa(ctx, cmpop, jltype, Twine()).first; if (SameType != ConstantInt::getTrue(ctx.builder.getContext())) { BasicBlock *SkipBB = BasicBlock::Create(ctx.builder.getContext(), "skip_xchg", ctx.f); BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "ok_xchg", ctx.f); @@ -2436,22 +2478,22 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Current->addIncoming(instr, SkipBB); ctx.builder.SetInsertPoint(BB); } - cmp = update_julia_type(ctx, cmp, jltype); + cmpop = update_julia_type(ctx, cmpop, jltype); if (intcast) { - emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); + emit_unbox_store(ctx, cmpop, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); Compare = ctx.builder.CreateLoad(realelty, intcast); } else { - Compare = emit_unbox(ctx, realelty, cmp, jltype); + Compare = emit_unbox(ctx, realelty, cmpop, jltype); } if (realelty != elty) Compare = ctx.builder.CreateZExt(Compare, elty); } - else if (cmp.isboxed || cmp.constant || jl_pointer_egal(jltype)) { - Compare = boxed(ctx, cmp); - needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(cmp.typ); - if (needloop && !cmp.isboxed) // try to use the same box in the compare now and later - cmp = mark_julia_type(ctx, Compare, true, cmp.typ); + else if (cmpop.isboxed || cmpop.constant || jl_pointer_egal(jltype)) { + Compare = boxed(ctx, cmpop); + needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(cmpop.typ); + if (needloop && !cmpop.isboxed) // try to use the same box in the compare now and later + cmpop = mark_julia_type(ctx, Compare, true, cmpop.typ); } else { Compare = Constant::getNullValue(ctx.types().T_prjlvalue); // TODO: does this need to be an invalid bit pattern? @@ -2485,7 +2527,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } if (ismodifyfield) { if (needlock) - emit_lockstate_value(ctx, needlock, false); + emit_lockstate_value(ctx, needlock, false); // unlock Value *realCompare = Compare; if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); @@ -2520,8 +2562,8 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) r = ctx.builder.CreateZExt(r, elty); if (needlock) - emit_lockstate_value(ctx, needlock, true); - cmp = oldval; + emit_lockstate_value(ctx, needlock, true); // relock + cmpop = oldval; } Value *Done; if (Order == AtomicOrdering::NotAtomic) { @@ -2541,7 +2583,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (issetfieldonce) Success = ctx.builder.CreateIsNull(first_ptr); else - Success = emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); + Success = emit_f_is(ctx, oldval, cmpop, first_ptr, nullptr); if (needloop && ismodifyfield) CmpPhi->addIncoming(load, ctx.builder.GetInsertBlock()); assert(Succ == nullptr); @@ -2599,12 +2641,12 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Done = ctx.builder.CreateIsNotNull(first_ptr); } else { - // Done = !(!Success && (first_ptr != NULL && oldval == cmp)) + // Done = !(!Success && (first_ptr != NULL && oldval == cmpop)) Done = emit_guarded_test(ctx, ctx.builder.CreateNot(Success), false, [&] { Value *first_ptr = nullptr; if (maybe_null_if_boxed) first_ptr = isboxed ? realinstr : extract_first_ptr(ctx, realinstr); - return emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); + return emit_f_is(ctx, oldval, cmpop, first_ptr, nullptr); }); Done = ctx.builder.CreateNot(Done); } @@ -4024,7 +4066,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, emit_lockstate_value(ctx, needlock, false); const jl_cgval_t argv[3] = { cmp, oldval, rhs }; if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, true); } else { Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); diff --git a/src/codegen.cpp b/src/codegen.cpp index af86b568c2b0f..473b0709e9f93 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2076,7 +2076,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -5124,10 +5124,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos namep += cast(TheCallee)->getName(); GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(TheCallee->getType()), - namep); + GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, GlobalVariable::ExternalLinkage, nullptr, namep); ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; } jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); @@ -5174,10 +5171,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); Type *pfunc = PointerType::getUnqual(ctx.builder.getContext()); if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, pfunc, false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(pfunc), - namep); + GV = new GlobalVariable(*jl_Module, pfunc, false, GlobalVariable::ExternalLinkage, nullptr, namep); ctx.emission_context.external_fns[std::make_tuple(fromexternal, false)] = GV; } jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); @@ -5206,10 +5200,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv, nargs, rt); + return emit_invoke(ctx, lival, argv, nargs, rt, false); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline) { ++EmittedInvokes; bool handled = false; @@ -5265,44 +5259,52 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR std::string name; StringRef protoname; bool need_to_emit = true; - bool cache_valid = ctx.use_cache || ctx.external_linkage; + bool cache_valid = (ctx.use_cache || ctx.external_linkage); bool external = false; // Check if we already queued this up auto it = ctx.call_targets.find(codeinst); - if (need_to_emit && it != ctx.call_targets.end()) { + if (it != ctx.call_targets.end()) { assert(it->second.specsig == specsig); protoname = it->second.decl->getName(); - need_to_emit = cache_valid = false; + if (always_inline) + it->second.private_linkage = true; + else + it->second.external_linkage = true; } - - // Check if it is already compiled (either JIT or externally) - if (need_to_emit && cache_valid) { - // optimization: emit the correct name immediately, if we know it + // Check if it is already compiled (either JIT or externally), and if so, re-use that name if possible + // This is just an optimization to emit the correct name immediately, if we know it, since the JIT and AOT code will be able to do this later also + if (cache_valid) { // TODO: use `emitted` map here too to try to consolidate names? uint8_t specsigflags; jl_callptr_t invoke; void *fptr; jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) { - protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); if (ctx.external_linkage) { // TODO: Add !specsig support to aotcompile.cpp // Check that the codeinst is containing native code if (specsig && (specsigflags & 0b100)) { - external = true; + external = !always_inline; need_to_emit = false; } } else { // ctx.use_cache need_to_emit = false; } + if (!need_to_emit && protoname.empty()) + protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); } } - if (need_to_emit) { + if (it != ctx.call_targets.end()) + need_to_emit = false; + else if (always_inline) + need_to_emit = true; + if (protoname.empty()) { raw_string_ostream(name) << (specsig ? "j_" : "j1_") << name_from_method_instance(mi) << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); protoname = StringRef(name); } + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) @@ -5311,7 +5313,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); - ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; + ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig, !always_inline, always_inline}; } } } @@ -6257,7 +6259,7 @@ static std::pair get_oc_function(jl_codectx_t &ctx, jl_met } if (need_to_emit) { - ctx.call_targets[ci] = {cc, return_roots, specsig ? specF : F, specsig ? F : nullptr, specsig}; + ctx.call_targets[ci] = {cc, return_roots, specsig ? specF : F, specsig ? F : nullptr, specsig, true, false}; } JL_GC_POP(); @@ -6780,6 +6782,71 @@ Function *get_or_emit_fptr1(StringRef preal_decl, Module *M) return cast(M->getOrInsertFunction(preal_decl, get_func_sig(M->getContext()), get_func_attrs(M->getContext())).getCallee()); } +static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, const jl_cgval_t &modifyop, jl_value_t *jltype, Type *elty, jl_cgval_t rhs, const Twine &fname, bool gcstack_arg) +{ + Module *M = ctx2.f->getParent(); + jl_codectx_t ctx(M->getContext(), ctx2.emission_context, ctx2.min_world, ctx2.max_world); + SmallVector ArgTy; + ArgTy.push_back(elty); + if (rhs.V) + ArgTy.push_back(rhs.V->getType()); + if (rhs.Vboxed) + ArgTy.push_back(rhs.Vboxed->getType()); + if (rhs.TIndex) + ArgTy.push_back(rhs.TIndex->getType()); + for (auto &root : rhs.inline_roots) + ArgTy.push_back(root->getType()); + if (gcstack_arg) + ArgTy.push_back(ctx.builder.getPtrTy()); + FunctionType *FT = FunctionType::get(elty, ArgTy, false); + Function *w = Function::Create(FT, GlobalVariable::PrivateLinkage, "", M); + jl_init_function(w, ctx.emission_context.TargetTriple); + w->addFnAttr(Attribute::AlwaysInline); + w->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + Function::arg_iterator AI = w->arg_begin(); + Argument *A = &*AI++; + // rebuild a copy of rhs from the arguments + if (rhs.V) + rhs.V = &*AI++; + if (rhs.Vboxed) + rhs.Vboxed = &*AI++; + if (rhs.TIndex) + rhs.TIndex = &*AI++; + for (size_t i = 0; i < rhs.inline_roots.size(); i++) + rhs.inline_roots[i] = &*AI++; + rhs.promotion_point = nullptr; + rhs.promotion_ssa = -1; + if (gcstack_arg) { + w->setCallingConv(CallingConv::Swift); + AttrBuilder param(ctx.builder.getContext()); + param.addAttribute(Attribute::SwiftSelf); + param.addAttribute(Attribute::NonNull); + Argument *gcstackarg = &*AI++; + gcstackarg->addAttrs(param); + gcstackarg->setName("pgcstack_arg"); + ctx.pgcstack = gcstackarg; + } + assert(AI == w->arg_end()); + ctx.f = w; + ctx.rettype = jltype; + BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); + ctx.builder.SetInsertPoint(b0); + DebugLoc noDbg; + ctx.builder.SetCurrentDebugLocation(noDbg); + allocate_gc_frame(ctx, b0); + const jl_cgval_t argv[3] = { op, mark_julia_type(ctx, A, false, jltype), rhs }; + jl_cgval_t ret = emit_invoke(ctx, modifyop, argv, 3, (jl_value_t*)jl_any_type, true); + emit_typecheck(ctx, ret, jltype, fname); + ret = update_julia_type(ctx, ret, jltype); + ctx.builder.CreateRet(emit_unbox(ctx, elty, ret, jltype)); + if (ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } + return w; +} + + Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Value *theFunc, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT { ++EmittedToJLInvokes; @@ -8998,7 +9065,7 @@ static jl_llvm_functions_t Instruction &prologue_end = ctx.builder.GetInsertBlock()->back(); // step 11a. Emit the entry safepoint - if (JL_FEAT_TEST(ctx, safepoint_on_entry)) + if (params.safepoint_on_entry && JL_FEAT_TEST(ctx, safepoint_on_entry)) emit_gc_safepoint(ctx.builder, ctx.types().T_size, get_current_ptls(ctx), ctx.tbaa().tbaa_const); // step 11b. Do codegen in control flow order @@ -9822,6 +9889,84 @@ jl_llvm_functions_t jl_emit_codeinst( return decls; } +/// Stolen from IRMover.cpp, since it is needlessly private there +void linkFunctionBody(Function &Dst, Function &Src) +{ + assert(Dst.isDeclaration() && !Src.isDeclaration()); + + // Link in the operands without remapping. + if (Src.hasPrefixData()) + Dst.setPrefixData(Src.getPrefixData()); + if (Src.hasPrologueData()) + Dst.setPrologueData(Src.getPrologueData()); + if (Src.hasPersonalityFn()) + Dst.setPersonalityFn(Src.getPersonalityFn()); + if (Src.hasPersonalityFn()) + Dst.setPersonalityFn(Src.getPersonalityFn()); + assert(Src.IsNewDbgInfoFormat == Dst.IsNewDbgInfoFormat); + + // Copy over the metadata attachments without remapping. + Dst.copyMetadata(&Src, 0); + + // Steal arguments and splice the body of Src into Dst. + Dst.stealArgumentListFrom(Src); + Dst.splice(Dst.end(), &Src); +} + +void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER +{ + jl_workqueue_t &edges = params.workqueue; + bool always_inline = false; + for (auto &it : edges) { + if (it.second.private_linkage) + always_inline = true; + } + if (!always_inline) + return; + jl_task_t *ct = jl_current_task; + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) + jl_code_info_t *src = nullptr; + params.safepoint_on_entry = false; + params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); + JL_GC_PUSH2(¶ms.temporary_roots, &src); + for (auto &it : edges) { + jl_code_instance_t *codeinst = it.first; + auto &proto = it.second; + if (!proto.private_linkage) + continue; + if (proto.decl->isDeclaration()) { + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + jl_method_t *def = mi->def.method; + if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); + if (src && jl_is_code_info(src) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) { + jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints + if (!result_m) + break; + // TODO: jl_optimize_roots(params, mi, *result_m.getModuleUnlocked()); // contains safepoints + Module &M = *result_m.getModuleUnlocked(); + if (decls.functionObject != "jl_fptr_args" && + decls.functionObject != "jl_fptr_sparam" && + decls.functionObject != "jl_f_opaque_closure_call") { + Function *F = M.getFunction(decls.functionObject); + F->eraseFromParent(); + } + if (!decls.specFunctionObject.empty()) { + Function *specF = M.getFunction(decls.specFunctionObject); + linkFunctionBody(*proto.decl, *specF); + proto.decl->addFnAttr(Attribute::InlineHint); + proto.decl->setLinkage(proto.external_linkage ? GlobalValue::AvailableExternallyLinkage : GlobalValue::PrivateLinkage); + specF->eraseFromParent(); + } + } + } + } + params.temporary_roots = nullptr; + JL_GC_POP(); + jl_gc_unsafe_leave(ct->ptls, gc_state); +} + // --- initialization --- static auto gv_for_global = new SmallVector, 0>(); static void global_jlvalue_to_llvm(JuliaVariable *var, jl_value_t **addr) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index bf49b7010b97b..99fb1b8f09bfb 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -71,7 +71,6 @@ STATISTIC(OptO0, "Number of modules optimized at level -O0"); STATISTIC(OptO1, "Number of modules optimized at level -O1"); STATISTIC(OptO2, "Number of modules optimized at level -O2"); STATISTIC(OptO3, "Number of modules optimized at level -O3"); -STATISTIC(ModulesMerged, "Number of modules merged"); STATISTIC(InternedGlobals, "Number of global constants interned in the string pool"); #ifdef _COMPILER_MSAN_ENABLED_ @@ -339,177 +338,196 @@ static DenseMap> incompl static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t ¶ms, bool forceall=false) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { jl_task_t *ct = jl_current_task; - decltype(params.workqueue) edges; + jl_workqueue_t edges; std::swap(params.workqueue, edges); for (auto &it : edges) { jl_code_instance_t *codeinst = it.first; JL_GC_PROMISE_ROOTED(codeinst); auto &proto = it.second; - // try to emit code for this item from the workqueue - StringRef invokeName = ""; - StringRef preal_decl = ""; - bool preal_specsig = false; - jl_callptr_t invoke = nullptr; - bool isedge = false; - assert(params.cache); - // Checking the cache here is merely an optimization and not strictly required - // But it must be consistent with the following invokenames lookup, which is protected by the engine_lock - uint8_t specsigflags; - void *fptr; - void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; // declare it is not a safepoint (or deadlock) in this file due to 0 parameter - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) - if (invoke == jl_fptr_args_addr) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - } - else if (specsigflags & 0b1) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - preal_specsig = true; - } - bool force = forceall || invoke != nullptr; - if (preal_decl.empty()) { - auto it = invokenames.find(codeinst); - if (it != invokenames.end()) { - auto &decls = it->second; - invokeName = decls.functionObject; - if (decls.functionObject == "jl_fptr_args") { - preal_decl = decls.specFunctionObject; - isedge = true; - } - else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { - preal_decl = decls.specFunctionObject; - preal_specsig = true; - isedge = true; - } - force = true; + if (proto.external_linkage || proto.decl->isDeclaration()) { // if it is not expected externally and has a definition locally, there is no need to patch this edge up + // try to emit code for this item from the workqueue + StringRef invokeName = ""; + StringRef preal_decl = ""; + bool preal_specsig = false; + jl_callptr_t invoke = nullptr; + bool isedge = false; + assert(params.cache); + // Checking the cache here is merely an optimization and not strictly required + // But it must be consistent with the following invokenames lookup, which is protected by the engine_lock + uint8_t specsigflags; + void *fptr; + void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; // declare it is not a safepoint (or deadlock) in this file due to 0 parameter + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); + //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) + if (invoke == jl_fptr_args_addr) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); } - } - if (preal_decl.empty()) { - // there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead - jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 0); - if (compiled_ci != codeinst) { - codeinst = compiled_ci; - uint8_t specsigflags; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) - if (invoke == jl_fptr_args_addr) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - } - else if (specsigflags & 0b1) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - preal_specsig = true; - } - if (preal_decl.empty()) { - auto it = invokenames.find(codeinst); - if (it != invokenames.end()) { - auto &decls = it->second; - invokeName = decls.functionObject; - if (decls.functionObject == "jl_fptr_args") { - preal_decl = decls.specFunctionObject; - isedge = true; - } - else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { - preal_decl = decls.specFunctionObject; - preal_specsig = true; - isedge = true; - } - } - } + else if (specsigflags & 0b1) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + preal_specsig = true; } - } - if (!preal_decl.empty() || force) { - // if we have a prototype emitted, compare it to what we emitted earlier - Module *mod = proto.decl->getParent(); - assert(proto.decl->isDeclaration()); - Function *pinvoke = nullptr; + bool force = forceall || invoke != nullptr; if (preal_decl.empty()) { - if (invoke != nullptr && invokeName.empty()) { - assert(invoke != jl_fptr_args_addr); - if (invoke == jl_fptr_sparam_addr) - invokeName = "jl_fptr_sparam"; - else if (invoke == jl_f_opaque_closure_call_addr) - invokeName = "jl_f_opaque_closure_call"; - else - invokeName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + auto it = invokenames.find(codeinst); + if (it != invokenames.end()) { + auto &decls = it->second; + invokeName = decls.functionObject; + if (decls.functionObject == "jl_fptr_args") { + preal_decl = decls.specFunctionObject; + isedge = true; + } + else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { + preal_decl = decls.specFunctionObject; + preal_specsig = true; + isedge = true; + } + force = true; } - pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); - if (!proto.specsig) - proto.decl->replaceAllUsesWith(pinvoke); - isedge = false; - } - if (proto.specsig && !preal_specsig) { - // get or build an fptr1 that can invoke codeinst - if (pinvoke == nullptr) - pinvoke = get_or_emit_fptr1(preal_decl, mod); - // emit specsig-to-(jl)invoke conversion - proto.decl->setLinkage(GlobalVariable::InternalLinkage); - //protodecl->setAlwaysInline(); - jl_init_function(proto.decl, params.TargetTriple); - // TODO: maybe this can be cached in codeinst->specfptr? - int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed - bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); - jl_gc_unsafe_leave(ct->ptls, gc_state); - preal_decl = ""; // no need to fixup the name } - if (!preal_decl.empty()) { - // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(preal_decl)) { - if (proto.decl != specfun) - proto.decl->replaceAllUsesWith(specfun); - } - else { - proto.decl->setName(preal_decl); + if (preal_decl.empty()) { + // there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead + jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 0); + if (compiled_ci != codeinst) { + codeinst = compiled_ci; + uint8_t specsigflags; + void *fptr; + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); + //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) + if (invoke == jl_fptr_args_addr) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + } + else if (specsigflags & 0b1) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + preal_specsig = true; + } + if (preal_decl.empty()) { + auto it = invokenames.find(codeinst); + if (it != invokenames.end()) { + auto &decls = it->second; + invokeName = decls.functionObject; + if (decls.functionObject == "jl_fptr_args") { + preal_decl = decls.specFunctionObject; + isedge = true; + } + else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { + preal_decl = decls.specFunctionObject; + preal_specsig = true; + isedge = true; + } + } + } } } - if (proto.oc) { // additionally, if we are dealing with an OC constructor, then we might also need to fix up the fptr1 reference too - assert(proto.specsig); - StringRef ocinvokeDecl = invokeName; - if (invoke != nullptr && ocinvokeDecl.empty()) { - // check for some special tokens used by opaque_closure.c and convert those to their real functions - assert(invoke != jl_fptr_args_addr); - assert(invoke != jl_fptr_sparam_addr); - if (invoke == jl_fptr_interpret_call_addr) - ocinvokeDecl = "jl_fptr_interpret_call"; - else if (invoke == jl_fptr_const_return_addr) - ocinvokeDecl = "jl_fptr_const_return"; - else if (invoke == jl_f_opaque_closure_call_addr) - ocinvokeDecl = "jl_f_opaque_closure_call"; - //else if (invoke == jl_interpret_opaque_closure_addr) - else - ocinvokeDecl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + if (!preal_decl.empty() || force) { + // if we have a prototype emitted, compare it to what we emitted earlier + Module *mod = proto.decl->getParent(); + Function *pinvoke = nullptr; + if (proto.decl->isDeclaration()) { + if (preal_decl.empty()) { + if (invoke != nullptr && invokeName.empty()) { + assert(invoke != jl_fptr_args_addr); + if (invoke == jl_fptr_sparam_addr) + invokeName = "jl_fptr_sparam"; + else if (invoke == jl_f_opaque_closure_call_addr) + invokeName = "jl_f_opaque_closure_call"; + else + invokeName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + } + pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); + if (!proto.specsig) { + proto.decl->replaceAllUsesWith(pinvoke); + proto.decl->eraseFromParent(); + proto.decl = pinvoke; + } + isedge = false; + } + if (proto.specsig && !preal_specsig) { + // get or build an fptr1 that can invoke codeinst + if (pinvoke == nullptr) + pinvoke = get_or_emit_fptr1(preal_decl, mod); + // emit specsig-to-(jl)invoke conversion + proto.decl->setLinkage(GlobalVariable::InternalLinkage); + //protodecl->setAlwaysInline(); + jl_init_function(proto.decl, params.TargetTriple); + // TODO: maybe this can be cached in codeinst->specfptr? + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed + bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); + jl_gc_unsafe_leave(ct->ptls, gc_state); + preal_decl = ""; // no need to fixup the name + } } - // if OC expected a specialized specsig dispatch, but we don't have it, use the inner trampoline here too - // XXX: this invoke translation logic is supposed to exactly match new_opaque_closure - if (!preal_specsig || ocinvokeDecl == "jl_f_opaque_closure_call" || ocinvokeDecl == "jl_fptr_interpret_call" || ocinvokeDecl == "jl_fptr_const_return") { - if (pinvoke == nullptr) - ocinvokeDecl = get_or_emit_fptr1(preal_decl, mod)->getName(); - else - ocinvokeDecl = pinvoke->getName(); + else if (proto.specsig && !preal_specsig) { + // privatize our definition, since for some reason we couldn't use the external one but have an internal one + proto.decl->setLinkage(GlobalValue::PrivateLinkage); + preal_decl = ""; // no need to fixup the name } - assert(!ocinvokeDecl.empty()); - assert(ocinvokeDecl != "jl_fptr_args"); - assert(ocinvokeDecl != "jl_fptr_sparam"); - // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) { - if (proto.oc != specfun) - proto.oc->replaceAllUsesWith(specfun); + if (!preal_decl.empty()) { + // merge and/or rename this prototype to the real function + if (Function *specfun = cast_or_null(mod->getNamedValue(preal_decl))) { + if (proto.decl != specfun) { + proto.decl->replaceAllUsesWith(specfun); + if (!proto.decl->isDeclaration() && specfun->isDeclaration()) + linkFunctionBody(*specfun, *proto.decl); + proto.decl->eraseFromParent(); + proto.decl = specfun; + } + } + else { + proto.decl->setName(preal_decl); + } } - else { - proto.oc->setName(ocinvokeDecl); + if (proto.oc) { // additionally, if we are dealing with an OC constructor, then we might also need to fix up the fptr1 reference too + assert(proto.specsig); + StringRef ocinvokeDecl = invokeName; + if (invoke != nullptr && ocinvokeDecl.empty()) { + // check for some special tokens used by opaque_closure.c and convert those to their real functions + assert(invoke != jl_fptr_args_addr); + assert(invoke != jl_fptr_sparam_addr); + if (invoke == jl_fptr_interpret_call_addr) + ocinvokeDecl = "jl_fptr_interpret_call"; + else if (invoke == jl_fptr_const_return_addr) + ocinvokeDecl = "jl_fptr_const_return"; + else if (invoke == jl_f_opaque_closure_call_addr) + ocinvokeDecl = "jl_f_opaque_closure_call"; + //else if (invoke == jl_interpret_opaque_closure_addr) + else + ocinvokeDecl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + } + // if OC expected a specialized specsig dispatch, but we don't have it, use the inner trampoline here too + // XXX: this invoke translation logic is supposed to exactly match new_opaque_closure + if (!preal_specsig || ocinvokeDecl == "jl_f_opaque_closure_call" || ocinvokeDecl == "jl_fptr_interpret_call" || ocinvokeDecl == "jl_fptr_const_return") { + if (pinvoke == nullptr) + ocinvokeDecl = get_or_emit_fptr1(preal_decl, mod)->getName(); + else + ocinvokeDecl = pinvoke->getName(); + } + assert(!ocinvokeDecl.empty()); + assert(ocinvokeDecl != "jl_fptr_args"); + assert(ocinvokeDecl != "jl_fptr_sparam"); + // merge and/or rename this prototype to the real function + if (Function *specfun = cast_or_null(mod->getNamedValue(ocinvokeDecl))) { + if (proto.oc != specfun) { + proto.oc->replaceAllUsesWith(specfun); + proto.oc->eraseFromParent(); + proto.oc = specfun; + } + } + else { + proto.oc->setName(ocinvokeDecl); + } } } + else { + isedge = true; + params.workqueue.push_back(it); + incomplete_rgraph[codeinst].push_back(callee); + } + if (isedge) + complete_graph[callee].push_back(codeinst); } - else { - isedge = true; - params.workqueue.push_back(it); - incomplete_rgraph[codeinst].push_back(callee); - } - if (isedge) - complete_graph[callee].push_back(codeinst); } return params.workqueue.size(); } @@ -580,10 +598,11 @@ static void complete_emit(jl_code_instance_t *edge) JL_NOTSAFEPOINT_LEAVE JL_NOT auto ¶ms = std::get<0>(it->second); params.tsctx_lock = params.tsctx.getLock(); assert(callee == it->first); + orc::ThreadSafeModule &M = emittedmodules[callee]; + emit_always_inline(M, params); // may safepoint int waiting = jl_analyze_workqueue(callee, params); // may safepoint assert(!waiting); (void)waiting; - Module *M = emittedmodules[callee].getModuleUnlocked(); - finish_params(M, params, sharedmodules); + finish_params(M.getModuleUnlocked(), params, sharedmodules); incompletemodules.erase(it); } } @@ -796,6 +815,7 @@ void jl_emit_codeinst_to_jit_impl( invokenames[codeinst] = std::move(decls); complete_emit(codeinst); params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock + emit_always_inline(result_m, params); int waiting = jl_analyze_workqueue(codeinst, params); if (waiting) { auto release = std::move(params.tsctx_lock); // unlock again before moving from it @@ -1725,7 +1745,8 @@ struct JuliaOJIT::DLSymOptimizer { Thunk = cast(GV.getInitializer()->stripPointerCasts()); assert(++Thunk->uses().begin() == Thunk->uses().end() && "Thunk should only have one use in PLT initializer!"); assert(Thunk->hasLocalLinkage() && "Thunk should not have non-local linkage!"); - } else { + } + else { GV.setLinkage(GlobalValue::PrivateLinkage); } auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), GV.getValueType()); @@ -2298,125 +2319,6 @@ void JuliaOJIT::optimizeDLSyms(Module &M) { JuliaOJIT *jl_ExecutionEngine; -// destructively move the contents of src into dest -// this assumes that the targets of the two modules are the same -// including the DataLayout and ModuleFlags (for example) -// and that there is no module-level assembly -// Comdat is also removed, since the JIT doesn't need it -void jl_merge_module(orc::ThreadSafeModule &destTSM, orc::ThreadSafeModule srcTSM) -{ - ++ModulesMerged; - destTSM.withModuleDo([&](Module &dest) JL_NOTSAFEPOINT { - srcTSM.withModuleDo([&](Module &src) JL_NOTSAFEPOINT { - assert(&dest != &src && "Cannot merge module with itself!"); - assert(&dest.getContext() == &src.getContext() && "Cannot merge modules with different contexts!"); - assert(dest.getDataLayout() == src.getDataLayout() && "Cannot merge modules with different data layouts!"); - assert(dest.getTargetTriple() == src.getTargetTriple() && "Cannot merge modules with different target triples!"); - - for (auto &SG : make_early_inc_range(src.globals())) { - GlobalVariable *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - // Replace a declaration with the definition: - if (dG && !dG->hasLocalLinkage()) { - if (SG.isDeclaration()) { - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - //// If we start using llvm.used, we need to enable and test this - //else if (!dG->isDeclaration() && dG->hasAppendingLinkage() && SG.hasAppendingLinkage()) { - // auto *dCA = cast(dG->getInitializer()); - // auto *sCA = cast(SG.getInitializer()); - // SmallVector Init; - // for (auto &Op : dCA->operands()) - // Init.push_back(cast_or_null(Op)); - // for (auto &Op : sCA->operands()) - // Init.push_back(cast_or_null(Op)); - // ArrayType *ATy = ArrayType::get(PointerType::get(dest.getContext()), Init.size()); - // GlobalVariable *GV = new GlobalVariable(dest, ATy, dG->isConstant(), - // GlobalValue::AppendingLinkage, ConstantArray::get(ATy, Init), "", - // dG->getThreadLocalMode(), dG->getType()->getAddressSpace()); - // GV->copyAttributesFrom(dG); - // SG.replaceAllUsesWith(GV); - // dG->replaceAllUsesWith(GV); - // GV->takeName(SG); - // SG.eraseFromParent(); - // dG->eraseFromParent(); - // continue; - //} - else { - assert(dG->isDeclaration() || dG->getInitializer() == SG.getInitializer()); - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - // Reparent the global variable: - SG.removeFromParent(); - dest.insertGlobalVariable(&SG); - // Comdat is owned by the Module - SG.setComdat(nullptr); - } - - for (auto &SG : make_early_inc_range(src)) { - Function *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - // Replace a declaration with the definition: - if (dG && !dG->hasLocalLinkage()) { - if (SG.isDeclaration()) { - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - else { - assert(dG->isDeclaration()); - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - // Reparent the global variable: - SG.removeFromParent(); - dest.getFunctionList().push_back(&SG); - // Comdat is owned by the Module - SG.setComdat(nullptr); - } - - for (auto &SG : make_early_inc_range(src.aliases())) { - GlobalAlias *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - if (dG && !dG->hasLocalLinkage()) { - if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - else { - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - SG.removeFromParent(); - dest.insertAlias(&SG); - } - - // metadata nodes need to be explicitly merged not just copied - // so there are special passes here for each known type of metadata - NamedMDNode *sNMD = src.getNamedMetadata("llvm.dbg.cu"); - if (sNMD) { - NamedMDNode *dNMD = dest.getOrInsertNamedMetadata("llvm.dbg.cu"); - for (MDNode *I : sNMD->operands()) { - dNMD->addOperand(I); - } - } - }); - }); -} - //TargetMachine pass-through methods std::unique_ptr JuliaOJIT::cloneTargetMachine() const diff --git a/src/jitlayers.h b/src/jitlayers.h index b411febd792b8..619e5f3757642 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -72,7 +72,6 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeContext, LLVMOrcThreadSafeCont DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule, LLVMOrcThreadSafeModuleRef) void addTargetPasses(legacy::PassManagerBase *PM, const Triple &triple, TargetIRAnalysis analysis) JL_NOTSAFEPOINT; -void jl_merge_module(orc::ThreadSafeModule &dest, orc::ThreadSafeModule src) JL_NOTSAFEPOINT; GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M) JL_NOTSAFEPOINT; DataLayout jl_create_datalayout(TargetMachine &TM) JL_NOTSAFEPOINT; @@ -210,6 +209,12 @@ struct jl_codegen_call_target_t { llvm::Function *decl; llvm::Function *oc; bool specsig; + bool external_linkage; // whether codegen would like this edge to be externally-available + bool private_linkage; // whether codegen would like this edge to be internally-available + // external = ExternalLinkage (similar to "extern") + // private = InternalLinkage (similar to "static") + // external+private = AvailableExternallyLinkage+ExternalLinkage or ExternalLinkage (similar to "static inline") + // neither = unused }; // reification of a call to jl_jit_abi_convert, so that it isn't necessary to parse the Modules to recover this info @@ -231,7 +236,7 @@ struct jl_codegen_params_t { DataLayout DL; Triple TargetTriple; - inline LLVMContext &getContext() { + inline LLVMContext &getContext() JL_NOTSAFEPOINT { return *tsctx.getContext(); } typedef StringMap SymMapGV; @@ -268,6 +273,7 @@ struct jl_codegen_params_t { bool cache = false; bool external_linkage = false; bool imaging_mode; + bool safepoint_on_entry = true; bool use_swiftcc = true; jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER : tsctx(std::move(ctx)), @@ -305,6 +311,9 @@ jl_llvm_functions_t jl_emit_codedecls( jl_code_instance_t *codeinst, jl_codegen_params_t ¶ms); +void linkFunctionBody(Function &Dst, Function &Src) JL_NOTSAFEPOINT; +void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER; + enum CompilationPolicy { Default = 0, Extern = 1, @@ -660,8 +669,8 @@ class JuliaOJIT { OptSelLayerT OptSelLayer; }; extern JuliaOJIT *jl_ExecutionEngine; -std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT; -inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT { +std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT; +inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT { auto lock = ctx.getLock(); return orc::ThreadSafeModule(jl_create_llvm_module(name, *ctx.getContext(), DL, triple), ctx); } diff --git a/src/julia.expmap.in b/src/julia.expmap.in index b28a714e75f69..5a3fbce0d1a82 100644 --- a/src/julia.expmap.in +++ b/src/julia.expmap.in @@ -30,7 +30,6 @@ _Z22jl_coverage_alloc_lineN4llvm9StringRefEi*; _Z22jl_malloc_data_pointerN4llvm9StringRefEi*; _jl_timing_*; - LLVMExtra*; JLJIT*; llvmGetPassPluginInfo*; diff --git a/src/llvm-expand-atomic-modify.cpp b/src/llvm-expand-atomic-modify.cpp new file mode 100644 index 0000000000000..7b7b3c8761c17 --- /dev/null +++ b/src/llvm-expand-atomic-modify.cpp @@ -0,0 +1,473 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +// TODO: move this feature into AtomicExpandImpl + +#include "llvm-version.h" +#include "passes.h" + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "julia.h" +#include "julia_assert.h" + +#define DEBUG_TYPE "expand-atomic-modify" +#undef DEBUG + +using namespace llvm; + +// This pass takes fake call instructions that look like this which were emitted by the front end: +// (oldval, newval) = call atomicmodify.iN(ptr %op, ptr align(N) %ptr, i8 immarg %SSID, i8 immarg %Ordering, ...) !rmwattributes +// where op is a function with a prototype of `iN (iN arg, ...)` +// Then rewrite that to +// oldval = atomicrmw op ptr, val ordering syncscope +// newval = op oldval, val +// Or to an equivalent RMWCmpXchgLoop if `op` isn't valid for atomicrmw + + +// from AtomicExpandImpl, with modification of failure order and added Attributes +using CreateWeakCmpXchgInstFun = + std::function; + +static void createWeakCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr, + Value *Loaded, Value *NewVal, Align AddrAlign, + AtomicOrdering MemOpOrder, SyncScope::ID SSID, Instruction &Attributes, + Value *&Success, Value *&NewLoaded) { + Type *OrigTy = NewVal->getType(); + + // This code can go away when cmpxchg supports FP types. + assert(!OrigTy->isPointerTy()); + bool NeedBitcast = OrigTy->isFloatingPointTy(); + if (NeedBitcast) { + IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits()); + NewVal = Builder.CreateBitCast(NewVal, IntTy); + Loaded = Builder.CreateBitCast(Loaded, IntTy); + } + + AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( + Addr, Loaded, NewVal, AddrAlign, MemOpOrder, + AtomicOrdering::Monotonic, // why does LLVM use AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder) here + SSID); + Pair->copyMetadata(Attributes); + Success = Builder.CreateExtractValue(Pair, 1, "success"); + NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); + + if (NeedBitcast) + NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy); +} + +// from AtomicExpandImpl, with modification of values returned +std::pair insertRMWCmpXchgLoop( + IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign, + AtomicOrdering MemOpOrder, SyncScope::ID SSID, Instruction &Attributes, + const std::function &PerformOp, + const CreateWeakCmpXchgInstFun &CreateWeakCmpXchg) { + LLVMContext &Ctx = Builder.getContext(); + BasicBlock *BB = Builder.GetInsertBlock(); + Function *F = BB->getParent(); + + // Given: atomicrmw some_op iN* %addr, iN %incr ordering + // + // The standard expansion we produce is: + // [...] + // %init_loaded = load atomic iN* %addr + // br label %loop + // loop: + // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ] + // %new = some_op iN %loaded, %incr + // %pair = cmpxchg iN* %addr, iN %loaded, iN %new + // %new_loaded = extractvalue { iN, i1 } %pair, 0 + // %success = extractvalue { iN, i1 } %pair, 1 + // br i1 %success, label %atomicrmw.end, label %loop + // atomicrmw.end: + // [...] + BasicBlock *ExitBB = + BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end"); + BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); + + // The split call above "helpfully" added a branch at the end of BB (to the + // wrong place), but we want a load. It's easiest to just remove + // the branch entirely. + std::prev(BB->end())->eraseFromParent(); + Builder.SetInsertPoint(BB); + LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign); + InitLoaded->setOrdering(AtomicOrdering::Unordered); // n.b. the original LLVM pass is missing this call so is actually mildly UB + Builder.CreateBr(LoopBB); + + // Start the main loop block now that we've taken care of the preliminaries. + Builder.SetInsertPoint(LoopBB); + PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded"); + Loaded->addIncoming(InitLoaded, BB); + + Value *NewVal = PerformOp(Builder, Loaded); + + Value *NewLoaded = nullptr; + Value *Success = nullptr; + + CreateWeakCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign, + MemOpOrder == AtomicOrdering::Unordered + ? AtomicOrdering::Monotonic + : MemOpOrder, + SSID, Attributes, Success, NewLoaded); + assert(Success && NewLoaded); + + Loaded->addIncoming(NewLoaded, LoopBB); + + Builder.CreateCondBr(Success, ExitBB, LoopBB); + + Builder.SetInsertPoint(ExitBB, ExitBB->begin()); + return {NewLoaded, NewVal}; +} + +// from AtomicExpandImpl +struct ReplacementIRBuilder : IRBuilder { + // Preserves the DebugLoc from I, and preserves still valid metadata. + explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL) + : IRBuilder(I->getContext(), DL) { + SetInsertPoint(I); + this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections}); + } +}; + +// Must check that either Target cannot observe or mutate global state +// or that no trailing instructions does so either. +// Depending on the choice, it can also decide whether it is better to move Target after RMW +// or to move RMW before Target (or meet somewhere in the middle). +// Currently conservatively implemented as there being no instruction in the +// function which writes memory (which includes any atomics). +// Excluding the Target itself, unless some other instruction might read memory to observe it. +static bool canReorderWithRMW(Instruction &Target, bool verifyop) +{ + if (!verifyop) + return true; + Function &Op = *Target.getFunction(); + // quick check: if Op is nosync and Target doesn't access any memory, then reordering is trivially valid + bool nosync = Op.hasNoSync(); + if (nosync && !Target.mayReadOrWriteMemory()) + return true; + // otherwise, scan the whole function to see if any function accesses memory + // in a way that would conflict with reordering the atomic read and write + bool mayRead = false; + for (auto &BB : Op) { + for (auto &I : BB) { + if (&I == &Target) + continue; + if (I.mayWriteToMemory()) + return false; + if (!mayRead) { + mayRead = I.mayReadFromMemory(); + if (!nosync && mayRead) + return false; + } + } + } + // if any other instruction read memory, then the ordering of any writes by the target instruction might be observed + return !(mayRead && Target.mayWriteToMemory()); +} + +static std::variant patternMatchAtomicRMWOp(Value *Old, Use **ValOp, Value *RetVal) +{ + bool verifyop = RetVal == nullptr; + assert(verifyop ? isa(Old) : isa(Old)); + Function *Op = verifyop ? cast(Old)->getParent() : nullptr; + if (verifyop && (Op->isDeclaration() || Op->isInterposable() || Op->isIntrinsic())) + return false; + // TODO: peek forward from Old through any trivial casts which don't affect the instruction (e.g. i64 to f64 and back) + if (RetVal == nullptr) { + if (Old->use_empty()) { + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Xchg; + } + if (!Old->hasOneUse()) + return false; + ReturnInst *Ret = nullptr; + for (auto &BB : *Op) { + if (isa(BB.getTerminator())) { + if (Ret != nullptr) + return false; + Ret = cast(BB.getTerminator()); + } + } + if (Ret == nullptr) + return false; + // Now examine the instruction list + RetVal = Ret->getReturnValue(); + if (!RetVal->hasOneUse()) + return false; + } + if (RetVal == Old) { + // special token indicating to convert to an atomic fence + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Or; + } + if (Old->use_empty()) { + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Xchg; + } + if (auto BinOp = dyn_cast(RetVal)) { + if ((BinOp->getOperand(0) == Old || (BinOp->isCommutative() && BinOp->getOperand(1) == Old)) && canReorderWithRMW(*BinOp, verifyop)) { + if (ValOp) *ValOp = &BinOp->getOperandUse(BinOp->getOperand(0) == Old ? 1 : 0); + switch (BinOp->getOpcode()) { + case Instruction::Add: + return AtomicRMWInst::Add; + case Instruction::Sub: + return AtomicRMWInst::Sub; + case Instruction::And: + return AtomicRMWInst::And; + case Instruction::Or: + return AtomicRMWInst::Or; + case Instruction::Xor: + return AtomicRMWInst::Xor; + case Instruction::FAdd: + return AtomicRMWInst::FAdd; + case Instruction::FSub: + return AtomicRMWInst::FSub; + default: + break; + } + } + if (BinOp->getOpcode() == Instruction::Xor) { + if (auto CI = dyn_cast(BinOp->getOperand(1))) { + if (CI->isAllOnesValue()) { + BinOp = dyn_cast(BinOp->getOperand(0)); + if (BinOp && BinOp->hasOneUse() && BinOp->getOpcode() == Instruction::And) { + if ((BinOp->getOperand(0) == Old || (BinOp->isCommutative() && BinOp->getOperand(1) == Old)) && canReorderWithRMW(*BinOp, verifyop)) { + if (ValOp) *ValOp = &BinOp->getOperandUse(BinOp->getOperand(0) == Old ? 1 : 0); + return AtomicRMWInst::Nand; + } + } + } + } + } + return false; + } else if (auto Intr = dyn_cast(RetVal)) { + if (Intr->arg_size() == 2) { + if ((Intr->getOperand(0) == Old || (Intr->isCommutative() && Intr->getOperand(1) == Old)) && canReorderWithRMW(*Intr, verifyop)) { + if (ValOp) *ValOp = &Intr->getOperandUse(Intr->getOperand(0) == Old ? 1 : 0); + switch (Intr->getIntrinsicID()) { + case Intrinsic::minnum: + return AtomicRMWInst::FMin; + case Intrinsic::maxnum: + return AtomicRMWInst::FMax; + case Intrinsic::smax: + return AtomicRMWInst::Max; + case Intrinsic::umax: + return AtomicRMWInst::UMax; + case Intrinsic::smin: + return AtomicRMWInst::Min; + case Intrinsic::umin: + return AtomicRMWInst::UMin; +#if JL_LLVM_VERSION >= 200000 + case Intrinsic::usub_sat: + return AtomicRMWInst::USubSat; +#endif + } + } + } + return false; + } + else if (auto Intr = dyn_cast(RetVal)) { + // TODO: decide inlining cost of Op, or check alwaysinline/inlinehint, before this? + for (auto &Arg : Intr->args()) { + if (Arg == Old) { + if (canReorderWithRMW(*Intr, verifyop)) { + if (ValOp) *ValOp = &Arg; + return true; + } + return false; + } + } + } + // TODO: does this need to deal with F->hasFnAttribute(Attribute::StrictFP)? + // TODO: does Fneg and Neg have expansions? + // TODO: be able to ignore some simple bitcasts (particularly f64 to i64) + // TODO: handle longer sequences (UIncWrap, UDecWrap, USubCond, and target-specific ones for CUDA) + return false; +} + +void expandAtomicModifyToCmpXchg(CallInst &Modify, + const CreateWeakCmpXchgInstFun &CreateWeakCmpXchg) { + Value *Ptr = Modify.getOperand(0); + Function *Op = dyn_cast(Modify.getOperand(1)); + if (!Op) { + Modify.getParent()->getParent()->print(errs()); + llvm_unreachable("expected immarg for function argument"); + } + AtomicOrdering Ordering = (AtomicOrdering)cast(Modify.getOperand(2))->getZExtValue(); + SyncScope::ID SSID = (SyncScope::ID)cast(Modify.getOperand(3))->getZExtValue(); + MaybeAlign Alignment = Modify.getParamAlign(0); + unsigned user_arg_start = Modify.getFunctionType()->getNumParams(); + Type *Ty = Modify.getFunctionType()->getReturnType()->getStructElementType(0); + + ReplacementIRBuilder Builder(&Modify, Modify.getModule()->getDataLayout()); + Builder.setIsFPConstrained(Modify.hasFnAttr(Attribute::StrictFP)); + + CallInst *ModifyOp; + { + SmallVector Args(1 + Modify.arg_size() - user_arg_start); + Args[0] = UndefValue::get(Ty); // Undef used as placeholder for Loaded / RMW; + for (size_t argi = 0; argi < Modify.arg_size() - user_arg_start; ++argi) { + Args[argi + 1] = Modify.getArgOperand(argi + user_arg_start); + } + SmallVector Defs; + Modify.getOperandBundlesAsDefs(Defs); + ModifyOp = Builder.CreateCall(Op, Args, Defs); + ModifyOp->setCallingConv(Op->getCallingConv()); + } + Use *LoadedOp = &ModifyOp->getOperandUse(0); + + Value *OldVal = nullptr; + Value *NewVal = nullptr; + auto BinOp = patternMatchAtomicRMWOp(Op->getArg(0), nullptr, nullptr); + if (BinOp != decltype(BinOp)(false)) { + Builder.SetInsertPoint(ModifyOp); + AtomicRMWInst *RMW = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Ptr, UndefValue::get(Ty), Alignment, Ordering, SSID); // Undef used as placeholder + RMW->copyMetadata(Modify); + Builder.SetInsertPoint(&Modify); + LoadedOp->set(RMW); + for (int attempts = 0; ; ) { + FreezeInst *TrackReturn = Builder.Insert(new FreezeInst(ModifyOp)); // Create a temporary TrackingVH so we can recover the NewVal after inlining + InlineFunctionInfo IFI; + if (!InlineFunction(*ModifyOp, IFI).isSuccess()) { + // Undo the attempt, since inlining failed + BinOp = false; + TrackReturn->eraseFromParent(); + break; + } + ModifyOp = nullptr; + NewVal = TrackReturn->getOperand(0); + TrackReturn->eraseFromParent(); + // NewVal might have been folded away by inlining so redo patternMatchAtomicRMWOp here + // tracing from RMW to NewVal, in case instsimplify folded something + Use *ValOp; + BinOp = patternMatchAtomicRMWOp(RMW, &ValOp, NewVal); + if (BinOp == decltype(BinOp)(true)) { + ModifyOp = cast(ValOp->getUser()); + LoadedOp = ValOp; + assert(LoadedOp->get() == RMW); + RMW->moveBefore(ModifyOp); // NewValInst is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + BinOp = false; + if (++attempts > 3) + break; + if (auto FOp = ModifyOp->getCalledFunction()) + BinOp = patternMatchAtomicRMWOp(FOp->getArg(LoadedOp->getOperandNo()), nullptr, nullptr); + else + break; + if (BinOp == decltype(BinOp)(false)) + break; + } else { + assert(BinOp != decltype(BinOp)(true)); + auto RMWOp = std::get(BinOp); + assert(RMWOp != AtomicRMWInst::BAD_BINOP); + assert(isa(RMW->getOperand(1))); // RMW was previously being used as the placeholder for Val + Value *Val; + if (ValOp != nullptr) { + RMW->moveBefore(cast(ValOp->getUser())); // ValOp is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + Val = ValOp->get(); + } else if (RMWOp == AtomicRMWInst::Xchg) { + Val = NewVal; + } else { + // convert to an atomic fence of the form: atomicrmw or %ptr, 0 + assert(RMWOp == AtomicRMWInst::Or); + Val = ConstantInt::getNullValue(Ty); + } + RMW->setOperation(RMWOp); + RMW->setOperand(1, Val); + OldVal = RMW; + break; + } + } + if (BinOp == decltype(BinOp)(false)) { + LoadedOp->set(UndefValue::get(Ty)); + RMW->eraseFromParent(); + } + } + + if (BinOp == decltype(BinOp)(false)) { + // FIXME: If FP exceptions are observable, we should force them off for the + // loop for the FP atomics. + std::tie(OldVal, NewVal) = insertRMWCmpXchgLoop( + Builder, Ty, Ptr, *Alignment, Ordering, SSID, Modify, + [&](IRBuilderBase &Builder, Value *Loaded) JL_NOTSAFEPOINT { + LoadedOp->set(Loaded); + ModifyOp->moveBefore(*Builder.GetInsertBlock(), Builder.GetInsertPoint()); + return ModifyOp; + }, + CreateWeakCmpXchg); + } + + for (auto user : make_early_inc_range(Modify.users())) { + if (auto EV = dyn_cast(user)) { + if (EV->getNumIndices() == 1) { + if (EV->use_empty()) { + EV->eraseFromParent(); + continue; + } + else if (EV->getIndices()[0] == 0) { + EV->replaceAllUsesWith(OldVal); + EV->eraseFromParent(); + continue; + } else if (EV->getIndices()[0] == 1) { + EV->replaceAllUsesWith(NewVal); + EV->eraseFromParent(); + continue; + } + } + } + } + if (!Modify.use_empty()) { + auto OldNewVal = Builder.CreateInsertValue(UndefValue::get(Modify.getType()), OldVal, 0); + OldNewVal = Builder.CreateInsertValue(OldNewVal, NewVal, 1); + Modify.replaceAllUsesWith(OldNewVal); + } + Modify.eraseFromParent(); +} + +static bool expandAtomicModify(Function &F) { + SmallVector AtomicInsts; + + // Changing control-flow while iterating through it is a bad idea, so gather a + // list of all atomic instructions before we start. + for (Instruction &I : instructions(F)) + if (auto CI = dyn_cast(&I)) { + auto callee = dyn_cast_or_null(CI->getCalledOperand()); + if (callee && callee->getName().starts_with("julia.atomicmodify.")) { + assert(CI->getFunctionType() == callee->getFunctionType()); + AtomicInsts.push_back(CI); + } + } + + bool MadeChange = !AtomicInsts.empty(); + for (auto *I : AtomicInsts) + expandAtomicModifyToCmpXchg(*I, createWeakCmpXchgInstFun); + return MadeChange; +} + +PreservedAnalyses ExpandAtomicModifyPass::run(Function &F, FunctionAnalysisManager &AM) +{ + if (expandAtomicModify(F)) { + return PreservedAnalyses::none(); + } + return PreservedAnalyses::all(); +} diff --git a/src/llvm-julia-passes.inc b/src/llvm-julia-passes.inc index 0cc36f799db00..bd223499f37af 100644 --- a/src/llvm-julia-passes.inc +++ b/src/llvm-julia-passes.inc @@ -16,6 +16,7 @@ FUNCTION_PASS("AllocOpt", AllocOptPass()) FUNCTION_PASS("PropagateJuliaAddrspaces", PropagateJuliaAddrspacesPass()) FUNCTION_PASS("GCInvariantVerifier", GCInvariantVerifierPass()) FUNCTION_PASS("FinalLowerGC", FinalLowerGCPass()) +FUNCTION_PASS("ExpandAtomicModify", ExpandAtomicModifyPass()) #endif //Loop passes diff --git a/src/passes.h b/src/passes.h index 83721525d6f7e..0c5a124ade952 100644 --- a/src/passes.h +++ b/src/passes.h @@ -43,6 +43,11 @@ struct FinalLowerGCPass : PassInfoMixin { static bool isRequired() { return true; } }; +struct ExpandAtomicModifyPass : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) JL_NOTSAFEPOINT; +}; + + // Module Passes struct CPUFeaturesPass : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) JL_NOTSAFEPOINT; diff --git a/src/pipeline.cpp b/src/pipeline.cpp index eb93943653b34..f91db6fc037d7 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -574,6 +574,7 @@ static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder * FunctionPassManager FPM; JULIA_PASS(FPM.addPass(LateLowerGCPass())); JULIA_PASS(FPM.addPass(FinalLowerGCPass())); + JULIA_PASS(FPM.addPass(ExpandAtomicModifyPass())); // after LateLowerGCPass so that all IPO is valid MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native))); @@ -590,7 +591,8 @@ static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder * FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - } else if (!options.remove_ni) { + } + else if (!options.remove_ni) { JULIA_PASS(MPM.addPass(RemoveNIPass())); } MPM.addPass(AfterIntrinsicLoweringMarkerPass()); diff --git a/test/llvmpasses/atomic-modify.ll b/test/llvmpasses/atomic-modify.ll new file mode 100644 index 0000000000000..23e1949f3ad0a --- /dev/null +++ b/test/llvmpasses/atomic-modify.ll @@ -0,0 +1,288 @@ +; This file is a part of Julia. License is MIT: https://julialang.org/license + +; RUN: opt --load-pass-plugin=libjulia-codegen%shlibext -passes='ExpandAtomicModify' -S %s | FileCheck %s + +declare {i8, i8} @julia.atomicmodify.i8(ptr, ptr, i8, i8, ...) +declare {double, double} @julia.atomicmodify.f64(ptr, ptr, i8, i8, ...) +declare double @llvm.maxnum.f64(double %Val0, double %Val1) + +define i8 @add.i8(i8 %x, i8 %y) { + %z = add i8 %x, %y + ret i8 %z +} + +define i8 @sub.i8(i8 %x, i8 %y) { + %z = sub i8 %x, %y + ret i8 %z +} + +define i8 @subx.i8(i8 %x, i8 %y) { + %z = sub i8 %y, %x + ret i8 %z +} + +define i8 @add.i8.zext(i8 %x, i1 %y) { + %y8 = zext i1 %y to i8 + %z = add i8 %x, %y8 + ret i8 %z +} + +define i8 @and.i8(i8 %x, i8 %y) { + %z = and i8 %x, %y + ret i8 %z +} + +define i8 @nand.i8(i8 %x, i8 %y) { + %z = and i8 %x, %y + %w = xor i8 %z, -1 + ret i8 %w +} + +define i8 @nand.i8.zext(i8 %x, i1 %y) { + %y8 = zext i1 %y to i8 + %z = and i8 %y8, %x + %w = xor i8 %z, -1 + ret i8 %w +} + +define i8 @xchg.i8(i8 %x, i8 %y) { + ret i8 %y +} + +define double @fadd.f64(double %x, double %y) { + %z = fadd double %y, %x + ret double %z +} + +define double @fmax.f64(double %x, double %y) { + %z = call double @llvm.maxnum.f64(double %y, double %x) + ret double %z +} + +define internal i8 @0(i8 %x, i8 %y) unnamed_addr { + %z = call i8 @add.i8(i8 %x, i8 %y) + ret i8 %z +} + +define internal i8 @1(i8 %x, i8 %y) unnamed_addr { + %z = call i8 @0(i8 %x, i8 %y) + ret i8 %z +} + +define internal i8 @2(i8 %x, i8 %y, ptr %f) unnamed_addr { + %z = call i8 %f(i8 %x, i8 %y) + ret i8 %z +} + +define i8 @mod_i8_add(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_add +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_add +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK-NEXT: [[newval:%.*]] = add i8 %0, %b +; CHECK-NEXT: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_addfence(ptr %a) { +; CHECK-LABEL: @mod_i8_addfence +; CHECK: %0 = atomicrmw or ptr %a, i8 0 release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_zext(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_add_zext +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK: %0 = atomicrmw add ptr %a, i8 [[b8]] release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8.zext, i8 5, i8 1, i1 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_zext_new(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_add_zext +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK-NEXT: %0 = atomicrmw add ptr %a, i8 [[b8]] release, align 1 +; CHECK-NEXT: [[newval:%.*]] = add i8 %0, [[b8]] +; CHECK-NEXT: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8.zext, i8 5, i8 1, i1 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_sub(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_sub +; CHECK: %0 = atomicrmw sub ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @sub.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_subx(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_subx +; CHECK: [[newval:%.*]] = call i8 @subx.i8(i8 %loaded, i8 %b) +; CHECK: [[success:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: [[oldval:%.*]] = extractvalue { i8, i1 } [[success:%.*]], 0 +; CHECK: ret i8 [[oldval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @subx.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_subx_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_subx_new +; CHECK: [[newval:%.*]] = call i8 @subx.i8(i8 %loaded, i8 %b) +; CHECK: [[oldval:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @subx.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_nand(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_nand +; CHECK: %0 = atomicrmw nand ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @nand.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_nand_new(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_nand_new +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK: %0 = atomicrmw nand ptr %a, i8 [[b8]] release, align 1 +; CHECK: [[newand:%.*]] = and i8 [[b8]], %0 +; CHECK: [[newval:%.*]] = xor i8 [[newand:%.*]], -1 +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @nand.i8.zext, i8 5, i8 1, i1 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_andxchg(ptr %a) { +; CHECK-LABEL: @mod_i8_andxchg +; CHECK: %0 = atomicrmw xchg ptr %a, i8 0 release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @and.i8, i8 5, i8 1, i8 0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_xchg(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_xchg +; CHECK: %0 = atomicrmw xchg ptr %a, i8 %b release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @xchg.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_xchg_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_xchg_new +; CHECK: %0 = atomicrmw xchg ptr %a, i8 %b release, align 1 +; CHECK-NEXT: ret i8 %b +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @xchg.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define double @mod_i8_fadd(ptr %a, double %b) { +; CHECK-LABEL: @mod_i8_fadd +; CHECK: %0 = atomicrmw fadd ptr %a, double %b release, align 8 +; CHECK: ret double %0 +top: + %oldnew = call {double, double} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.f64(ptr align(8) %a, ptr @fadd.f64, i8 5, i8 1, double %b) + %oldval = extractvalue {double, double} %oldnew, 0 + ret double %oldval +} + +define double @mod_i8_fmax(ptr %a, double %b) { +; CHECK-LABEL: @mod_i8_fmax +; CHECK: %0 = atomicrmw fmax ptr %a, double %b release, align 8 +; CHECK: ret double %0 +top: + %oldnew = call {double, double} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.f64(ptr align(8) %a, ptr @fmax.f64, i8 5, i8 1, double %b) + %oldval = extractvalue {double, double} %oldnew, 0 + ret double %oldval +} + +define i8 @mod_i8_indirect0(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect0 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @0, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect1(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect1 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @1, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect2(ptr %a, i8 %b, ptr %f) { +; CHECK-LABEL: @mod_i8_indirect2 +; CHECK: [[newval:%.*]] = call i8 %f(i8 %loaded, i8 %b) +; CHECK: [[success:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: [[oldval:%.*]] = extractvalue { i8, i1 } [[success:%.*]], 0 +; CHECK: ret i8 [[oldval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr %f) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect2_new(ptr %a, i8 %b, ptr %f) { +; CHECK-LABEL: @mod_i8_indirect2_new +; CHECK: [[newval:%.*]] = call i8 %f(i8 %loaded, i8 %b) +; CHECK: [[oldval:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr %f) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_indirect3(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect3 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr @0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} From 5cfdf66d30f40f6a42f6f70ac522ac208d5dfc92 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 9 Jan 2025 20:25:52 +0000 Subject: [PATCH 011/662] remove deprecated Threads.Atomics --- base/atomics.jl | 176 +++++-------------------------------------- test/threads_exec.jl | 38 +++------- 2 files changed, 30 insertions(+), 184 deletions(-) diff --git a/base/atomics.jl b/base/atomics.jl index e6f3a5654cbf7..432c9120939ac 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -1,7 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Core.Intrinsics: llvmcall - import .Base: setindex!, getindex, unsafe_convert import .Base.Sys: ARCH, WORD_SIZE @@ -13,34 +11,6 @@ export atomic_and!, atomic_nand!, atomic_or!, atomic_xor!, atomic_max!, atomic_min!, atomic_fence -## -# Filter out unsupported atomic types on platforms -# - 128-bit atomics do not exist on AArch32. -# - Omitting 128-bit types on 32bit x86 and ppc64 -# - LLVM doesn't currently support atomics on floats for ppc64 -# C++20 is adding limited support for atomics on float, but as of -# now Clang does not support that yet. -if Sys.ARCH === :i686 || startswith(string(Sys.ARCH), "arm") || - Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - const inttypes = (Int8, Int16, Int32, Int64, - UInt8, UInt16, UInt32, UInt64) -else - const inttypes = (Int8, Int16, Int32, Int64, Int128, - UInt8, UInt16, UInt32, UInt64, UInt128) -end -const floattypes = (Float16, Float32, Float64) -const arithmetictypes = (inttypes..., floattypes...) -# TODO: Support Ptr -if Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - const atomictypes = (inttypes..., Bool) -else - const atomictypes = (arithmetictypes..., Bool) -end - -const IntTypes = Union{inttypes...} -const FloatTypes = Union{floattypes...} -const ArithmeticTypes = Union{arithmetictypes...} -const AtomicTypes = Union{atomictypes...} """ Threads.Atomic{T} @@ -48,10 +18,6 @@ const AtomicTypes = Union{atomictypes...} Holds a reference to an object of type `T`, ensuring that it is only accessed atomically, i.e. in a thread-safe manner. -Only certain "simple" types can be used atomically, namely the -primitive boolean, integer, and float-point types. These are `Bool`, -`Int8`...`Int128`, `UInt8`...`UInt128`, and `Float16`...`Float64`. - New atomic objects can be created from a non-atomic values; if none is specified, the atomic object is initialized with zero. @@ -72,10 +38,10 @@ julia> x[] Atomic operations use an `atomic_` prefix, such as [`atomic_add!`](@ref), [`atomic_xchg!`](@ref), etc. """ -mutable struct Atomic{T<:AtomicTypes} - value::T - Atomic{T}() where {T<:AtomicTypes} = new(zero(T)) - Atomic{T}(value) where {T<:AtomicTypes} = new(value) +mutable struct Atomic{T} + @atomic value::T + Atomic{T}() where {T} = new(zero(T)) + Atomic{T}(value) where {T} = new(value) end Atomic() = Atomic{Int}() @@ -332,120 +298,21 @@ julia> x[] """ function atomic_min! end -unsafe_convert(::Type{Ptr{T}}, x::Atomic{T}) where {T} = convert(Ptr{T}, pointer_from_objref(x)) -setindex!(x::Atomic{T}, v) where {T} = setindex!(x, convert(T, v)) - -const llvmtypes = IdDict{Any,String}( - Bool => "i8", # julia represents bools with 8-bits for now. # TODO: is this okay? - Int8 => "i8", UInt8 => "i8", - Int16 => "i16", UInt16 => "i16", - Int32 => "i32", UInt32 => "i32", - Int64 => "i64", UInt64 => "i64", - Int128 => "i128", UInt128 => "i128", - Float16 => "half", - Float32 => "float", - Float64 => "double", -) -inttype(::Type{T}) where {T<:Integer} = T -inttype(::Type{Float16}) = Int16 -inttype(::Type{Float32}) = Int32 -inttype(::Type{Float64}) = Int64 - - -import ..Base.gc_alignment - -# All atomic operations have acquire and/or release semantics, depending on -# whether the load or store values. Most of the time, this is what one wants -# anyway, and it's only moderately expensive on most hardware. -for typ in atomictypes - lt = llvmtypes[typ] - ilt = llvmtypes[inttype(typ)] - rt = "$lt, $lt*" - irt = "$ilt, $ilt*" - @eval getindex(x::Atomic{$typ}) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ)) - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) - @eval setindex!(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ)) - ret void - """, Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - - # Note: atomic_cas! succeeded (i.e. it stored "new") if and only if the result is "cmp" - if typ <: Integer - @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire - %rv = extractvalue { $lt, i1 } %rs, 0 - ret $lt %rv - """, $typ, Tuple{Ptr{$typ},$typ,$typ}, - unsafe_convert(Ptr{$typ}, x), cmp, new) - else - @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - GC.@preserve x llvmcall($""" - %iptr = bitcast i8* %0 to $ilt* - %icmp = bitcast $lt %1 to $ilt - %inew = bitcast $lt %2 to $ilt - %irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire - %irv = extractvalue { $ilt, i1 } %irs, 0 - %rv = bitcast $ilt %irv to $lt - ret $lt %rv - """, $typ, Tuple{Ptr{$typ},$typ,$typ}, - unsafe_convert(Ptr{$typ}, x), cmp, new) - end - - arithmetic_ops = [:add, :sub] - for rmwop in [arithmetic_ops..., :xchg, :and, :nand, :or, :xor, :max, :min] - rmw = string(rmwop) - fn = Symbol("atomic_", rmw, "!") - if (rmw == "max" || rmw == "min") && typ <: Unsigned - # LLVM distinguishes signedness in the operation, not the integer type. - rmw = "u" * rmw - end - if rmwop in arithmetic_ops && !(typ <: ArithmeticTypes) continue end - if typ <: Integer - @eval $fn(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - else - rmwop === :xchg || continue - @eval $fn(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %iptr = bitcast i8* %0 to $ilt* - %ival = bitcast $lt %1 to $ilt - %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel - %rv = bitcast $ilt %irv to $lt - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - end - end -end - -# Provide atomic floating-point operations via atomic_cas! -const opnames = Dict{Symbol, Symbol}(:+ => :add, :- => :sub) -for op in [:+, :-, :max, :min] - opname = get(opnames, op, op) - @eval function $(Symbol("atomic_", opname, "!"))(var::Atomic{T}, val::T) where T<:FloatTypes - IT = inttype(T) - old = var[] - while true - new = $op(old, val) - cmp = old - old = atomic_cas!(var, cmp, new) - reinterpret(IT, old) == reinterpret(IT, cmp) && return old - # Temporary solution before we have gc transition support in codegen. - ccall(:jl_gc_safepoint, Cvoid, ()) - end - end -end +#const nand = (~) ∘ (&) # ComposedFunction generated very poor code quality +nand(x, y) = ~(x & y) + +getindex(x::Atomic) = @atomic :acquire x.value +setindex!(x::Atomic, v) = (@atomic :release x.value = v; x) +atomic_cas!(x::Atomic, cmp, new) = (@atomicreplace :acquire_release :acquire x.value cmp => new).old +atomic_add!(x::Atomic, v) = (@atomic :acquire_release x.value + v).first +atomic_sub!(x::Atomic, v) = (@atomic :acquire_release x.value - v).first +atomic_and!(x::Atomic, v) = (@atomic :acquire_release x.value & v).first +atomic_or!(x::Atomic, v) = (@atomic :acquire_release x.value | v).first +atomic_xor!(x::Atomic, v) = (@atomic :acquire_release x.value ⊻ v).first +atomic_nand!(x::Atomic, v) = (@atomic :acquire_release x.value nand v).first +atomic_xchg!(x::Atomic, v) = (@atomicswap :acquire_release x.value = v) +atomic_min!(x::Atomic, v) = (@atomic :acquire_release x.value min v).first +atomic_max!(x::Atomic, v) = (@atomic :acquire_release x.value max v).first """ Threads.atomic_fence() @@ -462,7 +329,4 @@ fences should not be necessary in most cases. For further details, see LLVM's `fence` instruction. """ -atomic_fence() = llvmcall(""" - fence seq_cst - ret void - """, Cvoid, Tuple{}) +atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent) diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 629f474f53a38..dc0bc407d2fb5 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -334,29 +334,12 @@ using Base.Threads end end -# Ensure only LLVM-supported types can be atomic -@test_throws TypeError Atomic{BigInt} -@test_throws TypeError Atomic{ComplexF64} - -if Sys.ARCH === :i686 || startswith(string(Sys.ARCH), "arm") || - Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - - @test_throws TypeError Atomic{Int128}() - @test_throws TypeError Atomic{UInt128}() -end - -if Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - @test_throws TypeError Atomic{Float16}() - @test_throws TypeError Atomic{Float32}() - @test_throws TypeError Atomic{Float64}() -end - function test_atomic_bools() x = Atomic{Bool}(false) - # Arithmetic functions are not defined. - @test_throws MethodError atomic_add!(x, true) - @test_throws MethodError atomic_sub!(x, true) - # All the rest are: + # Arithmetic functions such as true+true returns Int + @test_throws TypeError atomic_add!(x, true) + @test_throws TypeError atomic_sub!(x, true) + # All the rest are supported: for v in [true, false] @test x[] == atomic_xchg!(x, v) @test v == atomic_cas!(x, v, !v) @@ -462,10 +445,9 @@ end test_fence() # Test load / store with various types -let atomictypes = intersect((Int8, Int16, Int32, Int64, Int128, - UInt8, UInt16, UInt32, UInt64, UInt128, - Float16, Float32, Float64), - Base.Threads.atomictypes) +let atomictypes = (Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128, + Float16, Float32, Float64) for T in atomictypes var = Atomic{T}() var[] = 42 @@ -493,7 +475,7 @@ function test_atomic_cas!(var::Atomic{T}, range::StepRange{Int,Int}) where T end end end -for T in intersect((Int32, Int64, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float32, Float64) var = Atomic{T}() nloops = 1000 di = threadpoolsize(:default) @@ -507,7 +489,7 @@ function test_atomic_xchg!(var::Atomic{T}, i::Int, accum::Atomic{Int}) where T old = atomic_xchg!(var, T(i)) atomic_add!(accum, Int(old)) end -for T in intersect((Int32, Int64, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float32, Float64) accum = Atomic{Int}() var = Atomic{T}() nloops = 1000 @@ -522,7 +504,7 @@ function test_atomic_float(varadd::Atomic{T}, varmax::Atomic{T}, varmin::Atomic{ atomic_max!(varmax, T(i)) atomic_min!(varmin, T(i)) end -for T in intersect((Int32, Int64, Float16, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float16, Float32, Float64) varadd = Atomic{T}() varmax = Atomic{T}() varmin = Atomic{T}() From 2c7527b1aa27b8bf523af95289767b66eac5796e Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Fri, 28 Mar 2025 11:36:50 -0300 Subject: [PATCH 012/662] Teach alloc-opt to handle atomics a bit better (#57208) Fixes https://github.com/JuliaLang/julia/issues/57190 The fact that this passes thinks memcpy is a potential issue is quite annoying so it deserves a decent refactor, which flows through the type information from julia instead of trying to regenerate it on site, specially given that opaque pointers means we can't really instrospect into pointers at all --- src/llvm-alloc-helpers.cpp | 1 + src/llvm-alloc-opt.cpp | 5 +- test/atomics.jl | 11 ++ test/llvmpasses/alloc-opt-pass.ll | 180 +++++++++++++++++++++++++----- 4 files changed, 171 insertions(+), 26 deletions(-) diff --git a/src/llvm-alloc-helpers.cpp b/src/llvm-alloc-helpers.cpp index 194c6837860ca..a1ed66a190190 100644 --- a/src/llvm-alloc-helpers.cpp +++ b/src/llvm-alloc-helpers.cpp @@ -214,6 +214,7 @@ void jl_alloc::runEscapeAnalysis(llvm::CallInst *I, EscapeAnalysisRequiredArgs r } if (auto call = dyn_cast(inst)) { // TODO handle `memcmp` + // TODO handle `memcpy` which is used a lot more often since opaque pointers // None of the intrinsics should care if the memory is stack or heap allocated. auto callee = call->getCalledOperand(); if (auto II = dyn_cast(call)) { diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index bfc1b42444cd1..56bb1ab7c706b 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -758,7 +758,9 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref, AllocF auto replace_inst = [&] (Instruction *user) { Instruction *orig_i = cur.orig_i; Instruction *new_i = cur.new_i; - if (isa(user) || isa(user)) { + if (isa(user) || isa(user) || + isa(user) || isa(user)) { + // TODO: these atomics are likely removable if the user is the first argument user->replaceUsesOfWith(orig_i, new_i); } else if (auto call = dyn_cast(user)) { @@ -1131,6 +1133,7 @@ void Optimizer::splitOnStack(CallInst *orig_inst) return; } else if (isa(user) || isa(user)) { + // TODO: Downgrade atomics here potentially auto slot_idx = find_slot(offset); auto &slot = slots[slot_idx]; assert(slot.offset <= offset && slot.offset + slot.size >= offset); diff --git a/test/atomics.jl b/test/atomics.jl index 7e9f29c23ca10..2d4a713b1d30d 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -1099,3 +1099,14 @@ test_once_undef(Any) test_once_undef(Union{Nothing,Integer}) test_once_undef(UndefComplex{Any}) test_once_undef(UndefComplex{UndefComplex{Any}}) + +mutable struct Atomic57190 + @atomic x::Int +end + + +function add_one57190!() + @atomic (Atomic57190(0).x) += 1 +end + +@test add_one57190!() == 1 diff --git a/test/llvmpasses/alloc-opt-pass.ll b/test/llvmpasses/alloc-opt-pass.ll index 665687e86835d..83f2118412cc1 100644 --- a/test/llvmpasses/alloc-opt-pass.ll +++ b/test/llvmpasses/alloc-opt-pass.ll @@ -73,6 +73,11 @@ L3: ; preds = %L2, %L1, %0 } ; CHECK-LABEL: }{{$}} +declare void @external_function() + +declare ptr addrspace(10) @external_function2() + + ; CHECK-LABEL: @legal_int_types ; CHECK: alloca [12 x i8] ; CHECK-NOT: alloca i96 @@ -89,21 +94,6 @@ define void @legal_int_types() { } ; CHECK-LABEL: }{{$}} -declare void @external_function() - -declare ptr addrspace(10) @external_function2() - -declare ptr @julia.ptls_states() - -declare ptr @julia.get_pgcstack() - -declare noalias ptr addrspace(10) @julia.gc_alloc_obj(ptr, i64, ptr addrspace(10)) - -declare ptr @julia.pointer_from_objref(ptr addrspace(11)) - -declare token @llvm.julia.gc_preserve_begin(...) - -declare void @llvm.julia.gc_preserve_end(token) ; CHECK-LABEL: @memref_collision ; OPAQUE: call ptr @julia.ptls_states() @@ -171,13 +161,13 @@ define void @initializers() { %pgcstack = call ptr @julia.get_pgcstack() %ptls = call ptr @julia.ptls_states() %ptls_i8 = bitcast ptr %ptls to ptr - %var1 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 1, ptr addrspace(10) @tag) #1 + %var1 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 1, ptr addrspace(10) @tag) #4 %var2 = addrspacecast ptr addrspace(10) %var1 to ptr addrspace(11) %var3 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var2) - %var4 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 2, ptr addrspace(10) @tag) #2 + %var4 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 2, ptr addrspace(10) @tag) #7 %var5 = addrspacecast ptr addrspace(10) %var4 to ptr addrspace(11) %var6 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var5) - %var7 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 3, ptr addrspace(10) @tag) #3 + %var7 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 3, ptr addrspace(10) @tag) #1 %var8 = addrspacecast ptr addrspace(10) %var7 to ptr addrspace(11) %var9 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var8) ret void @@ -203,14 +193,154 @@ union_move9: ; No predecessors! } ; CHECK-LABEL: }{{$}} +@0 = private unnamed_addr constant ptr inttoptr (i64 4373799056 to ptr), !julia.constgv !0 +@1 = private unnamed_addr constant i64 0, align 8 + +; CHECK-LABEL: @cmpxchg +; CHECK: alloca +; CHECK: alloca +; CHECK: %20 = cmpxchg ptr %2, +define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself %0) #0 { + %2 = alloca i64, align 16 + %3 = call ptr @julia.get_pgcstack() + %4 = getelementptr inbounds i8, ptr %3, i32 -152 + %5 = getelementptr inbounds i8, ptr %4, i32 168 + %6 = load ptr, ptr %5, align 8, !tbaa !4 + %7 = getelementptr inbounds i8, ptr %6, i32 16 + %8 = load ptr, ptr %7, align 8, !tbaa !8, !invariant.load !0 + fence syncscope("singlethread") seq_cst + call void @julia.safepoint(ptr %8) + fence syncscope("singlethread") seq_cst + %9 = load ptr, ptr @0, align 8, !tbaa !8, !invariant.load !0, !alias.scope !10, !noalias !13, !nonnull !0, !dereferenceable !18, !align !19 + %10 = ptrtoint ptr %9 to i64 + %11 = inttoptr i64 %10 to ptr + %12 = getelementptr inbounds i8, ptr %3, i32 -152 + %13 = addrspacecast ptr %11 to ptr addrspace(10) + call void @llvm.lifetime.start.p0(i64 8, ptr %2) + %14 = call noalias nonnull align 8 dereferenceable(8) ptr addrspace(10) @julia.gc_alloc_obj(ptr %12, i64 8, ptr addrspace(10) %13) #7 + %15 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + call void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) align 8 %15, ptr align 8 @1, i64 8, i1 false), !tbaa !20, !alias.scope !23, !noalias !24 + %16 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + %17 = load atomic i64, ptr addrspace(11) %16 monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + br label %19 + +18: ; preds = %19 + ret i64 %21 + +19: ; preds = %19, %1 + %20 = phi i64 [ %17, %1 ], [ %23, %19 ] + %21 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself %3, i64 signext %20, i64 signext 1) + %22 = cmpxchg ptr addrspace(11) %16, i64 %20, i64 %21 seq_cst monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + %23 = extractvalue { i64, i1 } %22, 0 + %24 = extractvalue { i64, i1 } %22, 1 + br i1 %24, label %18, label %19 +} + +; CHECK-LABEL: }{{$}} +; CHECK-LABEL: @atomicrmw +; CHECK: alloca +; CHECK: alloca +; CHECK: atomicrmw xchg ptr %2, +define swiftcc i64 @"atomicrmw"(ptr nonnull swiftself %0) #0 { + %2 = alloca i64, align 16 + %3 = call ptr @julia.get_pgcstack() + %4 = getelementptr inbounds i8, ptr %3, i32 -152 + %5 = getelementptr inbounds i8, ptr %4, i32 168 + %6 = load ptr, ptr %5, align 8, !tbaa !4 + %7 = getelementptr inbounds i8, ptr %6, i32 16 + %8 = load ptr, ptr %7, align 8, !tbaa !8, !invariant.load !0 + fence syncscope("singlethread") seq_cst + call void @julia.safepoint(ptr %8) + fence syncscope("singlethread") seq_cst + %9 = load ptr, ptr @0, align 8, !tbaa !8, !invariant.load !0, !alias.scope !10, !noalias !13, !nonnull !0, !dereferenceable !18, !align !19 + %10 = ptrtoint ptr %9 to i64 + %11 = inttoptr i64 %10 to ptr + %12 = getelementptr inbounds i8, ptr %3, i32 -152 + %13 = addrspacecast ptr %11 to ptr addrspace(10) + call void @llvm.lifetime.start.p0(i64 8, ptr %2) + %14 = call noalias nonnull align 8 dereferenceable(8) ptr addrspace(10) @julia.gc_alloc_obj(ptr %12, i64 8, ptr addrspace(10) %13) #7 + %15 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + call void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) align 8 %15, ptr align 8 @1, i64 8, i1 false), !tbaa !20, !alias.scope !23, !noalias !24 + %16 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + %17 = load atomic i64, ptr addrspace(11) %16 monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + %18 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself %3, i64 signext %17, i64 signext 1) + %19 = atomicrmw xchg ptr addrspace(11) %16, i64 %18 seq_cst, align 8, !tbaa !25, !alias.scope !23, !noalias !24 ; preds = %19 + ret i64 %19 +} + +declare ptr @julia.ptls_states() + +declare ptr @julia.pointer_from_objref(ptr addrspace(11)) + +declare token @llvm.julia.gc_preserve_begin(...) + +declare void @llvm.julia.gc_preserve_end(token) + +declare ptr @julia.get_pgcstack() + +; Function Attrs: mustprogress nounwind willreturn memory(inaccessiblemem: readwrite) +declare nonnull align 8 dereferenceable(8) ptr addrspace(10) @ijl_box_int64(i64 signext) #2 + +; Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite) +declare void @julia.safepoint(ptr) #3 + +; Function Attrs: mustprogress nounwind willreturn allockind("alloc") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) +declare noalias nonnull ptr addrspace(10) @julia.gc_alloc_obj(ptr, i64, ptr addrspace(10)) #4 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #5 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p0.p11.i64(ptr noalias nocapture writeonly, ptr addrspace(11) noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p0.p11.i64(ptr noalias nocapture writeonly, ptr addrspace(11) noalias nocapture readonly, i64, i1 immarg) #5 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #5 + +declare swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself, i64 signext, i64 signext) #0 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #6 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #6 + +attributes #0 = { "probe-stack"="inline-asm" } +attributes #1 = { nounwind willreturn allockind("alloc,zeroed") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #2 = { mustprogress nounwind willreturn memory(inaccessiblemem: readwrite) } +attributes #3 = { memory(argmem: readwrite, inaccessiblemem: readwrite) } +attributes #4 = { mustprogress nounwind willreturn allockind("alloc") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #5 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } +attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #7 = { nounwind willreturn allockind("alloc,uninitialized") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #8 = { nounwind willreturn memory(inaccessiblemem: readwrite) } + +!llvm.module.flags = !{!1, !2, !3} + +!0 = !{} +!1 = !{i32 2, !"Dwarf Version", i32 4} +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 2, !"julia.optlevel", i32 2} +!4 = !{!5, !5, i64 0} +!5 = !{!"jtbaa_gcframe", !6, i64 0} +!6 = !{!"jtbaa", !7, i64 0} +!7 = !{!"jtbaa"} +!8 = !{!9, !9, i64 0, i64 1} +!9 = !{!"jtbaa_const", !6, i64 0} +!10 = !{!11} +!11 = !{!"jnoalias_const", !12} +!12 = !{!"jnoalias"} +!13 = !{!14, !15, !16, !17} +!14 = !{!"jnoalias_gcframe", !12} +!15 = !{!"jnoalias_stack", !12} +!16 = !{!"jnoalias_data", !12} +!17 = !{!"jnoalias_typemd", !12} +!18 = !{i64 56} +!19 = !{i64 16} +!20 = !{!21, !21, i64 0} +!21 = !{!"jtbaa_value", !22, i64 0} +!22 = !{!"jtbaa_data", !6, i64 0} +!23 = !{!16} +!24 = !{!14, !15, !17, !11} +!25 = !{!26, !26, i64 0} +!26 = !{!"jtbaa_mutab", !21, i64 0} -attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } -attributes #1 = { allockind("alloc") } -attributes #2 = { allockind("alloc,uninitialized") } -attributes #3 = { allockind("alloc,zeroed") } From 54197132f530bd7e7dbc84cfe56daa0b43bcfb0c Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Fri, 28 Mar 2025 19:07:23 +0100 Subject: [PATCH 013/662] Refactor IOBuffer code (#57570) I've been a little frustrated with the IOBuffer code. It contains a whole bunch of implicit invariants, and is poorly commented. It also has several bugs that ultimately stems from the code being unclear about its own assumptions. This is a refactoring of IOBuffer. The primary goals are: * Comment the code more heavily * Test the code more thoroughly The secondary goals are * Fix a few outstanding bugs * Add some minor performance improvements This is a purely internal refactoring with be no change in behaviour of `IOBuffer`, except straight up bugfixes. However, note that previous code may have relied on buggy behaviour. Fixing bugs may therefore cause breakage. ## Current changes ### **BEHAVIOUR CHANGES** * The following code used to not throw an error, but now does: `IOBuffer(b"abc"; maxsize=2)`. I consider this a bugfix. It should not be possible to construct an IOBuffer with a buffersize larger than `maxsize`. * It used to be possible to write to indices higher than `maxindex`, which could trigger a bug causing data loss. The bug has been fixed, but as a result, some IOBuffers may reach full capacity faster (really: reach it at the correct point), changing writing behaviour. ### Bugfixes * Do not corrupt data on `copyline` on a non-appending buffer * Respect `maxsize` even after `take!` (fix #57549) * Fix bug when copying from an appending iobuffer to itself * Fix bug where re-allocating the buffer may cause it to shrink, discarding data. * Fix bug where `truncate` may throw a wrong BoundsError * Fix a bug where truncating a buffer may not correctly removed mark at position that has been deleted * Fix a bug where initializing an IOBuffer without an explicit buffer and with `truncate=false` makes it contain the full buffer * Current behaviour for `reset` and `position` did not work for `PipeBuffer`. Fix that ### Changes to brittle code * Removed some tests that explicitly tested internal code and internal behaviour. Some of that behaviour has changed. * Changed some internal `PipeBuffer` behaviour, which did not respect writable IOBuffer's ownership of their buffer and therefore failed spuriously ### Performance improvements * Writing to dense IOBuffer now uses memmove and is up to 10x faster for long writes. * Minor optimisations (about ten percent) for writing to IOBuffers in general Closes #57549 Co-authored-by: Jameson Nash --- base/iobuffer.jl | 770 +++++++++++++++++++++++++++++++++-------------- base/stream.jl | 13 +- test/iobuffer.jl | 450 ++++++++++++++++++++++----- 3 files changed, 925 insertions(+), 308 deletions(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index ed0cd63183dc1..49bc25d10780d 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -1,45 +1,168 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## work with AbstractVector{UInt8} via I/O primitives ## +# IOBuffer is a Memory{UInt8} backed IO type for in-memory IO. + +# Here, u represents used bytes (already read), X represents bytes still to read, +# - represents bytes uninitialized data but which can be written to later. +# . represents bytes before offset, which the buffer will not touch, until +# a write operation happens. + +# .....uuuuuuuuuuuuuXXXXXXXXXXXXX------------ +# | | | | | | +# | offset ptr size | maxsize +# 1 lastindex(data) + +# N.B: `mark` does not correspond to any index in the buffer. Instead, it stores +# the mark at virtual offset in the buffer. + +# AFTER COMPACTION + +# XXXXXXXXXXXXX-------------------------- +# || | | | | +# |1 ptr size | maxsize +# | lastindex(data) +# offset (set to zero) + +# * The underlying array is always 1-indexed +# * The IOBuffer has full control (ownership) of the underlying array, only when +# buffer.write == true. +# * Unreachable data can be deleted in the buffer's data, shifting the whole thing to the left +# to make room for more data, without replacing or resizing data. +# This can be done only if the buffer is not seekable -# Stateful string mutable struct GenericIOBuffer{T<:AbstractVector{UInt8}} <: IO - data::T # T should support: getindex, setindex!, length, copyto!, similar, and (optionally) resize! - reinit::Bool # if true, data needs to be re-allocated (after take!) + # T should support: getindex, setindex!, length, copyto!, similar, size and (optionally) resize! + data::T + + # The user can take control of `data` out of this struct. When that happens, instead of eagerly allocating + # a new array, we set `.reinit` to true, and then allocate a new one when needed. + # If reinit is true, the buffer is writable, and offset_or_compacted and size is zero. See `take!` + reinit::Bool readable::Bool writable::Bool - seekable::Bool # if not seekable, implementation is free to destroy (compact) past read data - append::Bool # add data at end instead of at pointer - size::Int # end pointer (and write pointer if append == true) + offset - maxsize::Int # fixed array size (typically pre-allocated) - ptr::Int # read (and maybe write) pointer + offset - offset::Int # offset of ptr and size from actual start of data and actual size - mark::Int # reset mark location for ptr (or <0 for no mark) - function GenericIOBuffer{T}(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) where T<:AbstractVector{UInt8} - require_one_based_indexing(data) - return new(data, false, readable, writable, seekable, append, length(data), maxsize, 1, 0, -1) - end + # If not seekable, implementation is free to destroy (compact) data before ptr, unless + # it can be recovered using the mark by using `reset`. + # If it IS seekable, the user may always recover any data in 1:size by seeking, + # so no data can be destroyed. + # Non-seekable IOBuffers can only be constructed with `PipeBuffer`, which are writable, + # readable and append. + seekable::Bool + + # If true, write new data to the index size+1 instead of the index ptr. + append::Bool + + # Last index of `data` that has been written to. Data in size+1:end has not yet been used, + # and may contain arbitrary values. + # This value is always in 0 : lastindex(data) + size::Int + + # When the buffer is resized, or a new buffer allocated, this is the maximum size of the buffer. + # A new GenericIOBuffer may be constructed with an existing data larger than `maxsize`. + # When that happensm we must make sure to not have more than `maxsize` bytes in the buffer, + # else reallocating will lose data. So, never write to indices > `maxsize + get_offset(io)` + # This value is always in 0:typemax(Int). + maxsize::Int + + # Data is read/written from/to ptr, except in situations where append is true, in which case + # data is still read from ptr, but written to size+1. + # This value is always in offset + 1 : size+1 + ptr::Int + + # This field has two distinct meanings: + # If the value is positive, it encodes an offset of the start of the data in `data`. + # This is used if the buffer is instantiated from a Vector with non-zero memory offset. + # Then, the IOBuffer stores the underlying memory, and so the first data in the buffer + # is not at index 1. + # If the value is negative, then `-io.offset_or_compacted` gets the number of compacted + # bytes. That's the number of unused bytes deleted from a non-seekable stream to make space. + # We need to keep track of it in order to make `mark` and `position` etc work, that is, + # we need to know the virtual position of the mark even when an arbitrary number + # of unused bytes has been deleted due to compaction. + # Since compaction will move data in the buffer and thereby zero the offset, either the + # offset or the number of compacted bytes will be zero at any point, so both can be + # stored in one field. + # If offset: Value is always in 0:lastindex(data) + # If compacted: Value is in typemin(Int):0 + offset_or_compacted::Int + + # The mark is -1 if not set, else the zero-indexed virtual position of ptr in the buffer. + # Due to compaction and offset, this value is not an index into the buffer, but may be translated + # to an index. + # This value is in -1:typemax(Int) + mark::Int + + # Unsafe constructor which does not do any checking + global function _new_generic_iobuffer( + ::Type{T}, + data::T, + readable::Bool, + writable::Bool, + seekable::Bool, + append::Bool, + maxsize::Int, + ) where T<:AbstractVector{UInt8} + len = Int(length(data))::Int + return new{T}(data, false, readable, writable, seekable, append, len, maxsize, 1, 0, -1) + end +end + +function GenericIOBuffer{T}( + data::T, + readable::Bool, + writable::Bool, + seekable::Bool, + append::Bool, + maxsize::Integer, + truncate::Bool, + ) where T<:AbstractVector{UInt8} + require_one_based_indexing(data) + mz = Int(maxsize)::Int + len = Int(length(data))::Int + if !truncate && mz < len + throw(ArgumentError("maxsize must not be smaller than data length")) + end + buf = _new_generic_iobuffer(T, data, readable, writable, seekable, append, mz) + if truncate + buf.size = buf.offset_or_compacted + end + buf end const IOBuffer = GenericIOBuffer{Memory{UInt8}} function GenericIOBuffer(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) where T<:AbstractVector{UInt8} - GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize) + maxsize::Integer, truncate::Bool) where T<:AbstractVector{UInt8} + GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize, truncate) end + +# For this method, we use the underlying Memory of the vector. Therefore, we need to set the, +# ptr and size accordingly, so the buffer only uses the part of the memory that the vector does. function GenericIOBuffer(data::Vector{UInt8}, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) + maxsize::Integer, truncate::Bool) ref = data.ref - buf = GenericIOBuffer(ref.mem, readable, writable, seekable, append, maxsize) + mem = ref.mem offset = memoryrefoffset(ref) - 1 - buf.ptr += offset - buf.size = length(data) + offset - buf.offset = offset + # The user may pass a vector of length <= maxsize, but where the underlying memory + # is larger than maxsize. Don't throw an error in that case. + mz = Int(maxsize)::Int + if !truncate && mz < length(data) + throw(ArgumentError("maxsize must not be smaller than data length")) + end + buf = _new_generic_iobuffer(Memory{UInt8}, mem, readable, writable, seekable, append, mz) + buf.offset_or_compacted = offset + buf.ptr = offset + 1 + if truncate + buf.size = offset + else + buf.size = length(data) + offset + end return buf end +get_offset(io::GenericIOBuffer) = max(0, io.offset_or_compacted) +get_compacted(io::GenericIOBuffer) = max(0, -io.offset_or_compacted) + # allocate Vector{UInt8}s for IOBuffer storage that can efficiently become Strings StringMemory(n::Integer) = unsafe_wrap(Memory{UInt8}, _string_n(n)) StringVector(n::Integer) = wrap(Array, StringMemory(n)) @@ -111,17 +234,11 @@ function IOBuffer( truncate::Union{Bool,Nothing}=nothing, maxsize::Integer=typemax(Int), sizehint::Union{Integer,Nothing}=nothing) - if maxsize < 0 - throw(ArgumentError("negative maxsize")) - end if sizehint !== nothing sizehint!(data, sizehint) end flags = open_flags(read=read, write=write, append=append, truncate=truncate) - buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, Int(maxsize)) - if flags.truncate - buf.size = buf.offset - end + buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, maxsize, flags.truncate) return buf end @@ -131,17 +248,23 @@ function IOBuffer(; append::Union{Bool,Nothing}=nothing, truncate::Union{Bool,Nothing}=true, maxsize::Integer=typemax(Int), - sizehint::Union{Integer,Nothing}=nothing) - size = sizehint !== nothing ? Int(sizehint) : maxsize != typemax(Int) ? Int(maxsize) : 32 + sizehint::Union{Integer,Nothing}=nothing, + ) + mz = Int(maxsize)::Int + if mz < 0 + throw(ArgumentError("negative maxsize")) + end + size = if sizehint !== nothing + # Allow negative sizehint, just like `sizehint!` does + min(mz, max(0, Int(sizehint)::Int)) + else + min(mz, 32) + end flags = open_flags(read=read, write=write, append=append, truncate=truncate) - buf = IOBuffer( - StringMemory(size), - read=flags.read, - write=flags.write, - append=flags.append, - truncate=flags.truncate, - maxsize=maxsize) - fill!(buf.data, 0) + # A common usecase of IOBuffer is to incrementally construct strings. By using StringMemory + # as the default storage, we can turn the result into a string without copying. + buf = _new_generic_iobuffer(Memory{UInt8}, StringMemory(size), flags.read, flags.write, true, flags.append, mz) + buf.size = 0 return buf end @@ -158,21 +281,53 @@ If `data` is given, creates a `PipeBuffer` to operate on a data vector, optionally specifying a size beyond which the underlying `Array` may not be grown. """ PipeBuffer(data::AbstractVector{UInt8}=Memory{UInt8}(); maxsize::Int = typemax(Int)) = - GenericIOBuffer(data, true, true, false, true, maxsize) + GenericIOBuffer(data, true, true, false, true, maxsize, false) PipeBuffer(maxsize::Integer) = (x = PipeBuffer(StringMemory(maxsize), maxsize = maxsize); x.size = 0; x) +# Internal method where truncation IS supported +function _truncated_pipebuffer(data::AbstractVector{UInt8}=Memory{UInt8}(); maxsize::Int = typemax(Int)) + buf = PipeBuffer(data) + buf.size = get_offset(buf) + buf.maxsize = maxsize + buf +end + _similar_data(b::GenericIOBuffer, len::Int) = similar(b.data, len) _similar_data(b::IOBuffer, len::Int) = StringMemory(len) -function copy(b::GenericIOBuffer) - ret = typeof(b)(b.reinit ? _similar_data(b, 0) : b.writable ? - copyto!(_similar_data(b, length(b.data)), b.data) : b.data, - b.readable, b.writable, b.seekable, b.append, b.maxsize) - ret.size = b.size - ret.ptr = b.ptr - ret.mark = b.mark - ret.offset = b.offset - return ret +# Note: Copying may change the value of the position (and mark) for un-seekable streams. +# However, these values are not stable anyway due to compaction. + +function copy(b::GenericIOBuffer{T}) where T + if b.reinit + # If buffer is used up, allocate a new size-zero buffer + # Reinit implies writable, and that ptr, size, offset and mark are already the default values + return typeof(b)(_similar_data(b, 0), b.readable, b.writable, b.seekable, b.append, b.maxsize, false) + elseif b.writable + # Else, we just copy the reachable bytes. If buffer is seekable, all bytes + # after offset are reachable, since they can be seeked to + used_span = get_used_span(b) + compacted = first(used_span) - get_offset(b) - 1 + len = length(used_span) + data = copyto!(_similar_data(b, len), view(b.data, used_span)) + ret = typeof(b)(data, b.readable, b.writable, b.seekable, b.append, b.maxsize, false) + ret.size = len + # Copying data over implicitly compacts, and may add compaction + ret.offset_or_compacted = -get_compacted(b) - compacted + ret.ptr = b.ptr - first(used_span) + 1 + ret.mark = b.mark + return ret + else + # When the buffer is just readable, they can share the same data, so we just make + # a shallow copy of the IOBuffer struct. + # Use internal constructor because we want to allow b.maxsize to be larger than data, + # in case that is the case for `b`. + ret = _new_generic_iobuffer(T, b.data, b.readable, b.writable, b.seekable, b.append, b.maxsize) + ret.offset_or_compacted = b.offset_or_compacted + ret.ptr = b.ptr + ret.mark = b.mark + return ret + end end show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", @@ -180,9 +335,9 @@ show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", "writable=", b.writable, ", ", "seekable=", b.seekable, ", ", "append=", b.append, ", ", - "size=", b.size - b.offset, ", ", + "size=", b.size - get_offset(b), ", ", "maxsize=", b.maxsize == typemax(Int) ? "Inf" : b.maxsize, ", ", - "ptr=", b.ptr - b.offset, ", ", + "ptr=", b.ptr - get_offset(b), ", ", "mark=", b.mark, ")") @noinline function _throw_not_readable() @@ -192,7 +347,7 @@ end function unsafe_read(from::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt) from.readable || _throw_not_readable() - avail = bytesavailable(from) + avail = bytesavailable(from) % UInt adv = min(avail, nb) unsafe_read!(p, from.data, from.ptr, adv) from.ptr += adv @@ -221,7 +376,45 @@ function unsafe_read!(dest::Ptr{UInt8}, src::DenseBytes, so::Integer, nbytes::UI nothing end -function peek(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int32},Type{UInt32},Type{Int64},Type{UInt64},Type{Int128},Type{UInt128},Type{Float16},Type{Float32},Type{Float64}}) +const MultiByteBitNumberType = Union{ + Type{UInt16}, + Type{Int16}, + Type{UInt32}, + Type{Int32}, + Type{UInt64}, + Type{Int64}, + Type{UInt128}, + Type{Int128}, + Type{Float16}, + Type{Float32}, + Type{Float64}, +} + +function load_from_array(T::MultiByteBitNumberType, data::AbstractArray{UInt8}, from::Int) + x = if T <: AbstractFloat + uinttype(T)(0) + else + unsigned(T)(0) + end + for i in 0:sizeof(x)-1 + x |= typeof(x)(data[from + i]) << (8 * i) + end + reinterpret(T, ltoh(x)) +end + +function peek(from::GenericIOBuffer, T::MultiByteBitNumberType) + from.readable || _throw_not_readable() + avail = bytesavailable(from) + nb = sizeof(T) + if nb > avail + throw(EOFError()) + end + return load_from_array(T, from.data, from.ptr) +end + +# This method can use a pointer, since the underlying buffer is dense +# and memory backed +function peek(from::GenericIOBuffer{<:MutableDenseArrayType}, T::MultiByteBitNumberType) from.readable || _throw_not_readable() avail = bytesavailable(from) nb = sizeof(T) @@ -235,29 +428,12 @@ function peek(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int3 return x end -function read(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int32},Type{UInt32},Type{Int64},Type{UInt64},Type{Int128},Type{UInt128},Type{Float16},Type{Float32},Type{Float64}}) +function read(from::GenericIOBuffer, T::MultiByteBitNumberType) x = peek(from, T) from.ptr += sizeof(T) return x end -function read_sub(from::GenericIOBuffer, a::AbstractArray{T}, offs, nel) where T - require_one_based_indexing(a) - from.readable || _throw_not_readable() - if offs+nel-1 > length(a) || offs < 1 || nel < 0 - throw(BoundsError()) - end - if isa(a, MutableDenseArrayType{UInt8}) - nb = UInt(nel * sizeof(T)) - GC.@preserve a unsafe_read(from, pointer(a, offs), nb) - else - for i = offs:offs+nel-1 - a[i] = read(from, T) - end - end - return a -end - @inline function read(from::GenericIOBuffer, ::Type{UInt8}) from.readable || _throw_not_readable() ptr = from.ptr @@ -283,20 +459,35 @@ read(from::GenericIOBuffer, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(fro isreadable(io::GenericIOBuffer) = io.readable iswritable(io::GenericIOBuffer) = io.writable -filesize(io::GenericIOBuffer) = (io.seekable ? io.size - io.offset : bytesavailable(io)) +# Number of bytes that can be read from the buffer, if you seek to the start first. +filesize(io::GenericIOBuffer) = (io.seekable ? io.size - get_offset(io) : bytesavailable(io)) + +# Number of bytes that can be read from the buffer. bytesavailable(io::GenericIOBuffer) = io.size - io.ptr + 1 -position(io::GenericIOBuffer) = io.ptr - io.offset - 1 + +# TODO: Document that position for an unmarked and unseekable stream is invalid (and make it error?) +function position(io::GenericIOBuffer) + # Position is zero-indexed, but ptr is one-indexed, hence the -1 + io.ptr - io.offset_or_compacted - 1 +end function skip(io::GenericIOBuffer, n::Integer) skip(io, clamp(n, Int)) end + function skip(io::GenericIOBuffer, n::Int) + # In both cases, the result will never go to before the first position, + # nor beyond the last position, and will not throw an error unless the stream + # is not seekable and try to skip a negative number of bytes. if signbit(n) + # Skipping a negative number of bytes is equivalent to seeking backwards. seekto = clamp(widen(position(io)) + widen(n), Int) seek(io, seekto) # Does error checking else - n_max = io.size + 1 - io.ptr - io.ptr += min(n, n_max) + # Don't use seek in order to allow a non-seekable IO to still skip bytes. + # Handle overflow. + maxptr = io.size + 1 + io.ptr = n > maxptr || io.ptr - n > maxptr ? maxptr : io.ptr + n io end end @@ -304,16 +495,30 @@ end function seek(io::GenericIOBuffer, n::Integer) seek(io, clamp(n, Int)) end + +function translate_seek_position(io::GenericIOBuffer, n::Int) + # If there is an offset (the field F is positive), then there are F unused bytes at the beginning + # of the data, and we need to seek to n + F + 1. (Also compensate for `seek` being zero- + # indexed) + + # If bytes has been compacted (field F is negative), then F bytes has been deleted from + # the buffer, and a virtual position n means a position n + F in the data. + # Remember that F is negative, so n + F is subtracting from n. So we also end up with + # n + F + 1. + clamp(widen(n) + widen(io.offset_or_compacted) + widen(1), Int) +end + function seek(io::GenericIOBuffer, n::Int) if !io.seekable ismarked(io) || throw(ArgumentError("seek failed, IOBuffer is not seekable and is not marked")) n == io.mark || throw(ArgumentError("seek failed, IOBuffer is not seekable and n != mark")) end + # TODO: REPL.jl relies on the fact that this does not throw (by seeking past the beginning or end # of an GenericIOBuffer), so that would need to be fixed in order to throw an error here - #(n < 0 || n > io.size - io.offset) && throw(ArgumentError("Attempted to seek outside IOBuffer boundaries.")) - #io.ptr = n + io.offset + 1 - io.ptr = clamp(n, 0, io.size - io.offset) + io.offset + 1 + max_ptr = io.size + 1 + min_ptr = get_offset(io) + 1 + io.ptr = clamp(translate_seek_position(io, n), min_ptr, max_ptr) return io end @@ -322,113 +527,163 @@ function seekend(io::GenericIOBuffer) return io end -# choose a resize strategy based on whether `resize!` is defined: -# for a Vector, we use `resize!`, but for most other types, -# this calls `similar`+copy -function _resize!(io::GenericIOBuffer, sz::Int) - a = io.data - offset = io.offset - if applicable(resize!, a, sz) - if offset != 0 - size = io.size - size > offset && copyto!(a, 1, a, offset + 1, min(sz, size - offset)) - io.ptr -= offset - io.size -= offset - io.offset = 0 - end - resize!(a, sz) +# Resize the io's data to `new_size`, which must not be > io.maxsize. +# Use `resize!` if the data supports it, else reallocate a new one and +# copy the old data over. +# If not `exact` and resizing is not supported, overallocate in order to +# prevent excessive resizing. +function _resize!(io::GenericIOBuffer, new_size::Int, exact::Bool) + old_data = io.data + if applicable(resize!, old_data, new_size) + resize!(old_data, new_size) else - size = io.size - if size >= sz && sz != 0 - b = a - else - b = _similar_data(io, sz == 0 ? 0 : max(overallocation(size - io.offset), sz)) - end - size > offset && copyto!(b, 1, a, offset + 1, min(sz, size - offset)) - io.data = b - io.ptr -= offset - io.size -= offset - io.offset = 0 + new_size = exact ? new_size : min(io.maxsize, overallocation(new_size)) + used_span = get_used_span(io) + deleted = first(used_span) - 1 + compacted = deleted - get_offset(io) + new_data = _similar_data(io, new_size) + io.data = new_data + iszero(new_size) && return io + len_used = length(used_span) + iszero(len_used) || copyto!(new_data, 1, old_data, first(used_span), len_used) + # Copying will implicitly compact, and so compaction must be updated + io.offset_or_compacted = -get_compacted(io) - compacted + io.ptr -= deleted + io.size = len_used end return io end function truncate(io::GenericIOBuffer, n::Integer) io.writable || throw(ArgumentError("truncate failed, IOBuffer is not writeable")) + # Non-seekable buffers can only be constructed with `PipeBuffer`, which is explicitly + # documented to not be truncatable. io.seekable || throw(ArgumentError("truncate failed, IOBuffer is not seekable")) n < 0 && throw(ArgumentError("truncate failed, n bytes must be ≥ 0, got $n")) n > io.maxsize && throw(ArgumentError("truncate failed, $(n) bytes is exceeds IOBuffer maxsize $(io.maxsize)")) - n = Int(n) + n = Int(n)::Int + offset = get_offset(io) + current_size = io.size - offset if io.reinit - io.data = _similar_data(io, n) + # If reinit, we don't need to truncate anything but just reinitializes + # the buffer with zeros. Mark, ptr and offset has already been reset. + io.data = fill!(_similar_data(io, n), 0x00) io.reinit = false - elseif n > length(io.data) + io.offset - _resize!(io, n) - end - ismarked(io) && io.mark > n && unmark(io) - n += io.offset - io.data[io.size+1:n] .= 0 - io.size = n - io.ptr = min(io.ptr, n+1) + io.size = n + elseif n < current_size + # Else, if we need to shrink the iobuffer, we simply change the pointers without + # actually shrinking the underlying storage, or copying data. + + # Clear the mark if it points to data that has now been deleted. + if translate_seek_position(io, io.mark) > n+offset + io.mark = -1 + end + io.size = n + offset + io.ptr = min(io.ptr, n + offset + 1) + elseif n > current_size + if n + offset > io.maxsize + compact!(io) + end + _resize!(io, n + get_offset(io), false) + fill!(view(io.data, io.size + 1:min(length(io.data), n + get_offset(io))), 0x00) + io.size = min(length(io.data), n + get_offset(io)) + end return io end -function compact(io::GenericIOBuffer) - io.writable || throw(ArgumentError("compact failed, IOBuffer is not writeable")) - io.seekable && throw(ArgumentError("compact failed, IOBuffer is seekable")) - io.reinit && return - local ptr::Int, bytes_to_move::Int - if ismarked(io) && io.mark < position(io) - io.mark == 0 && return - ptr = io.mark + io.offset - bytes_to_move = bytesavailable(io) + (io.ptr - ptr) - else - ptr = io.ptr - bytes_to_move = bytesavailable(io) +# Ensure that the buffer has room for at least `nshort` more bytes, except when +# doing that would exceed maxsize. +@inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort)) + +@inline function ensureroom(io::GenericIOBuffer, nshort::UInt) + # If the IO is not writable, we call the slow path only to error. + # If reinit, the data has been handed out to the user, and the IOBuffer + # no longer controls it, so we need to allocate a new one. + if !io.writable || io.reinit + return ensureroom_reallocate(io, nshort) + end + # The fast path here usually checks there is already room, then does nothing. + # When append is true, new data is added after io.size, not io.ptr + existing_space = min(lastindex(io.data), io.maxsize + get_offset(io)) - (io.append ? io.size : io.ptr - 1) + if existing_space < nshort % Int + # Outline this function to make it more likely that ensureroom inlines itself + return ensureroom_slowpath(io, nshort, existing_space) end - copyto!(io.data, 1, io.data, ptr, bytes_to_move) - io.size -= ptr - 1 - io.ptr -= ptr - 1 - io.offset = 0 - return + return io end -@noinline function ensureroom_slowpath(io::GenericIOBuffer, nshort::UInt) +# Throw error (placed in this function to outline it) or reinit the buffer +@noinline function ensureroom_reallocate(io::GenericIOBuffer, nshort::UInt) io.writable || throw(ArgumentError("ensureroom failed, IOBuffer is not writeable")) - if io.reinit - io.data = _similar_data(io, nshort % Int) - io.reinit = false - end - if !io.seekable - if !ismarked(io) && io.ptr > io.offset+1 && io.size <= io.ptr - 1 - io.ptr = 1 - io.size = 0 - io.offset = 0 - else - datastart = (ismarked(io) ? io.mark : io.ptr - io.offset) - if (io.size-io.offset+nshort > io.maxsize) || - (datastart > 4096 && datastart > io.size - io.ptr) || - (datastart > 262144) - # apply somewhat arbitrary heuristics to decide when to destroy - # old, read data to make more room for new data - compact(io) - end + io.data = _similar_data(io, min(io.maxsize, nshort % Int)) + io.reinit = false + io.offset_or_compacted = -get_compacted(io) + return io +end + +# Here, we already know there is not enough room at the end of the io's data. +@noinline function ensureroom_slowpath(io::GenericIOBuffer, nshort::UInt, available_bytes::Int) + reclaimable_bytes = first(get_used_span(io)) - 1 + # Avoid resizing and instead compact the buffer, only if we gain enough bytes from + # doing so (at least 32 bytes and 1/8th of the data length). Also, if we would have + # to resize anyway, there would be no point in compacting, so also check that. + if ( + reclaimable_bytes ≥ 32 && + reclaimable_bytes ≥ length(io.data) >>> 3 && + (reclaimable_bytes + available_bytes) % UInt ≥ nshort + ) + compact!(io) + return io + end + + desired_size = length(io.data) + Int(nshort) - available_bytes + if desired_size > io.maxsize + # If we can't fit all the requested data in the new buffer, we need to + # fit as much as possible, so we must compact + if !iszero(reclaimable_bytes) + desired_size -= compact!(io) + end + # Max out the buffer size if we want more than the buffer size + if length(io.data) < io.maxsize + _resize!(io, io.maxsize, true) end + else + # Else, we request only the requested size, but set `exact` to `false`, + # in order to overallocate to avoid growing the buffer by too little + _resize!(io, desired_size, false) end - return + + return io end -@inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort)) -@inline function ensureroom(io::GenericIOBuffer, nshort::UInt) - if !io.writable || (!io.seekable && io.ptr > io.offset+1) || io.reinit - ensureroom_slowpath(io, nshort) - end - n = min((nshort % Int) + (io.append ? io.size : io.ptr-1) - io.offset, io.maxsize) - l = length(io.data) + io.offset - if n > l - _resize!(io, Int(n)) +# Get the indices in data which cannot be deleted +function get_used_span(io::IOBuffer) + # A seekable buffer can recover data before ptr + return if io.seekable + get_offset(io) + 1 : io.size + # If non-seekable, the mark can be used to recover data before ptr, + # so data at the mark and after must also be saved + elseif io.mark > -1 + min(io.ptr, translate_seek_position(io, io.mark)) : io.size + else + io.ptr : io.size end - return io +end + +# Delete any offset, and also compact data if buffer is not seekable. +# Return the number of bytes deleted +function compact!(io::GenericIOBuffer)::Int + offset = get_offset(io) + used_span = get_used_span(io) + deleted = first(used_span) - 1 + compacted = deleted - offset + iszero(deleted) && return 0 + data = io.data + copyto!(data, 1, data, deleted + 1, length(used_span)) + io.offset_or_compacted = -get_compacted(io) - compacted + io.ptr -= deleted + io.size -= deleted + return deleted end eof(io::GenericIOBuffer) = (io.ptr - 1 >= io.size) @@ -439,17 +694,17 @@ function closewrite(io::GenericIOBuffer) end @noinline function close(io::GenericIOBuffer{T}) where T + if io.writable && !io.reinit + _resize!(io, 0, true) + end io.readable = false io.writable = false io.seekable = false io.size = 0 - io.offset = 0 io.maxsize = 0 io.ptr = 1 io.mark = -1 - if io.writable && !io.reinit - io.data = _resize!(io, 0) - end + io.offset_or_compacted = -get_compacted(io) nothing end @@ -472,31 +727,42 @@ julia> String(take!(io)) ``` """ function take!(io::GenericIOBuffer) - ismarked(io) && unmark(io) + io.mark = -1 if io.seekable - nbytes = io.size - io.offset - data = copyto!(StringVector(nbytes), 1, io.data, io.offset + 1, nbytes) + # If the buffer is seekable, then the previously consumed bytes from ptr+1:size + # must still be output, as they are not truly gone. + # Hence, we output all bytes from 1:io.size + offset = get_offset(io) + nbytes = io.size - offset + data = copyto!(StringVector(nbytes), 1, io.data, offset + 1, nbytes) else + # Else, if not seekable, bytes from 1:ptr-1 are truly gone and should not + # be output. Hence, we output `bytesavailable`, which is ptr:size nbytes = bytesavailable(io) data = read!(io, StringVector(nbytes)) end if io.writable + io.reinit = true io.ptr = 1 io.size = 0 - io.offset = 0 + io.offset_or_compacted = 0 end return data end + +# This method is specialized because we know the underlying data is a Memory, so we can +# e.g. wrap directly in an array without copying. Otherwise the logic is the same as +# the generic method function take!(io::IOBuffer) - ismarked(io) && unmark(io) + io.mark = -1 if io.seekable nbytes = filesize(io) if nbytes == 0 || io.reinit data = StringVector(0) elseif io.writable - data = wrap(Array, memoryref(io.data, io.offset + 1), nbytes) + data = wrap(Array, memoryref(io.data, get_offset(io) + 1), nbytes) else - data = copyto!(StringVector(nbytes), 1, io.data, io.offset + 1, nbytes) + data = copyto!(StringVector(nbytes), 1, io.data, get_offset(io) + 1, nbytes) end else nbytes = bytesavailable(io) @@ -512,7 +778,7 @@ function take!(io::IOBuffer) io.reinit = true io.ptr = 1 io.size = 0 - io.offset = 0 + io.offset_or_compacted = 0 end return data end @@ -529,46 +795,79 @@ state. This should only be used internally for performance-critical It might save an allocation compared to `take!` (if the compiler elides the Array allocation), as well as omits some checks. """ -_unsafe_take!(io::IOBuffer) = - wrap(Array, io.size == io.offset ? - memoryref(Memory{UInt8}()) : - memoryref(io.data, io.offset + 1), - io.size - io.offset) +function _unsafe_take!(io::IOBuffer) + offset = get_offset(io) + mem = if io.size == offset + memoryref(Memory{UInt8}()) + else + memoryref(io.data, offset + 1) + end + wrap(Array, mem, io.size - offset) +end function write(to::IO, from::GenericIOBuffer) - written::Int = bytesavailable(from) + # This would cause an infinite loop, as it should read until the end, but more + # data is being written into it continuously. if to === from - from.ptr = from.size + 1 + throw(ArgumentError("Writing all content fron an IOBuffer into itself in invalid")) else - written = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(written)) - from.ptr += written + available = bytesavailable(from) + written = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(available)) + from.ptr = from.size + 1 end return written end function unsafe_write(to::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt) ensureroom(to, nb) - ptr = (to.append ? to.size+1 : to.ptr) - written = Int(min(nb, Int(length(to.data))::Int - ptr + 1)) - towrite = written - d = to.data - while towrite > 0 - @inbounds d[ptr] = unsafe_load(p) - ptr += 1 + size = to.size + append = to.append + ptr = append ? size+1 : to.ptr + data = to.data + to_write = min(nb, (min(Int(length(data))::Int, to.maxsize + get_offset(to)) - ptr + 1) % UInt) % Int + # Dispatch based on the type of data, to possibly allow using memcpy + _unsafe_write(data, p, ptr, to_write % UInt) + # Update to.size only if the ptr has advanced to higher than + # the previous size. Otherwise, we just overwrote existing data + to.size = max(size, ptr + to_write - 1) + # If to.append, we only update size, not ptr. + if !append + to.ptr = ptr + to_write + end + return to_write +end + +@inline function _unsafe_write(data::AbstractVector{UInt8}, p::Ptr{UInt8}, from::Int, nb::UInt) + for i in 0:nb-1 + data[from + i] = unsafe_load(p) p += 1 - towrite -= 1 end - to.size = max(to.size, ptr - 1) - if !to.append - to.ptr += written +end + +@inline function _unsafe_write(data::MutableDenseArrayType{UInt8}, p::Ptr{UInt8}, from::Int, nb::UInt) + # Calling `unsafe_copyto!` is very efficient for large arrays, but has some overhead + # for small (< 5 bytes) arrays. + # Since a common use case of IOBuffer is to construct strings incrementally, often + # one char at a time, it's crucial to be fast in the case of small arrays. + # This optimization only gives a minor 10% speed boost in the best case. + if nb < 5 + @inbounds for i in UInt(1):nb + data[from + (i % Int) - 1] = unsafe_load(p, i) + end + else + GC.@preserve data begin + ptr = Ptr{UInt8}(pointer(data, from))::Ptr{UInt8} + @inline unsafe_copyto!(ptr, p, nb) + end end - return written end @inline function write(to::GenericIOBuffer, a::UInt8) ensureroom(to, UInt(1)) ptr = (to.append ? to.size+1 : to.ptr) - if ptr > to.maxsize + # We have just ensured there is room for 1 byte, EXCEPT if we were to exceed + # maxsize. So, we just need to check that here. + if ptr > to.maxsize + get_offset(to) return 0 else to.data[ptr] = a @@ -581,31 +880,26 @@ end end readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)) + function readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb::Int) - nr = min(nb, bytesavailable(io)) - if length(b) < nr - resize!(b, nr) + io.readable || _throw_not_readable() + to_read = min(nb, bytesavailable(io)) + if length(b) < to_read + resize!(b, to_read) end - read_sub(io, b, 1, nr) - return nr + checkbounds(b, 1:to_read) + GC.@preserve b unsafe_read(io, pointer(b), to_read) + to_read end read(io::GenericIOBuffer) = read!(io, StringVector(bytesavailable(io))) + +# For IO buffers, all the data is immediately available. readavailable(io::GenericIOBuffer) = read(io) -read(io::GenericIOBuffer, nb::Integer) = read!(io, StringVector(min(nb, bytesavailable(io)))) -function occursin(delim::UInt8, buf::IOBuffer) - p = pointer(buf.data, buf.ptr) - q = GC.@preserve buf ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, delim, bytesavailable(buf)) - return q != C_NULL -end +read(io::GenericIOBuffer, nb::Integer) = read!(io, StringVector(min(nb, bytesavailable(io)))) function occursin(delim::UInt8, buf::GenericIOBuffer) - data = buf.data - for i = buf.ptr:buf.size - @inbounds b = data[i] - b == delim && return true - end - return false + return in(delim, view(buf.data, buf.ptr:buf.size)) end function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) @@ -622,21 +916,45 @@ function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) end function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) - copyuntil(out, s, 0x0a, keep=true) - line = out.data - i = out.size # XXX: this is only correct for appended data. if the data was inserted, only ptr should change - if keep || i == out.offset || line[i] != 0x0a + # If the data is copied into the middle of the buffer of `out` instead of appended to the end, + # and !keep, and the line copied ends with \r\n, then the copyuntil (even if keep=false) + # will overwrite one too many bytes with the new \r byte. + # Work around this by making a new temporary buffer. + # Could perhaps be done better + if !out.append && out.ptr < out.size + 1 + newbuf = IOBuffer() + copyuntil(newbuf, s, 0x0a, keep=true) + v = take!(newbuf) + # Remove \r\n or \n if present + if !keep + if length(v) > 1 && last(v) == UInt8('\n') + pop!(v) + end + if length(v) > 1 && last(v) == UInt8('\r') + pop!(v) + end + end + write(out, v) return out - elseif i < 2 || line[i-1] != 0x0d - i -= 1 else - i -= 2 - end - out.size = i - if !out.append - out.ptr = i+1 + # Else, we can just copy the data directly into the buffer, and then + # subtract the last one or two bytes depending on `keep`. + copyuntil(out, s, 0x0a, keep=true) + line = out.data + i = out.size + if keep || i == out.offset_or_compacted || line[i] != 0x0a + return out + elseif i < 2 || line[i-1] != 0x0d + i -= 1 + else + i -= 2 + end + out.size = i + if !out.append + out.ptr = i+1 + end + return out end - return out end function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) @@ -644,6 +962,7 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) # note: findfirst + copyto! is much faster than a single loop # except for nout ≲ 20. A single loop is 2x faster for nout=5. nout = nread = something(findfirst(==(0x0a), data), length(data))::Int + # Remove the 0x0a (newline) if not keep, and also remove the 0x0d (\r) if it is there if !keep && nout > 0 && data[nout] == 0x0a nout -= 1 nout > 0 && data[nout] == 0x0d && (nout -= 1) @@ -652,6 +971,7 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) io.ptr += nread return out end + copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) diff --git a/base/stream.jl b/base/stream.jl index 67af2bff3682c..ae3d4f3c821e0 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -615,9 +615,9 @@ end ## BUFFER ## ## Allocate space in buffer (for immediate use) function alloc_request(buffer::IOBuffer, recommended_size::UInt) - ensureroom(buffer, Int(recommended_size)) + ensureroom(buffer, recommended_size) ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data)-buffer.offset, buffer.maxsize) + buffer.offset - ptr + 1 + nb = min(length(buffer.data), buffer.maxsize + get_offset(buffer)) - ptr + 1 return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) end @@ -943,8 +943,7 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) nread = readbytes!(sbuf, a, nb) else initsize = length(a) - newbuf = PipeBuffer(a, maxsize=nb) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(a; maxsize=nb) nread = try s.buffer = newbuf write(newbuf, sbuf) @@ -991,8 +990,7 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) if bytesavailable(sbuf) >= nb unsafe_read(sbuf, p, nb) else - newbuf = PipeBuffer(unsafe_wrap(Array, p, nb), maxsize=Int(nb)) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(unsafe_wrap(Array, p, nb); maxsize=Int(nb)) try s.buffer = newbuf write(newbuf, sbuf) @@ -1600,8 +1598,7 @@ function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int) nread = readbytes!(sbuf, a, nb) else initsize = length(a) - newbuf = PipeBuffer(a, maxsize=nb) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(a; maxsize=nb) nread = try s.buffer = newbuf write(newbuf, sbuf) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index a9d58f4b7871e..7ed5c1f5b3ed6 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -6,6 +6,267 @@ ioslength(io::IOBuffer) = (io.seekable ? io.size : bytesavailable(io)) bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) +# Julia Base's internals uses the PipeBuffer, which is an unseekable IOBuffer. +# There are no public constructors to build such a buffer, but we need to test +# it anyway. +# I make a new method here such that if the implementation of Base.PipeBuffer +# changes, these tests will still work. +new_unseekable_buffer() = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, typemax(Int), false) + +@testset "Basic tests" begin + @test_throws ArgumentError IOBuffer(;maxsize=-1) + @test_throws ArgumentError IOBuffer([0x01]; maxsize=-1) + + # Test that sizehint actually will sizehint the vector, + v = UInt8[] + buf = IOBuffer(v; sizehint=64, write=true) + @test length(v.ref.mem) >= 64 + + # Test that you can't make an IOBuffer with a maxsize + # smaller than the size you actually give it + @test_throws ArgumentError IOBuffer([0x01, 0x02]; maxsize=1) + @test_throws ArgumentError IOBuffer(b"abcdefghij"; maxsize=8) +end + +@testset "Basic reading" begin + # Readavailable is equal to read + buf = IOBuffer("abcdef") + @test read(buf, UInt8) == UInt8('a') + @test bytesavailable(buf) == 5 + @test readavailable(buf) == b"bcdef" + + # Reading less than all the bytes + buf = IOBuffer(b"ABCDEFGHIJ") + @test read(buf, 1) == b"A" + @test read(buf, 3) == b"BCD" + + # Reading more bytes than available will not error + @test read(buf, 100) == b"EFGHIJ" + + # Passing truncate=false will still truncate an IOBuffer with no + # initialized data + @test isempty(read(IOBuffer(;sizehint=34, truncate=false))) +end + +@testset "Byte occursin GenericIOBuffer" begin + buf = IOBuffer(@view(collect(0x1f:0x3d)[1:end])) + @test occursin(0x1f, buf) + @test occursin(0x3d, buf) + @test occursin(0x2a, buf) + + @test !occursin(0xff, buf) + @test !occursin(0x00, buf) + + v = Vector{UInt8}("bcdefg") + pushfirst!(v, UInt8('a')) + buf = IOBuffer(v) + @test occursin(UInt8('a'), buf) + read(buf, UInt8) + @test !occursin(UInt8('a'), buf) + @test !occursin(0x00, buf) + + buf = IOBuffer("abcdefg") + @test occursin(UInt8('a'), buf) +end + +@testset "Non-Memory backed IOBuffer" begin + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf) == 0x02:0x0d + + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf, UInt8) == 0x02 + @test read(buf) == 0x03:0x0d + + v = view(collect(UInt8('a'):UInt8('z')), 4:10) + buf = IOBuffer(v, read=true, write=true) + @test read(buf, UInt8) == UInt8('d') + @test read(buf) == UInt8('e'):UInt8('j') + seekstart(buf) + @test read(buf, UInt8) == UInt8('d') + write(buf, UInt8('x')) + write(buf, "ABC") + seekstart(buf) + @test read(buf) == b"dxABCij" +end + +@testset "Copying" begin + # Test offset is preserved when copying + v = UInt8[] + pushfirst!(v, UInt8('a'), UInt8('b'), UInt8('c')) + buf = IOBuffer(v; write=true, read=true, append=true) + write(buf, "def") + read(buf, UInt16) + buf2 = copy(buf) + @test String(read(buf)) == "cdef" + @test String(read(buf2)) == "cdef" + + # Test copying with non-Memory backed GenericIOBuffer + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf, UInt16) == 0x0302 + buf2 = copy(buf) + @test isreadable(buf2) + @test !iswritable(buf2) + @test read(buf2) == 0x04:0x0d + + # Test copying a non-seekable stream + buf = new_unseekable_buffer() + write(buf, "abcdef") + read(buf, UInt16) + mark(buf) + read(buf, UInt16) + buf2 = copy(buf) + @test read(buf2) == b"ef" + reset(buf2) + @test read(buf2) == b"cdef" + + # Test copying seekable stream + buf = IOBuffer() + write(buf, "abcdef") + seekstart(buf) + read(buf) + mark(buf) + buf2 = copy(buf) + @test reset(buf2) == 6 + seekstart(buf2) + @test read(buf2) == b"abcdef" + + # Test copying a taken buffer + buf = IOBuffer() + write(buf, "abcdef") + take!(buf) + buf2 = copy(buf) + @test eof(buf2) + seekstart(buf2) + @test eof(buf2) +end + +@testset "copyuntil" begin + a = IOBuffer(b"abcdeajdgabdfg") + b = IOBuffer(collect(b"xx"); write=true, read=true, append=true) + copyuntil(b, a, UInt8('a')) + @test read(b) == b"xx" + seekstart(b) + copyuntil(b, a, UInt8('a'); keep=true) + @test read(b) == b"xxbcdea" + seekstart(b) + copyuntil(b, a, UInt('w')) + @test read(b) == b"xxbcdeajdgabdfg" +end + +@testset "copyline" begin + a = IOBuffer(b"abcde\nabc\r\nabc\n\r\nac") + b = IOBuffer() + copyline(b, a) + @test take!(copy(b)) == b"abcde" + copyline(b, a) + @test take!(copy(b)) == b"abcdeabc" + copyline(b, a; keep=true) + @test take!(copy(b)) == b"abcdeabcabc\n" + copyline(b, a; keep=false) + @test take!(copy(b)) == b"abcdeabcabc\n" + copyline(b, a; keep=false) + @test take!(copy(b)) == b"abcdeabcabc\nac" + + # Test a current bug in copyline + a = Base.SecretBuffer("abcde\r\n") + b = IOBuffer() + write(b, "xxxxxxxxxx") + seek(b, 2) + copyline(b, a; keep=false) + Base.shred!(a) + @test take!(b) == b"xxabcdexxx" +end + +@testset "take!" begin + a = IOBuffer("abc") + @test take!(a) == b"abc" + + v = UInt8[] + pushfirst!(v, 0x0a) + buf = IOBuffer(v; write=true, append=true) + write(buf, "def") + @test take!(buf) == b"\ndef" + + v = view(collect(b"abcdefghij"), 3:9) + buf = IOBuffer(v; write=true, read=true) + read(buf, UInt8) + write(buf, "xxy") + @test take!(buf) == b"cxxyghi" + + v = view(collect(b"abcdefghij"), 3:9) + buf = IOBuffer(v; write=true, read=true) + + # Take on unseekable buffer does not return used bytes. + buf = new_unseekable_buffer() + write(buf, 0x61) + write(buf, "bcd") + @test read(buf, UInt8) == 0x61 + @test take!(buf) == b"bcd" + + # Compaction is reset after take! + buf = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, 100, false) + write(buf, rand(UInt8, 50)) + read(buf, 40) + write(buf, rand(UInt8, 100)) + mark(buf) + read(buf, 70) + @test position(buf) == 110 + @test length(buf.data) <= 100 + v = take!(buf) + write(buf, 0xf1) + @test position(buf) == 0 + @test !ismarked(buf) +end + +@testset "maxsize is preserved" begin + # After take! + buf = IOBuffer(; maxsize=3) + print(buf, "abcdef") + @test take!(buf) == b"abc" + print(buf, "abcdef") + @test take!(buf) == b"abc" + + # After resizing + buf = IOBuffer(;maxsize=128) + write(buf, collect(0x00:0x10)) + write(buf, collect(0x11:0x30)) + write(buf, collect(0x31:0x98)) + write(buf, collect(0x99:0xff)) + seekstart(buf) + @test read(buf) == 0x00:UInt8(127) + + # Edge case: When passing a Vector, does not error if the + # underlying mem is larger than maxsize + v = pushfirst!([0x01], 0x02) + io = IOBuffer(v; maxsize=2) + @test read(io) == b"\x02\x01" + + # Buffer will not write past maxsize, even if given a larger buffer + # And also even if the data is taken and replaced + v = sizehint!(UInt8[], 128) + io = IOBuffer(v; write=true, read=true, maxsize=12) + write(io, 0x01:0x0f) + seekstart(io) + @test read(io) == 0x01:0x0c + @test write(io, 0x01) == 0 + @test write(io, "abc") == 0 + @test take!(io).ref.mem === v.ref.mem + write(io, 0x01:0x0f) + @test take!(io) == 0x01:0x0c +end + +@testset "Write to self" begin + buffer = IOBuffer() + @test_throws ArgumentError write(buffer, buffer) + + # Write to another IOBuffer with limited size + to = IOBuffer(;maxsize=4) + from = IOBuffer(collect(b"abcdefghi")) + write(to, from) + @test String(take!(to)) == "abcd" + @test eof(from) +end + @testset "Read/write empty IOBuffer" begin io = IOBuffer() @test eof(io) @@ -33,7 +294,7 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) @test position(io) == 0 truncate(io, 10) @test position(io) == 0 - @test all(io.data .== 0) + @test all(view(io.data, 1:10) .== 0) @test write(io, Int16[1, 2, 3, 4, 5, 6]) === 12 seek(io, 2) truncate(io, 10) @@ -67,22 +328,89 @@ end @test_throws ArgumentError write(io,UInt8[0]) @test String(take!(io)) == "hamster\nguinea pig\nturtle" @test String(take!(io)) == "hamster\nguinea pig\nturtle" #should be unchanged - @test_throws ArgumentError Base.compact(io) # not writeable close(io) end +@testset "Truncate" begin + # Fails for non-writable and non-seekable + @test_throws ArgumentError truncate(PipeBuffer(), 0) + @test_throws ArgumentError truncate(IOBuffer(b"abcde"), 3) + + # Standard use + buf = IOBuffer(collect(b"abcdef"); write=true, read=true) + truncate(buf, 4) + @test read(buf) == b"abcd" + @test take!(buf) == b"abcd" + + # Mark is removed if beyond the size + buf = IOBuffer() + write(buf, "abcde") + seek(buf, 4) + mark(buf) + truncate(buf, 4) + @test !ismarked(buf) + + # Making it larger + buf = IOBuffer(collect(b"abcdef"); write=true, read=true) + seek(buf, 3) + truncate(buf, 3) + write(buf, 'X') + mark(buf) + truncate(buf, 5) + @test ismarked(buf) + @test reset(buf) == 4 + @test take!(buf) == b"abcX\0" + + # With offset + v = pushfirst!(UInt8[0x62, 0x63, 0x64], 0x61) + buf = IOBuffer(v; write=true, read=true) + seekstart(buf) + read(buf, UInt8) + mark(buf) + truncate(buf, 7) + @test reset(buf) == 1 + @test take!(buf) == b"abcd\0\0\0" +end + +@testset "Position of compactable buffer" begin + # Set maxsize, because otherwise compaction it too hard to reason about, + # and this test will be brittle + io = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, 100, false) + write(io, "abcd") + read(io, UInt16) + @test position(io) == 2 + write(io, "abcde"^80) + @test position(io) == 2 + read(io, 60) + @test position(io) == 62 + mark(io) + # Trigger compaction + write(io, rand(UInt8, 50)) + @test position(io) == 62 + v1 = read(io, 20) + @test position(io) == 82 + @test reset(io) == 62 + @test position(io) == 62 + v2 = read(io, 20) + @test v1 == v2 +end + @testset "PipeBuffer" begin - io = PipeBuffer() + io = new_unseekable_buffer() @test_throws EOFError read(io,UInt8) @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 + + # PipeBuffer is append, so writing to it does not advance the position @test position(io) == 0 @test readline(io) == "pancakes" - Base.compact(io) @test readline(io) == "waffles" @test write(io,"whipped cream\n") > 0 @test readline(io) == "blueberries" + + # Pipebuffers do not support seeking, and therefore do not support truncation. @test_throws ArgumentError seek(io,0) @test_throws ArgumentError truncate(io,0) + @test readline(io) == "whipped cream" @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 @test readlines(io) == String["pancakes", "waffles", "blueberries"] @@ -116,58 +444,6 @@ end end rm(fname) end - - Base.compact(io) - @test position(io) == 0 - @test ioslength(io) == 0 - Base._resize!(io,0) - Base.ensureroom(io,50) - @test position(io) == 0 - @test ioslength(io) == 0 - @test length(io.data) == 50 - Base.ensureroom(io,10) - @test ioslength(io) == 0 - @test length(io.data) == 50 - io.maxsize = 75 - Base.ensureroom(io,100) - @test ioslength(io) == 0 - @test length(io.data) == 75 - seekend(io) - @test ioslength(io) == 0 - @test position(io) == 0 - write(io,zeros(UInt8,200)) - @test ioslength(io) == 75 - @test length(io.data) == 75 - write(io,1) - @test ioslength(io) == 75 - @test length(io.data) == 75 - write(io,[1,2,3]) - @test ioslength(io) == 75 - @test length(io.data) == 75 - skip(io,1) - @test write(io,UInt8(104)) === 1 - skip(io,3) - @test write(io,b"apples") === 3 - skip(io,71) - @test write(io,'y') === 1 - @test read(io, String) == "happy" - @test eof(io) - write(io,zeros(UInt8,73)) - write(io,'a') - write(io,'b') - write(io,'c') - write(io,'d') - write(io,'e') - @test ioslength(io) == 75 - @test length(io.data) == 75 - @test position(io) == 0 - skip(io,72) - @test String(take!(io)) == "\0ab" - @test String(take!(io)) == "" - - # issues 4021 - print(io, true) - close(io) end @testset "issue 5453" begin @@ -248,9 +524,6 @@ end truncate(io2, io2.size - 2) @test read(io2, String) == "goodnightmoonhelloworld" seek(io2, 0) - write(io2, io2) - @test read(io2, String) == "" - @test bufcontents(io2) == "goodnightmoonhelloworld" end # issue #11917 @@ -347,24 +620,42 @@ end @test n == 5 end -@testset "Base.compact" begin - a = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) - mark(a) # mark at position 0 - write(a, "Hello!") - @test Base.compact(a) === nothing # because pointer > mark - close(a) - b = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) - write(b, "Hello!") - read(b) - mark(b) # mark at position 6 - write(b, "Goodbye!") # now pointer is > mark but mark is > 0 - Base.compact(b) - @test readline(b) == "Goodbye!" - close(b) +@testset "Compacting" begin + # Compacting works + buf = Base.GenericIOBuffer(UInt8[], true, true, false, true, 20, false) + mark(buf) + write(buf, "Hello"^5) + reset(buf) + unmark(buf) + read(buf, UInt8) + read(buf, UInt8) + write(buf, "a!") + @test length(buf.data) == 20 + @test String(take!(buf)) == "llo" * "Hello"^3 * "a!" + + # Compacting does not do anything when mark == 0 + buf = Base.GenericIOBuffer(UInt8[], true, true, false, true, 5, false) + mark(buf) + write(buf, "Hello") + reset(buf) + mark(buf) + read(buf, UInt8) + read(buf, UInt8) + @test write(buf, "a!") == 0 + @test take!(buf) == b"llo" + + # Compacting without maxsize still works + buf = new_unseekable_buffer() + data = repeat(b"abcdefg", 100) + write(buf, data) + read(buf, 600) + data_len = length(buf.data) + write(buf, view(data, 1:500)) + @test length(buf.data) == data_len end @testset "peek(::GenericIOBuffer)" begin - io = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) + io = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int), false) write(io, "こんにちは") @test peek(io) == 0xe3 @test peek(io, Char) == 'こ' @@ -381,13 +672,22 @@ end v = @view a[1:2] io = IOBuffer() write(io,1) + write(io,0) seek(io,0) - @test Base.read_sub(io,v,1,1) == [1,0] + @test read!(io, v) == [1, 0] end @testset "with offset" begin b = pushfirst!([0x02], 0x01) @test take!(IOBuffer(b)) == [0x01, 0x02] + + # Read-only buffer does not take control of underlying buffer + v = pushfirst!([0x62, 0x63], 0x61) + buf = IOBuffer(v; write=false) + @test read(buf) == b"abc" + @test v == b"abc" # v is unchanged + + # Truncate end @testset "#54636 reading from non-dense vectors" begin From a046da5f259f74011d5dab6716ac9c9fbe4424d7 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Fri, 28 Mar 2025 11:46:39 -0700 Subject: [PATCH 014/662] lowering: Fix captured vars shadowed by an inner global declaration (#57648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As discovered in #57547, lowering isn't resolving references to captured variables when a global of the same name is declared in any scope in the current function: ``` julia> let g = 1 function f() let; global g = 2; end; return g # g is not resolved end; f() end ERROR: Found raw symbol in code returned from lowering. Expected all symbols to have been resolved to GlobalRef or slots. ``` `resolve-scopes` correctly detects that we're returning the local, but scope-blocks are removed in this pass. As a result, `analyze-vars-lambda` finds more globals than `resolve-scopes` did when calling `find-global-decls` (which does not peek inside nested scopes) on `f`. `analyze-vars-lambda` then calculates the set of captured variables in `f` to be: ``` captvars = (current_env ∩ free_vars) - (new_staticparams ∪ wrong_globals) ``` so `g` in this case is never captured. This bug was introduced (revealed, maybe) in #57051---the whole resolution step used to happen after lowering, so lowering passes disagreeing on scopes would be less visible. Fix: omit globals from the free variable list in `analyze-vars-lambda` in the first place. This way we don't need to call the scope-block-dependent `find-global-decls`. --- doc/src/manual/variables-and-scoping.md | 10 ++-- src/julia-syntax.scm | 24 ++++---- test/syntax.jl | 73 ++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/doc/src/manual/variables-and-scoping.md b/doc/src/manual/variables-and-scoping.md index ab2d969dd9b0e..ab0dbdb845d8e 100644 --- a/doc/src/manual/variables-and-scoping.md +++ b/doc/src/manual/variables-and-scoping.md @@ -136,12 +136,14 @@ inside of another local scope, the scope it creates is nested inside of all the local scopes that it appears within, which are all ultimately nested inside of the global scope of the module in which the code is evaluated. Variables in outer scopes are visible from any scope they contain — meaning that they can be -read and written in inner scopes — unless there is a local variable with the -same name that "shadows" the outer variable of the same name. This is true even -if the outer local is declared after (in the sense of textually below) an inner +read and written in inner scopes — unless there is a variable with the same name +that "shadows" the outer variable of the same name. This is true even if the +outer local is declared after (in the sense of textually below) an inner block. When we say that a variable "exists" in a given scope, this means that a variable by that name exists in any of the scopes that the current scope is -nested inside of, including the current one. +nested inside of, including the current one. If a variable's value is used in a +local scope, but nothing with its name exists in this scope, it is assumed to be +a global. Some programming languages require explicitly declaring new variables before using them. Explicit declaration works in Julia too: in any local scope, writing diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 301d9fd82e4c2..0e1618ed140a6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3357,11 +3357,11 @@ (define (lambda-all-vars e) (append (lam:argnames e) (caddr e))) -;; compute set of variables referenced in a lambda but not bound by it +;; compute set of non-global variables referenced in a lambda but not bound by it (define (free-vars- e tab) (cond ((or (eq? e UNUSED) (underscore-symbol? e)) tab) ((symbol? e) (put! tab e #t)) - ((and (pair? e) (eq? (car e) 'globalref)) tab) + ((and (pair? e) (memq (car e) '(global globalref))) tab) ((and (pair? e) (eq? (car e) 'break-block)) (free-vars- (caddr e) tab)) ((and (pair? e) (eq? (car e) 'with-static-parameters)) (free-vars- (cadr e) tab)) ((or (atom? e) (quoted? e)) tab) @@ -3385,9 +3385,15 @@ vi) tab)) +;; env: list of vinfo (includes any closure #self#; should not include globals) +;; captvars: list of vinfo +;; sp: list of symbol +;; new-sp: list of symbol (static params declared here) +;; methsig: `(call (core svec) ...) +;; tab: table of (name . var-info) (define (analyze-vars-lambda e env captvars sp new-sp methsig tab) (let* ((args (lam:args e)) - (locl (caddr e)) + (locl (lam:vinfo e)) (allv (nconc (map arg-name args) locl)) (fv (let* ((fv (diff (free-vars (lam:body e)) allv)) ;; add variables referenced in declared types for free vars @@ -3397,27 +3403,23 @@ fv)))) (append (diff dv fv) fv))) (sig-fv (if methsig (free-vars methsig) '())) - (glo (find-global-decls (lam:body e))) ;; make var-info records for vars introduced by this lambda (vi (nconc (map (lambda (decl) (make-var-info (decl-var decl))) args) (map make-var-info locl))) - (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v glo)) (not (memq v new-sp))) + (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v new-sp))) (memq v sig-fv))) sp)) ;; captured vars: vars from the environment that occur ;; in our set of free variables (fv). (cv (append (filter (lambda (v) (and (memq (vinfo:name v) fv) - (not (memq (vinfo:name v) new-sp)) - (not (memq (vinfo:name v) glo)))) + (not (memq (vinfo:name v) new-sp)))) env) (map make-var-info capt-sp))) (new-env (append vi ;; new environment: add our vars - (filter (lambda (v) - (and (not (memq (vinfo:name v) allv)) - (not (memq (vinfo:name v) glo)))) + (filter (lambda (v) (not (memq (vinfo:name v) allv))) env)))) (analyze-vars (lam:body e) new-env @@ -4043,7 +4045,7 @@ f(x) = yt(x) ((atom? e) e) (else (case (car e) - ((quote top core globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e) + ((quote top core global globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e) ((toplevel-only) ;; hack to avoid generating a (method x) expr for struct types (if (eq? (cadr e) 'struct) diff --git a/test/syntax.jl b/test/syntax.jl index 107bf42ba7267..ec6c2ffb00635 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2417,15 +2417,6 @@ macro a35391(b) end @test @a35391(0) === (0,) -# global declarations from the top level are not inherited by functions. -# don't allow such a declaration to override an outer local, since it's not -# clear what it should do. -@test Meta.lower(Main, :(let - x = 1 - let - global x - end - end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope") # note: this `begin` block must be at the top level _temp_33553 = begin global _x_this_remains_undefined @@ -2437,6 +2428,70 @@ end @test _temp_33553 == 2 @test !@isdefined(_x_this_remains_undefined) +module GlobalContainment +using Test +@testset "scope of global declarations" begin + + # global declarations from the top level are not inherited by functions. + # don't allow such a declaration to override an outer local, since it's not + # clear what it should do. + @test Meta.lower( + Main, + :(let + x = 1 + let + global x + end + end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope") + + # a declared global can shadow a local in an outer scope + @test let + function f() + g0 = 2 + let; global g0 = 1; end + a = () -> (global g0 = 1); a(); + return g0 + end + (f(), g0); + end === (2, 1) + @test let + function f() + let; global g2 = 1; end; + let; try; g2 = 2; catch _; end; end; + end + (f(), g2) + end === (2, 1) + + # an inner global declaration should not interfere with the closure (#57547) + @test let + g3 = 1 + function f() + let; global g3 = 2; end; + return g3 + end + f() + end === 1 + @test_throws UndefVarError let + function returns_global() + for i in 1 + global ge = 2 + end + return ge # local declared below + end + ge = returns_global() + end + @test let + function f(x::T) where T + function g(x) + let; global T = 1; end + x::T + end; g(x) + end; f(1) + end === 1 + +end +end + # lowering of adjoint @test (1 + im)' == 1 - im x = let var"'"(x) = 2x From 3c88fa574757163ebd13a92d2c6e66f9c7d2ba6b Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sat, 29 Mar 2025 11:58:32 -0400 Subject: [PATCH 015/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=2026bdeeeb1=20to=2093fb66db1=20(#57936)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: master Julia branch: master Old commit: 26bdeeeb1 New commit: 93fb66db1 Julia version: 1.13.0-DEV Pkg version: 1.12.0(Does not match) Bump invoked by: @IanButterworth Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/26bdeeeb10f3ab0c6cf7478cb8454a800b4578be...93fb66db1834ddf432141a4809b2b6395b5d3260 ``` $ git log --oneline 26bdeeeb1..93fb66db1 93fb66db1 help debug freebsd test failure (#4199) b323a3829 Artifacts: Add download size to `Artifacts.toml` (#4171) b808cb1e9 Add a helper script for making missing Pkg tags (#4194) dfd0b8327 Update codecov-action to v5 (#4191) ab5370466 Create dependabot.yml (#4190) 6ac0df4d8 allow authors to be a TOML array (#3710) ``` Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 | 1 - .../Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 | 1 - .../Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 | 1 + .../Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 create mode 100644 deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 diff --git a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 b/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 deleted file mode 100644 index b9ff96ecde0ea..0000000000000 --- a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -0d36bf3edd61c39515a647719e5af8b9 diff --git a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 b/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 deleted file mode 100644 index 093e0a263c7e5..0000000000000 --- a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -c5c91b56f3548a13851081e589ec7347a7ca8ab1de792645b1ec04ee0e2e57678fdb2721d6243450e278457b0edc42ece105d205a61ab9cdcb6ea293bbcbba9f diff --git a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 new file mode 100644 index 0000000000000..470acd2b454dc --- /dev/null +++ b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 @@ -0,0 +1 @@ +e9b791bef7662fe5f14d2cff005c9fa9 diff --git a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 new file mode 100644 index 0000000000000..8fefef4d95722 --- /dev/null +++ b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 @@ -0,0 +1 @@ +43416e2e16e59356254323110e7158148099a57769cf005abfe6c4f3dfc3877269420779e2925c0b371fc18117422fc6a82135cc978d2b32ef7c49a475f1c41e diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 2fdf646e80bbf..26b676128c1d8 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 26bdeeeb10f3ab0c6cf7478cb8454a800b4578be +PKG_SHA1 = 93fb66db1834ddf432141a4809b2b6395b5d3260 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From a5e0eabfdbecc40ff9c3d4ba86a4bc76036f352e Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Sat, 29 Mar 2025 12:36:42 -0400 Subject: [PATCH 016/662] Move `eachregion(::AnnotatedString)` implementation to `Base` (#57912) Excising part of https://github.com/JuliaLang/julia/pull/56194/ on the way to reviving that PR. --- base/strings/annotated.jl | 258 +++++++++++------------------------ base/strings/annotated_io.jl | 201 +++++++++++++++++++++++++++ base/strings/strings.jl | 1 + test/strings/annotated.jl | 48 +++++++ 4 files changed, 333 insertions(+), 175 deletions(-) create mode 100644 base/strings/annotated_io.jl diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 5142c32e1b70d..0da6c26751618 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -460,201 +460,109 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin AnnotatedString(String(take!(outstr)), annots) end -## AnnotatedIOBuffer - -struct AnnotatedIOBuffer <: AbstractPipe - io::IOBuffer - annotations::Vector{RegionAnnotation} -end - -AnnotatedIOBuffer(io::IOBuffer) = AnnotatedIOBuffer(io, Vector{RegionAnnotation}()) -AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) - -function show(io::IO, aio::AnnotatedIOBuffer) - show(io, AnnotatedIOBuffer) - size = filesize(aio.io) - print(io, '(', size, " byte", ifelse(size == 1, "", "s"), ", ", - length(aio.annotations), " annotation", ifelse(length(aio.annotations) == 1, "", "s"), ")") +struct RegionIterator{S <: AbstractString} + str::S + regions::Vector{UnitRange{Int}} + annotations::Vector{Vector{Annotation}} end -pipe_reader(io::AnnotatedIOBuffer) = io.io -pipe_writer(io::AnnotatedIOBuffer) = io.io - -# Useful `IOBuffer` methods that we don't get from `AbstractPipe` -position(io::AnnotatedIOBuffer) = position(io.io) -seek(io::AnnotatedIOBuffer, n::Integer) = (seek(io.io, n); io) -seekend(io::AnnotatedIOBuffer) = (seekend(io.io); io) -skip(io::AnnotatedIOBuffer, n::Integer) = (skip(io.io, n); io) -copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations)) - -annotations(io::AnnotatedIOBuffer) = io.annotations - -annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = - (_annotate!(io.annotations, range, label, val); io) - -function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) - astr = AnnotatedString(astr) - offset = position(io.io) - eof(io) || _clear_annotations_in_region!(io.annotations, offset+1:offset+ncodeunits(astr)) - _insert_annotations!(io, astr.annotations) - write(io.io, String(astr)) -end +Base.length(si::RegionIterator) = length(si.regions) -write(io::AnnotatedIOBuffer, c::AnnotatedChar) = - write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) -write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x) -write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) -write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) - -function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) - destpos = position(dest) - isappending = eof(dest) - srcpos = position(src) - nb = write(dest.io, src.io) - isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) - srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) - for annot in src.annotations if first(annot.region) >= srcpos] - _insert_annotations!(dest, srcannots, destpos - srcpos) - nb +Base.@propagate_inbounds function Base.iterate(si::RegionIterator, i::Integer=1) + if i <= length(si.regions) + @inbounds ((SubString(si.str, si.regions[i]), si.annotations[i]), i+1) + end end -# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) -# work as expected. -function write(io::AbstractPipe, s::Union{AnnotatedString, SubString{<:AnnotatedString}}) - if pipe_writer(io) isa AnnotatedIOBuffer - write(pipe_writer(io), s) - else - invoke(write, Tuple{IO, typeof(s)}, io, s) - end::Int -end -# Can't be part of the `Union` above because it introduces method ambiguities -function write(io::AbstractPipe, c::AnnotatedChar) - if pipe_writer(io) isa AnnotatedIOBuffer - write(pipe_writer(io), c) - else - invoke(write, Tuple{IO, typeof(c)}, io, c) - end::Int -end +Base.eltype(::RegionIterator{S}) where { S <: AbstractString} = + Tuple{SubString{S}, Vector{Annotation}} """ - _clear_annotations_in_region!(annotations::Vector{$RegionAnnotation}, span::UnitRange{Int}) + eachregion(s::AnnotatedString{S}) + eachregion(s::SubString{AnnotatedString{S}}) -Erase the presence of `annotations` within a certain `span`. +Identify the contiguous substrings of `s` with a constant annotations, and return +an iterator which provides each substring and the applicable annotations as a +`Tuple{SubString{S}, Vector{$Annotation}}`. -This operates by removing all elements of `annotations` that are entirely -contained in `span`, truncating ranges that partially overlap, and splitting -annotations that subsume `span` to just exist either side of `span`. +# Examples + +```jldoctest; setup=:(using Base: AnnotatedString, eachregion) +julia> collect(eachregion(AnnotatedString( + "hey there", [(1:3, :face, :bold), + (5:9, :face, :italic)]))) +3-element Vector{Tuple{SubString{String}, Vector{$Annotation}}}: + ("hey", [$Annotation((:face, :bold))]) + (" ", []) + ("there", [$Annotation((:face, :italic))]) +``` """ -function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, span::UnitRange{Int}) - # Clear out any overlapping pre-existing annotations. - filter!(ann -> first(ann.region) < first(span) || last(ann.region) > last(span), annotations) - extras = Tuple{Int, RegionAnnotation}[] - for i in eachindex(annotations) - annot = annotations[i] - region = annot.region - # Test for partial overlap - if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) - annotations[i] = - setindex(annot, - if first(region) < first(span) - first(region):first(span)-1 - else - last(span)+1:last(region) - end, - :region) - # If `span` fits exactly within `region`, then we've only copied over - # the beginning overhang, but also need to conserve the end overhang. - if first(region) < first(span) && last(span) < last(region) - push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) - end +function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) + isempty(s) || isempty(subregion) && + return RegionIterator(s.string, UnitRange{Int}[], Vector{Annotation}[]) + events = annotation_events(s, subregion) + isempty(events) && return RegionIterator(s.string, [subregion], [Annotation[]]) + annotvals = Annotation[ + (; label, value) for (; label, value) in annotations(s)] + regions = Vector{UnitRange{Int}}() + annots = Vector{Vector{Annotation}}() + pos = first(events).pos + if pos > first(subregion) + push!(regions, thisind(s, first(subregion)):prevind(s, pos)) + push!(annots, []) + end + activelist = Int[] + for event in events + if event.pos != pos + push!(regions, pos:prevind(s, event.pos)) + push!(annots, annotvals[activelist]) + pos = event.pos + end + if event.active + insert!(activelist, searchsortedfirst(activelist, event.index), event.index) + else + deleteat!(activelist, searchsortedfirst(activelist, event.index)) end end - # Insert any extra entries in the appropriate position - for (offset, (i, entry)) in enumerate(extras) - insert!(annotations, i + offset, entry) + if last(events).pos < nextind(s, last(subregion)) + push!(regions, last(events).pos:thisind(s, last(subregion))) + push!(annots, []) end - annotations + RegionIterator(s.string, regions, annots) end -""" - _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{$RegionAnnotation}, offset::Int = position(io)) +function eachregion(s::SubString{<:AnnotatedString}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) + if isempty(s) + RegionIterator(s.string, Vector{UnitRange{Int}}(), Vector{Vector{Annotation}}()) + else + eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) + end +end -Register new `annotations` in `io`, applying an `offset` to their regions. +""" + annotation_events(string::AbstractString, annots::Vector{$RegionAnnotation}, subregion::UnitRange{Int}) + annotation_events(string::AnnotatedString, subregion::UnitRange{Int}) -The largely consists of simply shifting the regions of `annotations` by `offset` -and pushing them onto `io`'s annotations. However, when it is possible to merge -the new annotations with recent annotations in accordance with the semantics -outlined in [`AnnotatedString`](@ref), we do so. More specifically, when there -is a run of the most recent annotations that are also present as the first -`annotations`, with the same value and adjacent regions, the new annotations are -merged into the existing recent annotations by simply extending their range. +Find all annotation "change events" that occur within a `subregion` of `annots`, +with respect to `string`. When `string` is styled, `annots` is inferred. -This is implemented so that one can say write an `AnnotatedString` to an -`AnnotatedIOBuffer` one character at a time without needlessly producing a -new annotation for each character. +Each change event is given in the form of a `@NamedTuple{pos::Int, active::Bool, +index::Int}` where `pos` is the position of the event, `active` is a boolean +indicating whether the annotation is being activated or deactivated, and `index` +is the index of the annotation in question. """ -function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionAnnotation}, offset::Int = position(io)) - run = 0 - if !isempty(io.annotations) && last(last(io.annotations).region) == offset - for i in reverse(axes(annotations, 1)) - annot = annotations[i] - first(annot.region) == 1 || continue - i <= length(io.annotations) || continue - if annot.label == last(io.annotations).label && annot.value == last(io.annotations).value - valid_run = true - for runlen in 1:i - new = annotations[begin+runlen-1] - old = io.annotations[end-i+runlen] - if last(old.region) != offset || first(new.region) != 1 || old.label != new.label || old.value != new.value - valid_run = false - break - end - end - if valid_run - run = i - break - end - end +function annotation_events(s::AbstractString, annots::Vector{RegionAnnotation}, subregion::UnitRange{Int}) + events = Vector{NamedTuple{(:pos, :active, :index), Tuple{Int, Bool, Int}}}() # Position, Active?, Annotation index + for (i, (; region)) in enumerate(annots) + if !isempty(intersect(subregion, region)) + start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) + start <= stop || continue # Currently can't handle empty regions + push!(events, (pos=thisind(s, start), active=true, index=i)) + push!(events, (pos=nextind(s, stop), active=false, index=i)) end end - for runindex in 0:run-1 - old_index = lastindex(io.annotations) - run + 1 + runindex - old = io.annotations[old_index] - new = annotations[begin+runindex] - io.annotations[old_index] = setindex(old, first(old.region):last(new.region)+offset, :region) - end - for index in run+1:lastindex(annotations) - annot = annotations[index] - start, stop = first(annot.region), last(annot.region) - push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) - end + sort(events, by=e -> e.pos) end -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} - if (start = position(io)) == 0 - AnnotatedString(read(io.io, T), copy(io.annotations)) - else - annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) - for annot in io.annotations if last(annot.region) > start] - AnnotatedString(read(io.io, T), annots) - end -end -read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{AbstractString}}) = read(io, AnnotatedString{String}) -read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{String}) - -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar} - pos = position(io) - char = read(io.io, T) - annots = [NamedTuple{(:label, :value)}(annot) for annot in io.annotations if pos+1 in annot.region] - AnnotatedChar(char, annots) -end -read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{AbstractChar}}) = read(io, AnnotatedChar{Char}) -read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char}) - -function truncate(io::AnnotatedIOBuffer, size::Integer) - truncate(io.io, size) - filter!(ann -> first(ann.region) <= size, io.annotations) - map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), - io.annotations, io.annotations) - io -end +annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) = + annotation_events(s.string, annotations(s), subregion) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl new file mode 100644 index 0000000000000..87db57b8030c9 --- /dev/null +++ b/base/strings/annotated_io.jl @@ -0,0 +1,201 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## AnnotatedIOBuffer + +struct AnnotatedIOBuffer <: AbstractPipe + io::IOBuffer + annotations::Vector{RegionAnnotation} +end + +AnnotatedIOBuffer(io::IOBuffer) = AnnotatedIOBuffer(io, Vector{RegionAnnotation}()) +AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) + +function show(io::IO, aio::AnnotatedIOBuffer) + show(io, AnnotatedIOBuffer) + size = filesize(aio.io) + print(io, '(', size, " byte", ifelse(size == 1, "", "s"), ", ", + length(aio.annotations), " annotation", ifelse(length(aio.annotations) == 1, "", "s"), ")") +end + +pipe_reader(io::AnnotatedIOBuffer) = io.io +pipe_writer(io::AnnotatedIOBuffer) = io.io + +# Useful `IOBuffer` methods that we don't get from `AbstractPipe` +position(io::AnnotatedIOBuffer) = position(io.io) +seek(io::AnnotatedIOBuffer, n::Integer) = (seek(io.io, n); io) +seekend(io::AnnotatedIOBuffer) = (seekend(io.io); io) +skip(io::AnnotatedIOBuffer, n::Integer) = (skip(io.io, n); io) +copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations)) + +annotations(io::AnnotatedIOBuffer) = io.annotations + +annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = + (_annotate!(io.annotations, range, label, val); io) + +function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) + astr = AnnotatedString(astr) + offset = position(io.io) + eof(io) || _clear_annotations_in_region!(io.annotations, offset+1:offset+ncodeunits(astr)) + _insert_annotations!(io, astr.annotations) + write(io.io, String(astr)) +end + +write(io::AnnotatedIOBuffer, c::AnnotatedChar) = + write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) +write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x) +write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) +write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) + +function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) + destpos = position(dest) + isappending = eof(dest) + srcpos = position(src) + nb = write(dest.io, src.io) + isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) + srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) + for annot in src.annotations if first(annot.region) >= srcpos] + _insert_annotations!(dest, srcannots, destpos - srcpos) + nb +end + +# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) +# work as expected. +function write(io::AbstractPipe, s::Union{AnnotatedString, SubString{<:AnnotatedString}}) + if pipe_writer(io) isa AnnotatedIOBuffer + write(pipe_writer(io), s) + else + invoke(write, Tuple{IO, typeof(s)}, io, s) + end::Int +end + +# Can't be part of the `Union` above because it introduces method ambiguities +function write(io::AbstractPipe, c::AnnotatedChar) + if pipe_writer(io) isa AnnotatedIOBuffer + write(pipe_writer(io), c) + else + invoke(write, Tuple{IO, typeof(c)}, io, c) + end::Int +end + +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} + if (start = position(io)) == 0 + AnnotatedString(read(io.io, T), copy(io.annotations)) + else + annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) + for annot in io.annotations if last(annot.region) > start] + AnnotatedString(read(io.io, T), annots) + end +end +read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{AbstractString}}) = read(io, AnnotatedString{String}) +read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{String}) + +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar} + pos = position(io) + char = read(io.io, T) + annots = [NamedTuple{(:label, :value)}(annot) for annot in io.annotations if pos+1 in annot.region] + AnnotatedChar(char, annots) +end +read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{AbstractChar}}) = read(io, AnnotatedChar{Char}) +read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char}) + +function truncate(io::AnnotatedIOBuffer, size::Integer) + truncate(io.io, size) + filter!(ann -> first(ann.region) <= size, io.annotations) + map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), + io.annotations, io.annotations) + io +end + +""" + _clear_annotations_in_region!(annotations::Vector{$RegionAnnotation}, span::UnitRange{Int}) + +Erase the presence of `annotations` within a certain `span`. + +This operates by removing all elements of `annotations` that are entirely +contained in `span`, truncating ranges that partially overlap, and splitting +annotations that subsume `span` to just exist either side of `span`. +""" +function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, span::UnitRange{Int}) + # Clear out any overlapping pre-existing annotations. + filter!(ann -> first(ann.region) < first(span) || last(ann.region) > last(span), annotations) + extras = Tuple{Int, RegionAnnotation}[] + for i in eachindex(annotations) + annot = annotations[i] + region = annot.region + # Test for partial overlap + if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) + annotations[i] = + setindex(annot, + if first(region) < first(span) + first(region):first(span)-1 + else + last(span)+1:last(region) + end, + :region) + # If `span` fits exactly within `region`, then we've only copied over + # the beginning overhang, but also need to conserve the end overhang. + if first(region) < first(span) && last(span) < last(region) + push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) + end + end + end + # Insert any extra entries in the appropriate position + for (offset, (i, entry)) in enumerate(extras) + insert!(annotations, i + offset, entry) + end + annotations +end + +""" + _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{$RegionAnnotation}, offset::Int = position(io)) + +Register new `annotations` in `io`, applying an `offset` to their regions. + +The largely consists of simply shifting the regions of `annotations` by `offset` +and pushing them onto `io`'s annotations. However, when it is possible to merge +the new annotations with recent annotations in accordance with the semantics +outlined in [`AnnotatedString`](@ref), we do so. More specifically, when there +is a run of the most recent annotations that are also present as the first +`annotations`, with the same value and adjacent regions, the new annotations are +merged into the existing recent annotations by simply extending their range. + +This is implemented so that one can say write an `AnnotatedString` to an +`AnnotatedIOBuffer` one character at a time without needlessly producing a +new annotation for each character. +""" +function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionAnnotation}, offset::Int = position(io)) + run = 0 + if !isempty(io.annotations) && last(last(io.annotations).region) == offset + for i in reverse(axes(annotations, 1)) + annot = annotations[i] + first(annot.region) == 1 || continue + i <= length(io.annotations) || continue + if annot.label == last(io.annotations).label && annot.value == last(io.annotations).value + valid_run = true + for runlen in 1:i + new = annotations[begin+runlen-1] + old = io.annotations[end-i+runlen] + if last(old.region) != offset || first(new.region) != 1 || old.label != new.label || old.value != new.value + valid_run = false + break + end + end + if valid_run + run = i + break + end + end + end + end + for runindex in 0:run-1 + old_index = lastindex(io.annotations) - run + 1 + runindex + old = io.annotations[old_index] + new = annotations[begin+runindex] + io.annotations[old_index] = setindex(old, first(old.region):last(new.region)+offset, :region) + end + for index in run+1:lastindex(annotations) + annot = annotations[index] + start, stop = first(annot.region), last(annot.region) + push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) + end +end diff --git a/base/strings/strings.jl b/base/strings/strings.jl index 8dae311f475b4..32975b6ea3fc7 100644 --- a/base/strings/strings.jl +++ b/base/strings/strings.jl @@ -11,3 +11,4 @@ import .Iterators: PartitionIterator include("strings/util.jl") include("strings/io.jl") +include("strings/annotated_io.jl") diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 8658c1b52a2ab..7f53740b9eec1 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -258,3 +258,51 @@ end write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) @test sprint(show, aio) == "Base.AnnotatedIOBuffer(5 bytes, 1 annotation)" end + +@testset "Eachregion" begin + annregions(str::String, annots::Vector{<:Tuple{UnitRange{Int}, Symbol, <:Any}}) = + [(s, Tuple.(a)) for (s, a) in Base.eachregion(Base.AnnotatedString(str, annots))] + # Regions that do/don't extend to the left/right edges + @test annregions(" abc ", [(2:4, :face, :bold)]) == + [(" ", []), + ("abc", [(:face, :bold)]), + (" ", [])] + @test annregions(" x ", [(2:2, :face, :bold)]) == + [(" ", []), + ("x", [(:face, :bold)]), + (" ", [])] + @test annregions(" x", [(2:2, :face, :bold)]) == + [(" ", []), + ("x", [(:face, :bold)])] + @test annregions("x ", [(1:1, :face, :bold)]) == + [("x", [(:face, :bold)]), + (" ", [])] + @test annregions("x", [(1:1, :face, :bold)]) == + [("x", [(:face, :bold)])] + # Overlapping/nested regions + @test annregions(" abc ", [(2:4, :face, :bold), (3:3, :face, :italic)]) == + [(" ", []), + ("a", [(:face, :bold)]), + ("b", [(:face, :bold), (:face, :italic)]), + ("c", [(:face, :bold)]), + (" ", [])] + @test annregions("abc-xyz", [(1:7, :face, :bold), (1:3, :face, :green), (4:4, :face, :yellow), (4:7, :face, :italic)]) == + [("abc", [(:face, :bold), (:face, :green)]), + ("-", [(:face, :bold), (:face, :yellow), (:face, :italic)]), + ("xyz", [(:face, :bold), (:face, :italic)])] + # Preserving annotation order + @test annregions("abcd", [(1:3, :face, :red), (2:2, :face, :yellow), (2:3, :face, :green), (2:4, :face, :blue)]) == + [("a", [(:face, :red)]), + ("b", [(:face, :red), (:face, :yellow), (:face, :green), (:face, :blue)]), + ("c", [(:face, :red), (:face, :green), (:face, :blue)]), + ("d", [(:face, :blue)])] + @test annregions("abcd", [(2:4, :face, :blue), (1:3, :face, :red), (2:3, :face, :green), (2:2, :face, :yellow)]) == + [("a", [(:face, :red)]), + ("b", [(:face, :blue), (:face, :red), (:face, :green), (:face, :yellow)]), + ("c", [(:face, :blue), (:face, :red), (:face, :green)]), + ("d", [(:face, :blue)])] + # Region starting after a character spanning multiple codepoints. + @test annregions("𝟏x", [(1:4, :face, :red)]) == + [("𝟏", [(:face, :red)]), + ("x", [])] +end From 470ff4860a88e680c0ac3684cacc01d91425178e Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sat, 29 Mar 2025 21:16:55 +0100 Subject: [PATCH 017/662] `REPL`: typeassert `displaysize` return value (#57919) Improve abstract type inference. This is OK because `displaysize` is documented to always return a `Tuple{Int, Int}` value. Also add a test. --- stdlib/REPL/src/Terminals.jl | 2 +- stdlib/REPL/test/repl.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/Terminals.jl b/stdlib/REPL/src/Terminals.jl index aba6bff73a607..14ea6dd3dff77 100644 --- a/stdlib/REPL/src/Terminals.jl +++ b/stdlib/REPL/src/Terminals.jl @@ -146,7 +146,7 @@ end @eval clear_line(t::UnixTerminal) = write(t.out_stream, $"\r$(CSI)0K") beep(t::UnixTerminal) = write(t.err_stream,"\x7") -Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream) +Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream)::Tuple{Int,Int} hascolor(t::TTYTerminal) = get(t.out_stream, :color, false)::Bool diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index d1233ee00da1b..ee787e55d78c8 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1953,6 +1953,10 @@ end end end +@testset "`displaysize` return type inference" begin + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{REPL.Terminals.UnixTerminal}) +end + @testset "Dummy Pkg prompt" begin # do this in an empty depot to test default for new users withenv("JULIA_DEPOT_PATH" => mktempdir() * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do From 4d2a350870b97059432ac8fa8de92b0b161c335b Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 30 Mar 2025 08:55:46 +0200 Subject: [PATCH 018/662] `Base`: shell escaping: inference improvement to prevent invalidation (#57915) --- base/shell.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/shell.jl b/base/shell.jl index e07fff128acfe..68925cbd5d5af 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -344,7 +344,7 @@ function shell_escape_csh(io::IO, args::AbstractString...) end shell_escape_csh(args::AbstractString...) = sprint(shell_escape_csh, args...; - sizehint = sum(sizeof.(args)) + length(args) * 3) + sizehint = sum(sizeof, args) + length(args) * 3) """ shell_escape_wincmd(s::AbstractString) @@ -494,4 +494,4 @@ function escape_microsoft_c_args(io::IO, args::AbstractString...) end escape_microsoft_c_args(args::AbstractString...) = sprint(escape_microsoft_c_args, args...; - sizehint = (sum(sizeof.(args)) + 3*length(args))) + sizehint = (sum(sizeof, args) + 3*length(args))) From 0802b2e8df45f711675ab69383d9620ebeb834d9 Mon Sep 17 00:00:00 2001 From: GHTaarn <62629455+GHTaarn@users.noreply.github.com> Date: Sun, 30 Mar 2025 23:15:21 +0200 Subject: [PATCH 019/662] Fixed documentation for Test (#57938) Corrected indentation in example code Correct indentation makes examples look neater and is easier to read --- stdlib/Test/docs/src/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index 7451f8434f4fa..a2922deebbd31 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -478,13 +478,13 @@ Using our knowledge of `Test.jl`, here are some example tests we could add to `m @testset "Testset 1" begin @test 2 == simple_add(1, 1) @test 3.5 == simple_add(1, 2.5) - @test_throws MethodError simple_add(1, "A") - @test_throws MethodError simple_add(1, 2, 3) + @test_throws MethodError simple_add(1, "A") + @test_throws MethodError simple_add(1, 2, 3) end @testset "Testset 2" begin @test 1.0 == type_multiply(1.0, 1.0) - @test isa(type_multiply(2.0, 2.0), Float64) + @test isa(type_multiply(2.0, 2.0), Float64) @test_throws MethodError type_multiply(1, 2.5) end ``` From 6d83987359d156c35ff1224ec335672af5703a8c Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 30 Mar 2025 20:36:12 -0400 Subject: [PATCH 020/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=2093fb66db1=20to=206f309f17f=20(#57948)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 | 1 + .../Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 | 1 + .../Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 | 1 - .../Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 create mode 100644 deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 diff --git a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 new file mode 100644 index 0000000000000..f1a567935cfc6 --- /dev/null +++ b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 @@ -0,0 +1 @@ +cdbbe19efa29dcaa89fe167e15de449d diff --git a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 new file mode 100644 index 0000000000000..fc895ae726858 --- /dev/null +++ b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 @@ -0,0 +1 @@ +7ddce211c686a4c91ac7806dd134e4d0a3abcfe8d0a5c6568697a26e4dd0164c60fb2611421e5e7821819145cdbda85ecd1eec0724409346affcb457a41f5aee diff --git a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 deleted file mode 100644 index 470acd2b454dc..0000000000000 --- a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -e9b791bef7662fe5f14d2cff005c9fa9 diff --git a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 b/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 deleted file mode 100644 index 8fefef4d95722..0000000000000 --- a/deps/checksums/Pkg-93fb66db1834ddf432141a4809b2b6395b5d3260.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -43416e2e16e59356254323110e7158148099a57769cf005abfe6c4f3dfc3877269420779e2925c0b371fc18117422fc6a82135cc978d2b32ef7c49a475f1c41e diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 26b676128c1d8..d356689c391f6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 93fb66db1834ddf432141a4809b2b6395b5d3260 +PKG_SHA1 = 6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From df1b07ec4a5953df89f0c32f742d7a0b16111a70 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 31 Mar 2025 03:58:55 -0400 Subject: [PATCH 021/662] fix trimming for scheduler task (#57926) This unfortunately passed tests because it segfaults (sometimes) on another thread while the main thread successfully executes. Soon we'll have a general solution for tasks but this fixes it for now. --- base/task.jl | 2 +- contrib/juliac-buildscript.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/task.jl b/base/task.jl index 15ac907c18988..cddf1fc854f4c 100644 --- a/base/task.jl +++ b/base/task.jl @@ -1152,7 +1152,7 @@ end end const get_sched_task = OncePerThread{Task}() do - @task wait_forever() + Task(wait_forever) end function ensure_rescheduled(othertask::Task) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 363330e4a3a79..0549afc0e1508 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -197,6 +197,7 @@ let mod = Base.include(Base.__toplevel__, inputfile) #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) entrypoint(Base.task_done_hook, (Task,)) entrypoint(Base.wait, ()) + entrypoint(Base.wait_forever, ()) entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) entrypoint(Base.checktaskempty, ()) if add_ccallables From fcf492d4b39c2becde6cd5ca3bb63fcbc7a308d3 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:00:17 +0200 Subject: [PATCH 022/662] `_precompilepkgs`: interactive progress display: fix unintended capture (#57932) The variable `str` also exists in one of the enclosing closures. Use a new variable, as was surely intended, instead of capturing and mutating the `str`. Improves the sysimage's resistance to invalidation. --- base/precompilation.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 5392e119d25a2..3ab9fcad5aee6 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -835,8 +835,8 @@ function _precompilepkgs(pkgs::Vector{String}, # window between print cycles termwidth = displaysize(io)[2] - 4 if !final_loop - str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) - print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") + s = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) + print(iostr, Base._truncate_at_width_or_chars(true, s, termwidth), "\n") end for pkg_config in pkg_queue_show dep, config = pkg_config From fe613d4129f38efccb7821f0d37523abc649cb91 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 31 Mar 2025 04:01:04 -0400 Subject: [PATCH 023/662] fix trimming size regression due to handling binding backedges in the wrong place (#57927) --- src/staticdata.c | 4 +++- test/trimming/trimming.jl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 5b6bb5fdf2187..edef580600b84 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -798,7 +798,6 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ // ... or point to Base functions accessed by the runtime (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { - record_field_change((jl_value_t**)&b->backedges, NULL); jl_queue_for_serialization(s, b); } } @@ -1044,6 +1043,9 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ record_field_change((jl_value_t **)&tn->mt, NULL); } } + else if (jl_is_binding(v)) { + record_field_change((jl_value_t**)&((jl_binding_t*)v)->backedges, NULL); + } } char *data = (char*)jl_data_ptr(v); size_t i, np = layout->npointers; diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index 4a0bb14ceebe2..69d9adf9b003f 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -4,7 +4,7 @@ let exe_suffix = splitext(Base.julia_exename())[2] hello_exe = joinpath(@__DIR__, "hello" * exe_suffix) @test readchomp(`$hello_exe`) == "Hello, world!" - @test filesize(hello_exe) < 20_000_000 + @test filesize(hello_exe) < 2_000_000 basic_jll_exe = joinpath(@__DIR__, "basic_jll" * exe_suffix) lines = split(readchomp(`$basic_jll_exe`), "\n") From 89271dc5a8a3236a148413409abb019bf973f6de Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 31 Mar 2025 04:03:19 -0400 Subject: [PATCH 024/662] staticdata: Memoize `type_in_worklist` query (#57917) When pre-compiling `stdlib/` this cache has a 91% hit rate, so this seems fairly profitable. It also dramatically improves some pathological cases, a few of which have been hit in the wild (arguably due to inference bugs) Without this PR, this package takes exponentially long to pre-compile: ```julia function BigType(N) (N == 0) && return Nothing T = BigType(N-1) return Pair{T,T} end foo(::Type{T}) where T = T precompile(foo, (Type{BigType(40)},)) ``` For an in-the-wild test case hit by a customer, this reduces pre-compilation time from over an hour to just ~two and a half minutes. Resolves #53331. --- src/staticdata.c | 76 +++++++++++++++++++++++++++--------------- src/staticdata_utils.c | 70 +++++++++++++++++++++++--------------- 2 files changed, 94 insertions(+), 52 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index edef580600b84..b007fc04eeb4b 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -90,6 +90,22 @@ static const size_t WORLD_AGE_REVALIDATION_SENTINEL = 0x1; JL_DLLEXPORT size_t jl_require_world = ~(size_t)0; JL_DLLEXPORT _Atomic(size_t) jl_first_image_replacement_world = ~(size_t)0; +// This structure is used to store hash tables for the memoization +// of queries in staticdata.c (currently only `type_in_worklist`). +typedef struct { + htable_t type_in_worklist; +} jl_query_cache; + +static void init_query_cache(jl_query_cache *cache) +{ + htable_new(&cache->type_in_worklist, 0); +} + +static void destroy_query_cache(jl_query_cache *cache) +{ + htable_free(&cache->type_in_worklist); +} + #include "staticdata_utils.c" #include "precompile_utils.c" @@ -552,6 +568,7 @@ typedef struct { jl_array_t *method_roots_list; htable_t method_roots_index; uint64_t worklist_key; + jl_query_cache *query_cache; jl_ptls_t ptls; jl_image_t *image; int8_t incremental; @@ -675,14 +692,13 @@ static int jl_needs_serialization(jl_serializer_state *s, jl_value_t *v) JL_NOTS return 1; } - -static int caching_tag(jl_value_t *v) JL_NOTSAFEPOINT +static int caching_tag(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; jl_value_t *m = mi->def.value; if (jl_is_method(m) && jl_object_in_image(m)) - return 1 + type_in_worklist(mi->specTypes); + return 1 + type_in_worklist(mi->specTypes, query_cache); } if (jl_is_binding(v)) { jl_globalref_t *gr = ((jl_binding_t*)v)->globalref; @@ -697,24 +713,24 @@ static int caching_tag(jl_value_t *v) JL_NOTSAFEPOINT if (jl_is_tuple_type(dt) ? !dt->isconcretetype : dt->hasfreetypevars) return 0; // aka !is_cacheable from jltypes.c if (jl_object_in_image((jl_value_t*)dt->name)) - return 1 + type_in_worklist(v); + return 1 + type_in_worklist(v, query_cache); } jl_value_t *dtv = jl_typeof(v); if (jl_is_datatype_singleton((jl_datatype_t*)dtv)) { - return 1 - type_in_worklist(dtv); // these are already recached in the datatype in the image + return 1 - type_in_worklist(dtv, query_cache); // these are already recached in the datatype in the image } return 0; } -static int needs_recaching(jl_value_t *v) JL_NOTSAFEPOINT +static int needs_recaching(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { - return caching_tag(v) == 2; + return caching_tag(v, query_cache) == 2; } -static int needs_uniquing(jl_value_t *v) JL_NOTSAFEPOINT +static int needs_uniquing(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { assert(!jl_object_in_image(v)); - return caching_tag(v) == 1; + return caching_tag(v, query_cache) == 1; } static void record_field_change(jl_value_t **addr, jl_value_t *newval) JL_NOTSAFEPOINT @@ -839,7 +855,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ jl_datatype_t *dt = (jl_datatype_t*)v; // ensure all type parameters are recached jl_queue_for_serialization_(s, (jl_value_t*)dt->parameters, 1, 1); - if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance)) { + if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance, s->query_cache)) { assert(jl_needs_serialization(s, dt->instance)); // should be true, since we visited dt // do not visit dt->instance for our template object as it leads to unwanted cycles here // (it may get serialized from elsewhere though) @@ -850,7 +866,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (s->incremental && jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; jl_value_t *def = mi->def.value; - if (needs_uniquing(v)) { + if (needs_uniquing(v, s->query_cache)) { // we only need 3 specific fields of this (the rest are not used) jl_queue_for_serialization(s, mi->def.value); jl_queue_for_serialization(s, mi->specTypes); @@ -865,7 +881,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ record_field_change((jl_value_t**)&mi->cache, NULL); } else { - assert(!needs_recaching(v)); + assert(!needs_recaching(v, s->query_cache)); } // n.b. opaque closures cannot be inspected and relied upon like a // normal method since they can get improperly introduced by generated @@ -875,7 +891,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ // error now. } if (s->incremental && jl_is_binding(v)) { - if (needs_uniquing(v)) { + if (needs_uniquing(v, s->query_cache)) { jl_binding_t *b = (jl_binding_t*)v; jl_queue_for_serialization(s, b->globalref->mod); jl_queue_for_serialization(s, b->globalref->name); @@ -1102,9 +1118,9 @@ static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, i // Items that require postorder traversal must visit their children prior to insertion into // the worklist/serialization_order (and also before their first use) if (s->incremental && !immediate) { - if (jl_is_datatype(t) && needs_uniquing(v)) + if (jl_is_datatype(t) && needs_uniquing(v, s->query_cache)) immediate = 1; - if (jl_is_datatype_singleton((jl_datatype_t*)t) && needs_uniquing(v)) + if (jl_is_datatype_singleton((jl_datatype_t*)t) && needs_uniquing(v, s->query_cache)) immediate = 1; } @@ -1267,7 +1283,7 @@ static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t * static void record_uniquing(jl_serializer_state *s, jl_value_t *fld, uintptr_t offset) JL_NOTSAFEPOINT { - if (s->incremental && jl_needs_serialization(s, fld) && needs_uniquing(fld)) { + if (s->incremental && jl_needs_serialization(s, fld) && needs_uniquing(fld, s->query_cache)) { if (jl_is_datatype(fld) || jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(fld))) arraylist_push(&s->uniquing_types, (void*)(uintptr_t)offset); else if (jl_is_method_instance(fld) || jl_is_binding(fld)) @@ -1491,7 +1507,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // write header if (object_id_expected) write_uint(f, jl_object_id(v)); - if (s->incremental && jl_needs_serialization(s, (jl_value_t*)t) && needs_uniquing((jl_value_t*)t)) + if (s->incremental && jl_needs_serialization(s, (jl_value_t*)t) && needs_uniquing((jl_value_t*)t, s->query_cache)) arraylist_push(&s->uniquing_types, (void*)(uintptr_t)(ios_pos(f)|1)); if (f == s->const_data) write_uint(s->const_data, ((uintptr_t)t->smalltag << 4) | GC_OLD_MARKED | GC_IN_IMAGE); @@ -1502,7 +1518,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED layout_table.items[item] = (void*)(reloc_offset | (f == s->const_data)); // store the inverse mapping of `serialization_order` (`id` => object-as-streampos) if (s->incremental) { - if (needs_uniquing(v)) { + if (needs_uniquing(v, s->query_cache)) { if (jl_typetagis(v, jl_binding_type)) { jl_binding_t *b = (jl_binding_t*)v; if (b->globalref == NULL) @@ -1531,7 +1547,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(jl_is_datatype_singleton(t) && "unreachable"); } } - else if (needs_recaching(v)) { + else if (needs_recaching(v, s->query_cache)) { arraylist_push(jl_is_datatype(v) ? &s->fixup_types : &s->fixup_objs, (void*)reloc_offset); } } @@ -1964,7 +1980,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } void *superidx = ptrhash_get(&serialization_order, dt->super); - if (s->incremental && superidx != HT_NOTFOUND && from_seroder_entry(superidx) > item && needs_uniquing((jl_value_t*)dt->super)) + if (s->incremental && superidx != HT_NOTFOUND && from_seroder_entry(superidx) > item && needs_uniquing((jl_value_t*)dt->super, s->query_cache)) arraylist_push(&s->uniquing_super, dt->super); } else if (jl_is_typename(v)) { @@ -2875,13 +2891,14 @@ JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val, int insert) static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *newly_inferred, /* outputs */ jl_array_t **extext_methods JL_REQUIRE_ROOTED_SLOT, jl_array_t **new_ext_cis JL_REQUIRE_ROOTED_SLOT, - jl_array_t **edges JL_REQUIRE_ROOTED_SLOT) + jl_array_t **edges JL_REQUIRE_ROOTED_SLOT, + jl_query_cache *query_cache) { // extext_methods: [method1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist // edges: [caller1, ext_targets, ...] for worklist-owned methods calling external methods // Save the inferred code from newly inferred, external methods - *new_ext_cis = queue_external_cis(newly_inferred); + *new_ext_cis = queue_external_cis(newly_inferred, query_cache); // Collect method extensions and edges data *extext_methods = jl_alloc_vec_any(0); @@ -2911,7 +2928,8 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new // In addition to the system image (where `worklist = NULL`), this can also save incremental images with external linkage static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_array_t *worklist, jl_array_t *extext_methods, - jl_array_t *new_ext_cis, jl_array_t *edges) + jl_array_t *new_ext_cis, jl_array_t *edges, + jl_query_cache *query_cache) { htable_new(&field_replace, 0); htable_new(&bits_replace, 0); @@ -3018,6 +3036,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, ios_mem(&gvar_record, 0); ios_mem(&fptr_record, 0); jl_serializer_state s = {0}; + s.query_cache = query_cache; s.incremental = !(worklist == NULL); s.s = &sysimg; s.const_data = &const_data; @@ -3375,11 +3394,14 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli int64_t datastartpos = 0; JL_GC_PUSH4(&mod_array, &extext_methods, &new_ext_cis, &edges); + jl_query_cache query_cache; + init_query_cache(&query_cache); + if (worklist) { mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) // Generate _native_data` if (_native_data != NULL) { - jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL); + jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL, &query_cache); jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); *_native_data = jl_precompile_worklist(worklist, extext_methods, new_ext_cis); jl_precompile_toplevel_module = NULL; @@ -3410,7 +3432,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli assert((ct->reentrant_timing & 0b1110) == 0); ct->reentrant_timing |= 0b1000; if (worklist) { - jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, &edges); + jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, &edges, &query_cache); if (!emit_split) { write_int32(f, 0); // No clone_targets write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); @@ -3422,7 +3444,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, edges); + jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, edges, &query_cache); if (_native_data != NULL) native_functions = NULL; // make sure we don't run any Julia code concurrently before this point @@ -3451,6 +3473,8 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } } + destroy_query_cache(&query_cache); + JL_GC_POP(); *s = f; if (emit_split) diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index e9f464b64470e..9bfd4c355efe6 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -131,63 +131,81 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) JL_UNLOCK(&newly_inferred_mutex); } - // compute whether a type references something internal to worklist // and thus could not have existed before deserialize // and thus does not need delayed unique-ing -static int type_in_worklist(jl_value_t *v) JL_NOTSAFEPOINT +static int type_in_worklist(jl_value_t *v, jl_query_cache *cache) JL_NOTSAFEPOINT { if (jl_object_in_image(v)) return 0; // fast-path for rejection + + void *cached = HT_NOTFOUND; + if (cache != NULL) + cached = ptrhash_get(&cache->type_in_worklist, v); + + // fast-path for memoized results + if (cached != HT_NOTFOUND) + return cached == v; + + int result = 0; if (jl_is_uniontype(v)) { jl_uniontype_t *u = (jl_uniontype_t*)v; - return type_in_worklist(u->a) || - type_in_worklist(u->b); + result = type_in_worklist(u->a, cache) || + type_in_worklist(u->b, cache); } else if (jl_is_unionall(v)) { jl_unionall_t *ua = (jl_unionall_t*)v; - return type_in_worklist((jl_value_t*)ua->var) || - type_in_worklist(ua->body); + result = type_in_worklist((jl_value_t*)ua->var, cache) || + type_in_worklist(ua->body, cache); } else if (jl_is_typevar(v)) { jl_tvar_t *tv = (jl_tvar_t*)v; - return type_in_worklist(tv->lb) || - type_in_worklist(tv->ub); + result = type_in_worklist(tv->lb, cache) || + type_in_worklist(tv->ub, cache); } else if (jl_is_vararg(v)) { jl_vararg_t *tv = (jl_vararg_t*)v; - if (tv->T && type_in_worklist(tv->T)) - return 1; - if (tv->N && type_in_worklist(tv->N)) - return 1; + result = ((tv->T && type_in_worklist(tv->T, cache)) || + (tv->N && type_in_worklist(tv->N, cache))); } else if (jl_is_datatype(v)) { jl_datatype_t *dt = (jl_datatype_t*)v; - if (!jl_object_in_image((jl_value_t*)dt->name)) - return 1; - jl_svec_t *tt = dt->parameters; - size_t i, l = jl_svec_len(tt); - for (i = 0; i < l; i++) - if (type_in_worklist(jl_tparam(dt, i))) - return 1; + if (!jl_object_in_image((jl_value_t*)dt->name)) { + result = 1; + } + else { + jl_svec_t *tt = dt->parameters; + size_t i, l = jl_svec_len(tt); + for (i = 0; i < l; i++) { + if (type_in_worklist(jl_tparam(dt, i), cache)) { + result = 1; + break; + } + } + } } else { - return type_in_worklist(jl_typeof(v)); + return type_in_worklist(jl_typeof(v), cache); } - return 0; + + // Memoize result + if (cache != NULL) + ptrhash_put(&cache->type_in_worklist, (void*)v, result ? (void*)v : NULL); + + return result; } // When we infer external method instances, ensure they link back to the // package. Otherwise they might be, e.g., for external macros. // Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable -static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_t *stack) +static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_t *stack, jl_query_cache *query_cache) { jl_module_t *mod = mi->def.module; if (jl_is_method(mod)) mod = ((jl_method_t*)mod)->module; assert(jl_is_module(mod)); uint8_t is_precompiled = jl_atomic_load_relaxed(&mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; - if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(mi->specTypes)) { + if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(mi->specTypes, query_cache)) { return 1; } if (!mi->backedges) { @@ -211,7 +229,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, jl_code_instance_t *be; i = get_next_edge(mi->backedges, i, NULL, &be); JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here - int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack); + int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack, query_cache); if (child_found == 1 || child_found == 2) { // found what we were looking for, so terminate early found = 1; @@ -243,7 +261,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, // from the worklist or explicitly added by a `precompile` statement, and // (4) are the most recently computed result for that method. // These will be preserved in the image. -static jl_array_t *queue_external_cis(jl_array_t *list) +static jl_array_t *queue_external_cis(jl_array_t *list, jl_query_cache *query_cache) { if (list == NULL) return NULL; @@ -262,7 +280,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { - int found = has_backedge_to_worklist(mi, &visited, &stack); + int found = has_backedge_to_worklist(mi, &visited, &stack, query_cache); assert(found == 0 || found == 1 || found == 2); assert(stack.len == 0); if (found == 1 && jl_atomic_load_relaxed(&ci->max_world) == ~(size_t)0) { From 3a3926eba563e147d7d6a48c44fb7f47088d24c5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 31 Mar 2025 04:04:21 -0400 Subject: [PATCH 025/662] fix opaque_closure sparam capture (#57928) Fix #54236 Fix #54357 --- src/julia-syntax.scm | 1 + test/opaque_closure.jl | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 0e1618ed140a6..0b6f56ac838a6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -4109,6 +4109,7 @@ f(x) = yt(x) (capt-var-access v fname opaq) v))) cvs))) + (set-car! (cdddr (lam:vinfo lam2)) '()) ;; must capture static_parameters as values inside opaque_closure `(new_opaque_closure ,(cadr e) ,(or (caddr e) '(call (core apply_type) (core Union))) ,(or (cadddr e) '(core Any)) ,allow-partial (opaque_closure_method (null) ,nargs ,isva ,functionloc ,(convert-lambda lam2 (car (lam:args lam2)) #f '() (symbol-to-idx-map cvs) parsed-method-stack)) diff --git a/test/opaque_closure.jl b/test/opaque_closure.jl index 6c988b068a668..7b02578a86621 100644 --- a/test/opaque_closure.jl +++ b/test/opaque_closure.jl @@ -390,3 +390,20 @@ let ir = first(only(Base.code_ircode(sin, (Int,)))) oc = Core.OpaqueClosure(ir; do_compile=false) @test oc(1) == sin(1) end + +function typed_add54236(::Type{T}) where T + return @opaque (x::Int)->T(x) + T(1) +end +let f = typed_add54236(Float64) + @test f isa Core.OpaqueClosure + @test f(32) === 33.0 +end + +f54357(g, ::Type{AT}) where {AT} = Base.Experimental.@opaque AT->_ (args...) -> g((args::AT)...) +let f = f54357(+, Tuple{Int,Int}) + @test f isa Core.OpaqueClosure + @test f(32, 34) === 66 + g = f54357(+, Tuple{Float64,Float64}) + @test g isa Core.OpaqueClosure + @test g(32.0, 34.0) === 66.0 +end From 2f34760fb2c16646f1b0ecdd3667bb7b509dbc27 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:19:26 +0200 Subject: [PATCH 026/662] `Base`: typeassert `displaysize` return value (#57920) Improve abstract type inference. This is OK because `displaysize` is documented to always return a `Tuple{Int, Int}` value. Also add tests. --- base/show.jl | 2 +- test/show.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index 7dfe36fb696c5..fa666bea22985 100644 --- a/base/show.jl +++ b/base/show.jl @@ -442,7 +442,7 @@ get(io::IO, key, default) = default keys(io::IOContext) = keys(io.dict) keys(io::IO) = keys(ImmutableDict{Symbol,Any}()) -displaysize(io::IOContext) = haskey(io, :displaysize) ? io[:displaysize]::Tuple{Int,Int} : displaysize(io.io) +displaysize(io::IOContext) = haskey(io, :displaysize) ? io[:displaysize]::Tuple{Int,Int} : displaysize(io.io)::Tuple{Int,Int} show_circular(io::IO, @nospecialize(x)) = false function show_circular(io::IOContext, @nospecialize(x)) diff --git a/test/show.jl b/test/show.jl index 24a7b574f759c..44996a7b630b2 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1708,6 +1708,13 @@ end "[3.141592653589793 3.141592653589793; 3.141592653589793 3.141592653589793]" end +@testset "`displaysize` return type inference" begin + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{IO}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{IOContext}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{Base.TTY}) +end + @testset "Array printing with limited rows" begin arrstr = let buf = IOBuffer() function (A, rows) From 73ac39e88c52626b11b933b0e0dbfd20c2166fd8 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 31 Mar 2025 16:51:09 -0400 Subject: [PATCH 027/662] CI: Record one result with duration (#57954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helps fill this out with something meaningful, which is better than the current zero duration and the old overcounting. https://buildkite.com/organizations/julialang/analytics/suites/julialang-base?branch=master Screenshot 2025-03-30 at 11 19 20 PM --- test/buildkitetestjson.jl | 9 ++++++++- test/runtests.jl | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index e9f969483c2aa..dbc08761c4dff 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -258,9 +258,16 @@ function serialize_testset_result_file(dir::String, testset::Test.DefaultTestSet end # deserilalizes the results files and writes them to collated JSON files of 5000 max results -function write_testset_json_files(dir::String) +function write_testset_json_files(dir::String, testset::Test.DefaultTestSet) data = Dict{String,Any}[] read_files = String[] + # Set one result to represent the overall duration, given results have no duration + overall_ts = result_dict(testset) + # don't set location or file name for this result. They aren't required by BK + overall_ts["result"] = "unknown" + overall_ts["name"] = replace(get(ENV, "BUILDKITE_LABEL", "job label not found"), r":\w+:\s*" => "") + push!(data, overall_ts) + # Load all the serialized results files for res_dat in filter!(x -> occursin(r"^results.*\.dat$", x), readdir(dir)) res_file = joinpath(dir, res_dat) append!(data, Serialization.deserialize(res_file)) diff --git a/test/runtests.jl b/test/runtests.jl index 60139a4691bd7..dec69cf4f306d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -342,8 +342,6 @@ cd(@__DIR__) do end end - BuildkiteTestJSON.write_testset_json_files(@__DIR__) - #= ` Construct a testset on the master node which will hold results from all the test files run on workers and on node1. The loop goes through the results, @@ -369,6 +367,7 @@ cd(@__DIR__) do Test.TESTSET_PRINT_ENABLE[] = false o_ts = Test.DefaultTestSet("Overall") o_ts.time_end = o_ts.time_start + o_ts_duration # manually populate the timing + BuildkiteTestJSON.write_testset_json_files(@__DIR__, o_ts) Test.push_testset(o_ts) completed_tests = Set{String}() for (testname, (resp,), duration) in results From 1a4e0943a7d15f9affce6efee1a853972feba42e Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 1 Apr 2025 07:32:52 +0530 Subject: [PATCH 028/662] Add `AbstractOneTo` and have `OneTo` be its subtype (#56902) Currently, `Base` defines `similar` for `Base.OneTo`, with the understanding that offset axes will be handled elsewhere. However, `Base.OneTo` is just one possible one-based range, and there are others such as `StaticArrays.SOneTo` that exist in the ecosystem. `Base` doesn't provide a method to handle a combination of different one-based ranges in `similar`, which leaves the packages in an awkward position: they need to define methods like ```julia similar(A::AbstractArray, ::Type{T}, shape::HeterogeneousShapeTuple) where {T} = similar(A, T, homogenize_shape(shape)) ``` where `HeterogeneousShapeTuple` is defined as ```julia Tuple{Vararg{Union{Integer, Base.OneTo, SOneTo}}} ``` https://github.com/JuliaArrays/StaticArrays.jl/blob/07c12450d1b3481dda4b503564ae4a5cb4e27ce4/src/abstractarray.jl#L141-L146 Unfortunately, such methods are borderline type-piracy, as noted in https://github.com/JuliaArrays/StaticArrays.jl/issues/1248. In particular, if the narrower `Base` method that handles `Union{Integer, OneTo}` is removed, then this method explicitly becomes pirating. A solution to this situation is to have `Base` handle all one-based ranges, such that arbitrary combinations of one-based ranges hit fallback methods in `Base`. This PR is a first step in this direction. We add the abstract type `AbstractOneTo`, and have `OneTo` be its subtype. We also add methods to `similar` and `reshape` that accept `AbstractOneTo` arguments. This makes it unnecessary for packages to dispatch on awkward combinations of `Union{Integer, OneTo}` and custom one-based axes, as the base implementation would handle such cases already. There may be other methods that accept an `AbstractOneTo` instead of a `OneTo`, but these may be addressed in separate PRs. Also, there may be one-based ranges that can't subtype `AbstractOneTo`, and a full solution that accepts such ranges as well needs to be implemented through a trait. This may also be handled in a separate PR. --------- Co-authored-by: Tim Holy --- NEWS.md | 3 ++- base/abstractarray.jl | 15 ++++++++++----- base/range.jl | 9 ++++++++- base/reshapedarray.jl | 3 +++ test/abstractarray.jl | 21 +++++++++++++++++++++ test/testhelpers/SizedArrays.jl | 14 +++++--------- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2aec4f53588a6..7e6aae67f0def 100644 --- a/NEWS.md +++ b/NEWS.md @@ -25,7 +25,8 @@ New library functions New library features -------------------- -`sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). +* `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). +* `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). Standard library changes ------------------------ diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 46907af4aacf6..18fa1a79534a1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -818,15 +818,18 @@ julia> similar(falses(10), Float64, 2, 4) See also: [`undef`](@ref), [`isassigned`](@ref). """ similar(a::AbstractArray{T}) where {T} = similar(a, T) -similar(a::AbstractArray, ::Type{T}) where {T} = similar(a, T, to_shape(axes(a))) -similar(a::AbstractArray{T}, dims::Tuple) where {T} = similar(a, T, to_shape(dims)) -similar(a::AbstractArray{T}, dims::DimOrInd...) where {T} = similar(a, T, to_shape(dims)) -similar(a::AbstractArray, ::Type{T}, dims::DimOrInd...) where {T} = similar(a, T, to_shape(dims)) +similar(a::AbstractArray, ::Type{T}) where {T} = similar(a, T, axes(a)) +similar(a::AbstractArray{T}, dims::Tuple) where {T} = similar(a, T, dims) +similar(a::AbstractArray{T}, dims::DimOrInd...) where {T} = similar(a, T, dims) +similar(a::AbstractArray, ::Type{T}, dims::DimOrInd...) where {T} = similar(a, T, dims) # Similar supports specifying dims as either Integers or AbstractUnitRanges or any mixed combination # thereof. Ideally, we'd just convert Integers to OneTos and then call a canonical method with the axes, # but we don't want to require all AbstractArray subtypes to dispatch on Base.OneTo. So instead we # define this method to convert supported axes to Ints, with the expectation that an offset array # package will define a method with dims::Tuple{Union{Integer, UnitRange}, Vararg{Union{Integer, UnitRange}}} +similar(a::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, AbstractOneTo}, Vararg{Union{Integer, AbstractOneTo}}}) where {T} = similar(a, T, to_shape(dims)) +# legacy method for packages that specialize similar(A::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, OneTo, CustomAxis}, Vararg{Union{Integer, OneTo, CustomAxis}}} +# leaving this method in ensures that Base owns the more specific method similar(a::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, OneTo}, Vararg{Union{Integer, OneTo}}}) where {T} = similar(a, T, to_shape(dims)) # similar creates an Array by default similar(a::AbstractArray, ::Type{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) @@ -837,7 +840,7 @@ to_shape(dims::DimsOrInds) = map(to_shape, dims)::DimsOrInds # each dimension to_shape(i::Int) = i to_shape(i::Integer) = Int(i) -to_shape(r::OneTo) = Int(last(r)) +to_shape(r::AbstractOneTo) = Int(last(r)) to_shape(r::AbstractUnitRange) = r """ @@ -863,6 +866,8 @@ would create a 1-dimensional logical array whose indices match those of the columns of `A`. """ similar(::Type{T}, dims::DimOrInd...) where {T<:AbstractArray} = similar(T, dims) +similar(::Type{T}, shape::Tuple{Union{Integer, AbstractOneTo}, Vararg{Union{Integer, AbstractOneTo}}}) where {T<:AbstractArray} = similar(T, to_shape(shape)) +# legacy method for packages that specialize similar(::Type{T}, dims::Tuple{Union{Integer, OneTo, CustomAxis}, Vararg{Union{Integer, OneTo, CustomAxis}}) similar(::Type{T}, shape::Tuple{Union{Integer, OneTo}, Vararg{Union{Integer, OneTo}}}) where {T<:AbstractArray} = similar(T, to_shape(shape)) similar(::Type{T}, dims::Dims) where {T<:AbstractArray} = T(undef, dims) diff --git a/base/range.jl b/base/range.jl index 1f776879e7bac..d86e0092d9897 100644 --- a/base/range.jl +++ b/base/range.jl @@ -451,6 +451,13 @@ if isdefined(Main, :Base) end end +""" + Base.AbstractOneTo + +Abstract type for ranges that start at 1 and have a step size of 1. +""" +abstract type AbstractOneTo{T} <: AbstractUnitRange{T} end + """ Base.OneTo(n) @@ -458,7 +465,7 @@ Define an `AbstractUnitRange` that behaves like `1:n`, with the added distinction that the lower limit is guaranteed (by the type system) to be 1. """ -struct OneTo{T<:Integer} <: AbstractUnitRange{T} +struct OneTo{T<:Integer} <: AbstractOneTo{T} stop::T # invariant: stop >= zero(stop) function OneTo{T}(stop) where {T<:Integer} throwbool(r) = (@noinline; throw(ArgumentError("invalid index: $r of type Bool"))) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index d5292a73052eb..9146dd5497078 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -120,6 +120,9 @@ julia> reshape(1:6, 2, 3) reshape reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims) +reshape(parent::AbstractArray, shp::Tuple{Union{Integer,AbstractOneTo}, Vararg{Union{Integer,AbstractOneTo}}}) = reshape(parent, to_shape(shp)) +# legacy method for packages that specialize reshape(parent::AbstractArray, shp::Tuple{Union{Integer,OneTo,CustomAxis}, Vararg{Union{Integer,OneTo,CustomAxis}}}) +# leaving this method in ensures that Base owns the more specific method reshape(parent::AbstractArray, shp::Tuple{Union{Integer,OneTo}, Vararg{Union{Integer,OneTo}}}) = reshape(parent, to_shape(shp)) reshape(parent::AbstractArray, dims::Tuple{Integer, Vararg{Integer}}) = reshape(parent, map(Int, dims)) reshape(parent::AbstractArray, dims::Dims) = _reshape(parent, dims) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index d1f30eacafacc..fc3a893b2250f 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2227,3 +2227,24 @@ end @test_throws MethodError isreal(G) end end + +@testset "similar/reshape for AbstractOneTo" begin + A = [1,2] + @testset "reshape" begin + @test reshape(A, 2, SizedArrays.SOneTo(1)) == reshape(A, 2, 1) + @test reshape(A, Base.OneTo(2), SizedArrays.SOneTo(1)) == reshape(A, 2, 1) + @test reshape(A, SizedArrays.SOneTo(1), 2) == reshape(A, 1, 2) + @test reshape(A, SizedArrays.SOneTo(1), Base.OneTo(2)) == reshape(A, 1, 2) + end + @testset "similar" begin + b = similar(A, SizedArrays.SOneTo(1), big(2)) + @test b isa Array{Int, 2} + @test size(b) == (1, 2) + b = similar(A, SizedArrays.SOneTo(1), Base.OneTo(2)) + @test b isa Array{Int, 2} + @test size(b) == (1, 2) + b = similar(A, SizedArrays.SOneTo(1), 2, Base.OneTo(2)) + @test b isa Array{Int, 3} + @test size(b) == (1, 2, 2) + end +end diff --git a/test/testhelpers/SizedArrays.jl b/test/testhelpers/SizedArrays.jl index e52e965a64859..961784b89ab68 100644 --- a/test/testhelpers/SizedArrays.jl +++ b/test/testhelpers/SizedArrays.jl @@ -14,7 +14,7 @@ import LinearAlgebra: mul! export SizedArray -struct SOneTo{N} <: AbstractUnitRange{Int} end +struct SOneTo{N} <: Base.AbstractOneTo{Int} end SOneTo(N) = SOneTo{N}() Base.length(::SOneTo{N}) where {N} = N Base.size(r::SOneTo) = (length(r),) @@ -58,14 +58,6 @@ Base.parent(S::SizedArray) = S.data +(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = SizedArray{SZ}(S1.data + S2.data) ==(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = S1.data == S2.data -homogenize_shape(t::Tuple) = (_homogenize_shape(first(t)), homogenize_shape(Base.tail(t))...) -homogenize_shape(::Tuple{}) = () -_homogenize_shape(x::Integer) = x -_homogenize_shape(x::AbstractUnitRange) = length(x) -const Dims = Union{Integer, Base.OneTo, SOneTo} -function Base.similar(::Type{A}, shape::Tuple{Dims, Vararg{Dims}}) where {A<:AbstractArray} - similar(A, homogenize_shape(shape)) -end function Base.similar(::Type{A}, shape::Tuple{SOneTo, Vararg{SOneTo}}) where {A<:AbstractArray} R = similar(A, length.(shape)) SizedArray{length.(shape)}(R) @@ -74,6 +66,10 @@ function Base.similar(x::SizedArray, ::Type{T}, shape::Tuple{SOneTo, Vararg{SOne sz = map(length, shape) SizedArray{sz}(similar(parent(x), T, sz)) end +function Base.reshape(x::AbstractArray, shape::Tuple{SOneTo, Vararg{SOneTo}}) + sz = map(length, shape) + SizedArray{length.(sz)}(reshape(x, length.(sz))) +end const SizedMatrixLike = Union{SizedMatrix, Transpose{<:Any, <:SizedMatrix}, Adjoint{<:Any, <:SizedMatrix}} From 69a22cf4bcaaf54e9bf08f2d4ed9e4e77f0ad1e4 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:05:56 +0200 Subject: [PATCH 029/662] `append_c_digits`: typeassert `Int` to improve inference (#57950) Makes the sysimage more resistant to method invalidation, when defining a new `Integer` subtype with a right bitshift method. --- base/intfuncs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 9733790b03b04..e14f48170f0e8 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -845,7 +845,7 @@ function append_c_digits(olength::Int, digits::Unsigned, buf, pos::Int) while i >= 2 d, c = divrem(digits, 0x64) digits = oftype(digits, d) - @inbounds d100 = _dec_d100[(c % Int) + 1] + @inbounds d100 = _dec_d100[(c % Int)::Int + 1] @inbounds buf[pos + i - 2] = d100 % UInt8 @inbounds buf[pos + i - 1] = (d100 >> 0x8) % UInt8 i -= 2 From f61fa851c6c5623434b6c97f382b800f634d99f3 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 1 Apr 2025 04:59:43 -0400 Subject: [PATCH 030/662] CI: add results.json tags back (#57966) --- test/buildkitetestjson.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index dbc08761c4dff..a0a220d18a1fb 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -69,6 +69,8 @@ function result_dict(testset::Test.DefaultTestSet, prefix::String="") "arch" => string(Sys.ARCH), "julia_version" => string(VERSION), "testset" => testset.description, + "job_group" => get(ENV, "BUILDKITE_GROUP_LABEL", "unknown"), + "job_label" => get(ENV, "BUILDKITE_LABEL", "unknown"), ), # note we drop some of this from common_data before merging into individual results "history" => if !isnothing(testset.time_end) From 13311f324e850fefddfcdf43d6c93b9365e2cf46 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 1 Apr 2025 23:19:54 +1300 Subject: [PATCH 031/662] Update Documenter 1.8.1 => 1.10.0 (#57952) --- doc/Manifest.toml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 8dadbc0ecde2c..669c22ee4c382 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" +julia_version = "1.13.0-DEV" manifest_format = "2.0" project_hash = "1e9ffa7d4739f7d125a5e2c66af8747a8effd889" @@ -28,9 +28,9 @@ version = "1.11.0" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.6" +version = "0.7.8" [[deps.Dates]] deps = ["Printf"] @@ -38,27 +38,26 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" [[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +git-tree-sha1 = "e7b7e6f178525d17c720ab9c081e4ef04429f860" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" +version = "0.9.4" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "182a9a3fe886587ba230a417f1651a4cbc2b92d4" +git-tree-sha1 = "0efa18ca40b9928422d782b3191c032fd0c90682" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.8.1" +version = "1.10.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" +version = "1.7.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.2+0" +version = "2.6.5+0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -72,9 +71,9 @@ version = "1.3.1" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +git-tree-sha1 = "2f6d6f7e6d6de361865d4394b802c02fc944fc7c" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.46.2+0" +version = "2.49.0+0" [[deps.IOCapture]] deps = ["Logging", "Random"] @@ -89,9 +88,9 @@ version = "1.11.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.6.1" +version = "1.7.0" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -140,9 +139,9 @@ version = "1.11.0" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.17.0+1" +version = "1.18.0+0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -174,7 +173,7 @@ version = "1.3.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+2" +version = "3.0.16+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -198,9 +197,9 @@ weakdeps = ["REPL"] [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +git-tree-sha1 = "0efd947a0c34740721a1af4225fe83431b40c950" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" +version = "1.3.0" [[deps.Preferences]] deps = ["TOML"] From e8b52e6edf511a1256cadf7e2985ea17a2866efa Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 13 Mar 2025 18:10:25 +0000 Subject: [PATCH 032/662] bindings: backdate initial module statements For model simplicity (and eventually for better unspecialized compilation), try to backdate initial module operations back to world 0 if nobody could have observed them before, rather than putting each incremental operation in a separate, but unobserved world. --- base/Base_compiler.jl | 7 ++-- base/sysimg.jl | 19 +++------ doc/src/devdocs/init.md | 2 +- src/builtins.c | 10 ++--- src/gf.c | 2 +- src/jl_exported_funcs.inc | 2 - src/jltypes.c | 8 ++-- src/julia.h | 3 +- src/julia_internal.h | 8 ++-- src/module.c | 52 +++++++++++++++++------- src/toplevel.c | 83 +++++++++++++++++---------------------- 11 files changed, 97 insertions(+), 99 deletions(-) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 7c63e169e9462..d4a106564a076 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -1,9 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -baremodule Base +module Base -using Core -using Core.Intrinsics, Core.IR +using .Core.Intrinsics, .Core.IR # to start, we're going to use a very simple definition of `include` # that doesn't require any function (except what we can get from the `Core` top-module) @@ -375,5 +374,5 @@ Core._setparser!(fl_parse) # Further definition of Base will happen in Base.jl if loaded. -end # baremodule Base +end # module Base using .Base diff --git a/base/sysimg.jl b/base/sysimg.jl index c12ddcd71c66f..8adb05ece0b2c 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -1,19 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# Can be loaded on top of either an existing system image built from -# `Base_compiler.jl` or standalone, in which case we will build it now. -let had_compiler = isdefined(Main, :Base) -if had_compiler; else -include("Base_compiler.jl") -end - -Core.include(Base, "Base.jl") +Base.Core.include(Base, "Base.jl") # finish populating Base (currently just has the Compiler) -had_compiler && ccall(:jl_init_restored_module, Cvoid, (Any,), Base) -end +# Set up Main module by importing from Base +using .Base +using .Base.MainInclude # ans, err, and sometimes Out -# Set up Main module -using Base.MainInclude # ans, err, and sometimes Out +ccall(:jl_init_restored_module, Cvoid, (Any,), Base) # These definitions calls Base._include rather than Base.include to get # one-frame stacktraces for the common case of using include(fname) in Main. @@ -59,7 +52,7 @@ definition of `eval`, which evaluates expressions in that module. const eval = Core.EvalInto(Main) # Ensure this file is also tracked -pushfirst!(Base._included_files, (@__MODULE__, abspath(@__FILE__))) +pushfirst!(Base._included_files, (Main, abspath(@__FILE__))) # set up depot & load paths to be able to find stdlib packages Base.init_depot_path() diff --git a/doc/src/devdocs/init.md b/doc/src/devdocs/init.md index 1e0e1173f8695..23012d6ba1eb7 100644 --- a/doc/src/devdocs/init.md +++ b/doc/src/devdocs/init.md @@ -63,7 +63,7 @@ the [LLVM library](https://llvm.org). If there is no sysimg file (`!jl_options.image_file`) then the `Core` and `Main` modules are created and `boot.jl` is evaluated: -`jl_core_module = jl_new_module(jl_symbol("Core"))` creates the Julia `Core` module. +`jl_core_module = jl_new_module(jl_symbol("Core"), NULL)` creates the Julia `Core` module. [`jl_init_intrinsic_functions()`](https://github.com/JuliaLang/julia/blob/master/src/intrinsics.cpp) creates a new Julia module `Intrinsics` containing constant `jl_intrinsic_type` symbols. These define diff --git a/src/builtins.c b/src/builtins.c index 243d14f34d3bd..67900dc10584f 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2396,8 +2396,7 @@ static void add_intrinsic(jl_module_t *inm, const char *name, enum intrinsic f) { jl_value_t *i = jl_permbox32(jl_intrinsic_type, 0, (int32_t)f); jl_sym_t *sym = jl_symbol(name); - jl_set_const(inm, sym, i); - jl_module_public(inm, sym, 1); + jl_set_initial_const(inm, sym, i, 1); } void jl_init_intrinsic_properties(void) JL_GC_DISABLED @@ -2413,9 +2412,8 @@ void jl_init_intrinsic_properties(void) JL_GC_DISABLED void jl_init_intrinsic_functions(void) JL_GC_DISABLED { - jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics"), NULL); - inm->parent = jl_core_module; - jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm); + jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1); + jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0); jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call); jl_mk_builtin_func( (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type), @@ -2437,7 +2435,7 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED static void add_builtin(const char *name, jl_value_t *v) { - jl_set_const(jl_core_module, jl_symbol(name), v); + jl_set_initial_const(jl_core_module, jl_symbol(name), v, 0); } jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt) diff --git a/src/gf.c b/src/gf.c index 714bf0b285cc6..cadc0e4bfb6cf 100644 --- a/src/gf.c +++ b/src/gf.c @@ -294,7 +294,7 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a if (dt == NULL) { // Builtins are specially considered available from world 0 jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); - jl_set_const(jl_core_module, sname, f); + jl_set_initial_const(jl_core_module, sname, f, 0); dt = (jl_datatype_t*)jl_typeof(f); } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index e4fd664478123..fcc0454a79dd8 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -2,7 +2,6 @@ #define JL_RUNTIME_EXPORTED_FUNCS(XX) \ XX(jl_active_task_stack) \ - XX(jl_add_standard_imports) \ XX(jl_adopt_thread) \ XX(jl_alignment) \ XX(jl_alloc_array_1d) \ @@ -98,7 +97,6 @@ XX(jl_cstr_to_string) \ XX(jl_current_exception) \ XX(jl_debug_method_invalidation) \ - XX(jl_defines_or_exports_p) \ XX(jl_deprecate_binding) \ XX(jl_dlclose) \ XX(jl_dlopen) \ diff --git a/src/jltypes.c b/src/jltypes.c index cda47118541dd..3853f07013e94 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3287,10 +3287,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, 0, 0, 3); core = jl_new_module(jl_symbol("Core"), NULL); - core->parent = core; jl_type_typename->mt->module = core; jl_core_module = core; - core = NULL; // not ready yet to use + core = NULL; // not actually ready yet to use tv = jl_svec1(tvar("Backend")); jl_addrspace_typename = @@ -3381,9 +3380,8 @@ void jl_init_types(void) JL_GC_DISABLED core = jl_core_module; jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); // export own name, so "using Foo" makes "Foo" itself visible - jl_set_const(core, core->name, (jl_value_t*)core); - jl_module_public(core, core->name, 1); - jl_set_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem); + jl_set_initial_const(core, core->name, (jl_value_t*)core, 1); + jl_set_initial_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem, 0); core = NULL; jl_expr_type = diff --git a/src/julia.h b/src/julia.h index 66458d997b666..2ecdde9272f0a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2093,13 +2093,13 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b JL_PROPAGATES_ROOT, size_t new_world); JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); -JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); +void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported); JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_value_t *jl_checked_swap(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_value_t *jl_checked_replace(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *expected, jl_value_t *rhs); @@ -2115,7 +2115,6 @@ JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported); JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var); -JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m); // eq hash tables JL_DLLEXPORT jl_genericmemory_t *jl_eqtable_put(jl_genericmemory_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted); diff --git a/src/julia_internal.h b/src/julia_internal.h index cf9ed6f08f4af..288918e4b18f8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -849,6 +849,7 @@ JL_DLLEXPORT void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind, size_t new_world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno); +void jl_module_initial_using(jl_module_t *to, jl_module_t *from); STATIC_INLINE struct _jl_module_using *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; STATIC_INLINE jl_module_t *module_usings_getmod(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to); @@ -921,7 +922,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; -extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; +extern jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED; extern arraylist_t *jl_entrypoint_mis; @@ -1255,9 +1256,8 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); -JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent); -JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name); -JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name); +jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name); +jl_module_t *jl_add_standard_imports(jl_module_t *m); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp) JL_PROPAGATES_ROOT; diff --git a/src/module.c b/src/module.c index 07b79b707f024..e1507f816860e 100644 --- a/src/module.c +++ b/src/module.c @@ -401,7 +401,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_bindin return NULL; } -JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) +static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) { jl_task_t *ct = jl_current_task; const jl_uuid_t uuid_zero = {0, 0}; @@ -410,7 +410,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) jl_set_typetagof(m, jl_module_tag, 0); assert(jl_is_symbol(name)); m->name = name; - m->parent = parent; + m->parent = parent ? parent : m; m->istopmod = 0; m->uuid = uuid_zero; static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing @@ -437,23 +437,22 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) return m; } -JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name) +static void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name) { if (jl_core_module) { // Bootstrap: Before jl_core_module is defined, we don't have enough infrastructure // for bindings, so Core itself gets special handling in jltypes.c if (default_using_core) { - jl_module_using(m, jl_core_module); + jl_module_initial_using(m, jl_core_module); } if (self_name) { // export own name, so "using Foo" makes "Foo" itself visible - jl_set_const(m, m->name, (jl_value_t*)m); - jl_module_public(m, m->name, 1); + jl_set_initial_const(m, m->name, (jl_value_t*)m, 1); } } } -JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name) +jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name) { jl_module_t *m = jl_new_module__(name, parent); JL_GC_PUSH1(&m); @@ -1231,6 +1230,19 @@ void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) JL_UNLOCK(&from->lock); } +void jl_module_initial_using(jl_module_t *to, jl_module_t *from) +{ + struct _jl_module_using new_item = { + .mod = from, + .min_world = 0, + .max_world = ~(size_t)0 + }; + arraylist_grow(&to->usings, sizeof(struct _jl_module_using)/sizeof(void*)); + memcpy(&to->usings.items[to->usings.len-3], &new_item, sizeof(struct _jl_module_using)); + jl_gc_wb(to, from); + jl_add_usings_backedge(from, to); +} + JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) { if (to == from) @@ -1325,6 +1337,7 @@ JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) { + // caller must hold world_counter_lock jl_binding_t *b = jl_get_module_binding(from, s, 1); JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; @@ -1370,13 +1383,6 @@ JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // u return jl_atomic_load(&b->value) != NULL; } -JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var) -{ - jl_binding_t *b = jl_get_module_binding(m, var, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && ((bpart->kind & PARTITION_FLAG_EXPORTED) || jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL); -} - JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); @@ -1475,9 +1481,25 @@ JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *va jl_checked_assignment(bp, m, var, val); } +JL_DLLEXPORT void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported) +{ + // this function is only valid during initialization, so there is no risk of data races her are not too important to use + int kind = PARTITION_KIND_CONST | (exported ? PARTITION_FLAG_EXPORTED : 0); + // jl_declare_constant_val3(NULL, m, var, (jl_value_t*)jl_any_type, kind, 0); + jl_binding_t *bp = jl_get_module_binding(m, var, 1); + jl_binding_partition_t *bpart = jl_get_binding_partition(bp, 0); + assert(bpart->min_world == 0); + jl_atomic_store_relaxed(&bpart->max_world, ~(size_t)0); // jl_check_new_binding_implicit likely incorrectly truncated it + if (exported) + jl_atomic_fetch_or_relaxed(&bp->flags, BINDING_FLAG_PUBLICP); + bpart->kind = kind | (bpart->kind & PARTITION_MASK_FLAG); + bpart->restriction = val; + jl_gc_wb(bpart, val); +} + JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT) { - // this function is mostly only used during initialization, so the data races here are not too important to us + // this function is dangerous and unsound. do not use. jl_binding_t *bp = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); bpart->min_world = 0; diff --git a/src/toplevel.c b/src/toplevel.c index 174bb0ef4dd22..60903aaf31ee6 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -34,26 +34,24 @@ htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; // During incremental compilation, the following gets set -JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module = NULL; // the toplevel module currently being defined +jl_module_t *jl_precompile_toplevel_module = NULL; // the toplevel module currently being defined -JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m) +jl_module_t *jl_add_standard_imports(jl_module_t *m) { jl_module_t *base_module = jl_base_relative_to(m); assert(base_module != NULL); // using Base - jl_module_using(m, base_module); + jl_module_initial_using(m, base_module); + return base_module; } // create a new top-level module void jl_init_main_module(void) { assert(jl_main_module == NULL); - jl_main_module = jl_new_module(jl_symbol("Main"), NULL); - jl_main_module->parent = jl_main_module; - jl_set_const(jl_main_module, jl_symbol("Core"), - (jl_value_t*)jl_core_module); - jl_set_const(jl_core_module, jl_symbol("Main"), - (jl_value_t*)jl_main_module); + jl_main_module = jl_new_module_(jl_symbol("Main"), NULL, 0, 1); // baremodule Main; end + jl_set_initial_const(jl_core_module, jl_symbol("Main"), (jl_value_t*)jl_main_module, 0); // const Main.Core = Core + jl_set_initial_const(jl_main_module, jl_symbol("Core"), (jl_value_t*)jl_core_module, 0); // const Core.Main = Main } static jl_function_t *jl_module_get_initializer(jl_module_t *m JL_PROPAGATES_ROOT) @@ -138,38 +136,14 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex // If we have `Base`, don't also try to import `Core` - the `Base` exports are a superset. // While we allow multiple imports of the same binding from different modules, various error printing // performs reflection on which module a binding came from and we'd prefer users see "Base" here. - jl_module_t *newm = jl_new_module__(name, is_parent__toplevel__ ? NULL : parent_module); + jl_module_t *newm = jl_new_module_(name, is_parent__toplevel__ ? NULL : parent_module, std_imports && jl_base_module != NULL ? 0 : 1, 1); jl_value_t *form = (jl_value_t*)newm; JL_GC_PUSH1(&form); JL_LOCK(&jl_modules_mutex); ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1)); JL_UNLOCK(&jl_modules_mutex); - - jl_add_default_names(newm, std_imports && jl_base_module != NULL ? 0 : 1, 1); - - jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; - // copy parent environment info into submodule newm->uuid = parent_module->uuid; - if (is_parent__toplevel__) { - newm->parent = newm; - jl_register_root_module(newm); - if (jl_options.incremental) { - jl_precompile_toplevel_module = newm; - } - } - else { - jl_declare_constant_val(NULL, parent_module, name, (jl_value_t*)newm); - } - - if (parent_module == jl_main_module && name == jl_symbol("Base")) { - // pick up Base module during bootstrap - jl_base_module = newm; - } - - size_t last_age = ct->world_age; - - // add standard imports unless baremodule jl_array_t *exprs = ((jl_expr_t*)jl_exprarg(ex, 2))->args; int lineno = 0; const char *filename = "none"; @@ -182,25 +156,42 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex filename = jl_symbol_name((jl_sym_t*)file); } } - if (std_imports) { - if (jl_base_module != NULL) { - jl_add_standard_imports(newm); - jl_datatype_t *include_into = (jl_datatype_t *)jl_get_global(jl_base_module, jl_symbol("IncludeInto")); - if (include_into) { - form = jl_new_struct(include_into, newm); - jl_set_const(newm, jl_symbol("include"), form); - } + newm->file = jl_symbol(filename); + jl_gc_wb_knownold(newm, newm->file); + newm->line = lineno; + + // add standard imports unless baremodule + if (std_imports && jl_base_module != NULL) { + jl_module_t *base = jl_add_standard_imports(newm); + jl_datatype_t *include_into = (jl_datatype_t *)jl_get_global(base, jl_symbol("IncludeInto")); + if (include_into) { + form = jl_new_struct(include_into, newm); + jl_set_initial_const(newm, jl_symbol("include"), form, 0); } jl_datatype_t *eval_into = (jl_datatype_t *)jl_get_global(jl_core_module, jl_symbol("EvalInto")); if (eval_into) { form = jl_new_struct(eval_into, newm); - jl_set_const(newm, jl_symbol("eval"), form); + jl_set_initial_const(newm, jl_symbol("eval"), form, 0); } } - newm->file = jl_symbol(filename); - jl_gc_wb_knownold(newm, newm->file); - newm->line = lineno; + jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; + size_t last_age = ct->world_age; + + if (parent_module == jl_main_module && name == jl_symbol("Base") && jl_base_module == NULL) { + // pick up Base module during bootstrap + jl_base_module = newm; + } + + if (is_parent__toplevel__) { + jl_register_root_module(newm); + if (jl_options.incremental) { + jl_precompile_toplevel_module = newm; + } + } + else { + jl_declare_constant_val(NULL, parent_module, name, (jl_value_t*)newm); + } for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form From 43766717f6a57b7e6995d01a07fa1f70fcaf0272 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 13 Mar 2025 18:33:14 +0000 Subject: [PATCH 033/662] bindings: batch export/public changes from one statement No real major effect, basically just making printing look tidier and avoiding spamming the lock. --- src/jl_exported_funcs.inc | 1 - src/julia.h | 2 +- src/module.c | 8 +++----- src/toplevel.c | 29 ++++++++++++++++++++++------- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index fcc0454a79dd8..883a8d74df1bb 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -313,7 +313,6 @@ XX(jl_module_names) \ XX(jl_module_parent) \ XX(jl_module_getloc) \ - XX(jl_module_public) \ XX(jl_module_public_p) \ XX(jl_module_use) \ XX(jl_module_using) \ diff --git a/src/julia.h b/src/julia.h index 2ecdde9272f0a..115ccb6c2008f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2112,7 +2112,7 @@ JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *fro JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); -JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported); +int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world); JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var); diff --git a/src/module.c b/src/module.c index e1507f816860e..7d77e933f8193 100644 --- a/src/module.c +++ b/src/module.c @@ -1335,12 +1335,10 @@ JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym return (jl_value_t*)b; } -JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) +int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world) { // caller must hold world_counter_lock jl_binding_t *b = jl_get_module_binding(from, s, 1); - JL_LOCK(&world_counter_lock); - size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); int was_exported = (bpart->kind & PARTITION_FLAG_EXPORTED) != 0; if (jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP) { @@ -1355,9 +1353,9 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_PUBLICP); if (was_exported != exported) { jl_replace_binding_locked2(b, bpart, bpart->restriction, bpart->kind | PARTITION_FLAG_EXPORTED, new_world); - jl_atomic_store_release(&jl_world_counter, new_world); + return 1; } - JL_UNLOCK(&world_counter_lock); + return 0; } JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // unlike most queries here, this is currently seq_cst diff --git a/src/toplevel.c b/src/toplevel.c index 60903aaf31ee6..cbf220f0016f1 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -925,14 +925,29 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val } else if (head == jl_export_sym || head == jl_public_sym) { int exp = (head == jl_export_sym); - for (size_t i = 0; i < jl_array_nrows(ex->args); i++) { - jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); - if (!jl_is_symbol(name)) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - exp ? "syntax: malformed \"export\" statement" : - "syntax: malformed \"public\" statement"); - jl_module_public(m, name, exp); + volatile int any_new = 0; + JL_LOCK(&world_counter_lock); + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; + JL_TRY { + for (size_t i = 0; i < jl_array_nrows(ex->args); i++) { + jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); + if (!jl_is_symbol(name)) + jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, + exp ? "syntax: malformed \"export\" statement" : + "syntax: malformed \"public\" statement"); + if (jl_module_public_(m, name, exp, new_world)) + any_new = 1; + } + } + JL_CATCH { + if (any_new) + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); + jl_rethrow(); } + if (any_new) + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); JL_GC_POP(); return jl_nothing; } From d6f5b7fdb310d9d41033b55e862a1a8b61ec1cc7 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 1 Apr 2025 10:50:22 -0400 Subject: [PATCH 034/662] =?UTF-8?q?Be=20more=20careful=20about=20iterator?= =?UTF-8?q?=20invalidation=20during=20recursive=20invalida=E2=80=A6=20(#57?= =?UTF-8?q?934)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …tion It is possible for one MethodInstance to have a backedge to itself (`typejoin` is a prime example). This didn't used to be much of a problem prior to #57625 because we would just delete the backedges list at the top of invalidation. However, now that we're more selective, we need to be careful not to move backedges around while a frame higher on the stack may be looking at it. To this end, move the compaction part of the deletion into a separate pass and only delete (but don't move around) backedges while a frame higher on the stack may be looking at it. Fixes #57696 --- Compiler/test/invalidation.jl | 5 + src/gf.c | 237 +++++++++++++++++++--------------- src/julia.h | 5 +- src/julia_internal.h | 20 +++ src/method.c | 37 +++++- src/staticdata_utils.c | 7 +- 6 files changed, 204 insertions(+), 107 deletions(-) diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index db9d3ac06048f..32f462f06f4e9 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -283,3 +283,8 @@ begin take!(GLOBAL_BUFFER) @test isnothing(pr48932_caller_inlined(42)) @test "42" == String(take!(GLOBAL_BUFFER)) end + +# Issue #57696 +# This test checks for invalidation of recursive backedges. However, unfortunately, the original failure +# manifestation was an unreliable segfault or an assertion failure, so we don't have a more compact test. +@test success(`$(Base.julia_cmd()) -e 'Base.typejoin(x, ::Type) = 0; exit()'`) diff --git a/src/gf.c b/src/gf.c index 714bf0b285cc6..d43a2d32e2759 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1831,43 +1831,133 @@ JL_DLLEXPORT void jl_invalidate_code_instance(jl_code_instance_t *replaced, size } static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth) { - jl_array_t *backedges = replaced_mi->backedges; - if (backedges) { - // invalidate callers (if any) + uint8_t recursion_flags = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(replaced_mi, &recursion_flags); + if (!backedges) + return; + // invalidate callers (if any) + if (!replaced_ci) { + // We know all backedges are deleted - clear them eagerly + // Clears both array and flags replaced_mi->backedges = NULL; - JL_GC_PUSH1(&backedges); - size_t i = 0, l = jl_array_nrows(backedges); - size_t ins = 0; - jl_code_instance_t *replaced; - while (i < l) { - jl_value_t *invokesig = NULL; - i = get_next_edge(backedges, i, &invokesig, &replaced); - JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges - if (replaced_ci) { - // If we're invalidating a particular codeinstance, only invalidate - // this backedge it actually has an edge for our codeinstance. - jl_svec_t *edges = jl_atomic_load_relaxed(&replaced->edges); - for (size_t j = 0; j < jl_svec_len(edges); ++j) { - jl_value_t *edge = jl_svecref(edges, j); - if (edge == (jl_value_t*)replaced_mi || edge == (jl_value_t*)replaced_ci) - goto found; - } - // Keep this entry in the backedge list, but compact it - ins = set_next_edge(backedges, ins, invokesig, replaced); - continue; - found:; + jl_atomic_fetch_and_relaxed(&replaced_mi->flags, ~MI_FLAG_BACKEDGES_ALL); + } + JL_GC_PUSH1(&backedges); + size_t i = 0, l = jl_array_nrows(backedges); + size_t ins = 0; + jl_code_instance_t *replaced; + while (i < l) { + jl_value_t *invokesig = NULL; + i = get_next_edge(backedges, i, &invokesig, &replaced); + if (!replaced) { + ins = i; + continue; + } + JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges + if (replaced_ci) { + // If we're invalidating a particular codeinstance, only invalidate + // this backedge it actually has an edge for our codeinstance. + jl_svec_t *edges = jl_atomic_load_relaxed(&replaced->edges); + for (size_t j = 0; j < jl_svec_len(edges); ++j) { + jl_value_t *edge = jl_svecref(edges, j); + if (edge == (jl_value_t*)replaced_mi || edge == (jl_value_t*)replaced_ci) + goto found; } - invalidate_code_instance(replaced, max_world, depth); + ins = set_next_edge(backedges, ins, invokesig, replaced); + continue; + found:; + ins = clear_next_edge(backedges, ins, invokesig, replaced); + jl_atomic_fetch_or(&replaced_mi->flags, MI_FLAG_BACKEDGES_DIRTY); + /* fallthrough */ + } + invalidate_code_instance(replaced, max_world, depth); + if (replaced_ci && !replaced_mi->backedges) { + // Fast-path early out. If `invalidate_code_instance` invalidated + // the entire mi via a recursive edge, there's no point to keep + // iterating - they'll already have been invalidated. + break; } - if (replaced_ci && ins != 0) { - jl_array_del_end(backedges, l - ins); - // If we're only invalidating one ci, we don't know which ci any particular - // backedge was for, so we can't delete them. Put them back. - replaced_mi->backedges = backedges; - jl_gc_wb(replaced_mi, backedges); + } + if (replaced_ci) + jl_mi_done_backedges(replaced_mi, recursion_flags); + JL_GC_POP(); +} + +enum morespec_options { + morespec_unknown, + morespec_isnot, + morespec_is +}; + +// check if `type` is replacing `m` with an ambiguity here, given other methods in `d` that already match it +static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_t *const *d, size_t n, jl_value_t *isect, jl_value_t *isect2, char *morespec) +{ + size_t k; + for (k = 0; k < n; k++) { + jl_method_t *m2 = d[k]; + // see if m2 also fully covered this intersection + if (m == m2 || !(jl_subtype(isect, m2->sig) || (isect2 && jl_subtype(isect2, m2->sig)))) + continue; + if (morespec[k] == (char)morespec_unknown) + morespec[k] = (char)(jl_type_morespecific(m2->sig, type) ? morespec_is : morespec_isnot); + if (morespec[k] == (char)morespec_is) + // not actually shadowing this--m2 will still be better + return 0; + // if type is not more specific than m (thus now dominating it) + // then there is a new ambiguity here, + // since m2 was also a previous match over isect, + // see if m was previously dominant over all m2 + // or if this was already ambiguous before + if (ambig != morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { + // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type + return 0; } - JL_GC_POP(); } + return 1; +} + +static int _invalidate_dispatch_backedges(jl_method_instance_t *mi, jl_value_t *type, jl_method_t *m, + jl_method_t *const *d, size_t n, int replaced_dispatch, int ambig, + size_t max_world, char *morespec) +{ + uint8_t backedge_recursion_flags = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(mi, &backedge_recursion_flags); + if (!backedges) + return 0; + size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + int invalidated_any = 0; + while (mi->backedges && ib < nb) { + ib = get_next_edge(backedges, ib, &invokeTypes, &caller); + if (!caller) { + insb = ib; + continue; + } + JL_GC_PROMISE_ROOTED(caller); // propagated by get_next_edge from backedges + int replaced_edge; + if (invokeTypes) { + // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes + if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) + replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself + else + replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); + } + else { + replaced_edge = replaced_dispatch; + } + if (replaced_edge) { + invalidate_code_instance(caller, max_world, 1); + insb = clear_next_edge(backedges, insb, invokeTypes, caller); + jl_atomic_fetch_or(&mi->flags, MI_FLAG_BACKEDGES_DIRTY); + invalidated_any = 1; + } + else { + insb = set_next_edge(backedges, insb, invokeTypes, caller); + } + } + jl_mi_done_backedges(mi, backedge_recursion_flags); + return invalidated_any; } // invalidate cached methods that overlap this definition @@ -1898,20 +1988,22 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, JL_LOCK(&callee->def.method->writelock); if (jl_atomic_load_relaxed(&allow_new_worlds)) { int found = 0; + jl_array_t *backedges = jl_mi_get_backedges(callee); // TODO: use jl_cache_type_(invokesig) like cache_method does to save memory - if (!callee->backedges) { + if (!backedges) { // lazy-init the backedges array - callee->backedges = jl_alloc_vec_any(0); - jl_gc_wb(callee, callee->backedges); + backedges = jl_alloc_vec_any(0); + callee->backedges = backedges; + jl_gc_wb(callee, backedges); } else { - size_t i = 0, l = jl_array_nrows(callee->backedges); + size_t i = 0, l = jl_array_nrows(backedges); for (i = 0; i < l; i++) { // optimized version of while (i < l) i = get_next_edge(callee->backedges, i, &invokeTypes, &mi); - jl_value_t *ciedge = jl_array_ptr_ref(callee->backedges, i); + jl_value_t *ciedge = jl_array_ptr_ref(backedges, i); if (ciedge != (jl_value_t*)caller) continue; - jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(callee->backedges, i - 1) : NULL; + jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(backedges, i - 1) : NULL; if (invokeTypes && jl_is_method_instance(invokeTypes)) invokeTypes = NULL; if ((invokesig == NULL && invokeTypes == NULL) || @@ -1922,7 +2014,7 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, } } if (!found) - push_edge(callee->backedges, invokesig, caller); + push_edge(backedges, invokesig, caller); } JL_UNLOCK(&callee->def.method->writelock); } @@ -2111,13 +2203,13 @@ static int erase_method_backedges(jl_typemap_entry_t *def, void *closure) for (i = 0; i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i); if ((jl_value_t*)mi != jl_nothing) { - mi->backedges = NULL; + mi->backedges = 0; } } } else { jl_method_instance_t *mi = (jl_method_instance_t*)specializations; - mi->backedges = NULL; + mi->backedges = 0; } JL_UNLOCK(&method->writelock); return 1; @@ -2189,39 +2281,6 @@ static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **is return 1; } -enum morespec_options { - morespec_unknown, - morespec_isnot, - morespec_is -}; - -// check if `type` is replacing `m` with an ambiguity here, given other methods in `d` that already match it -static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_t *const *d, size_t n, jl_value_t *isect, jl_value_t *isect2, char *morespec) -{ - size_t k; - for (k = 0; k < n; k++) { - jl_method_t *m2 = d[k]; - // see if m2 also fully covered this intersection - if (m == m2 || !(jl_subtype(isect, m2->sig) || (isect2 && jl_subtype(isect2, m2->sig)))) - continue; - if (morespec[k] == (char)morespec_unknown) - morespec[k] = (char)(jl_type_morespecific(m2->sig, type) ? morespec_is : morespec_isnot); - if (morespec[k] == (char)morespec_is) - // not actually shadowing this--m2 will still be better - return 0; - // if type is not more specific than m (thus now dominating it) - // then there is a new ambiguity here, - // since m2 was also a previous match over isect, - // see if m was previously dominant over all m2 - // or if this was already ambiguous before - if (ambig != morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { - // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type - return 0; - } - } - return 1; -} - jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) { JL_TIMING(ADD_METHOD, ADD_METHOD); @@ -2386,35 +2445,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) // found that this specialization dispatch got replaced by m // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); // but ignore invoke-type edges - jl_array_t *backedges = mi->backedges; - if (backedges) { - size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); - jl_value_t *invokeTypes; - jl_code_instance_t *caller; - while (ib < nb) { - ib = get_next_edge(backedges, ib, &invokeTypes, &caller); - JL_GC_PROMISE_ROOTED(caller); // propagated by get_next_edge from backedges - int replaced_edge; - if (invokeTypes) { - // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes - if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) - replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself - else - replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); - } - else { - replaced_edge = replaced_dispatch; - } - if (replaced_edge) { - invalidate_code_instance(caller, max_world, 1); - invalidated = 1; - } - else { - insb = set_next_edge(backedges, insb, invokeTypes, caller); - } - } - jl_array_del_end(backedges, nb - insb); - } + invalidated = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); if (_jl_debug_method_invalidation && invalidated) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); diff --git a/src/julia.h b/src/julia.h index 66458d997b666..e7ffdc11a1d64 100644 --- a/src/julia.h +++ b/src/julia.h @@ -404,13 +404,16 @@ struct _jl_method_instance_t { } def; // pointer back to the context for this code jl_value_t *specTypes; // argument types this was specialized for jl_svec_t *sparam_vals; // static parameter values, indexed by def.method->sig - jl_array_t *backedges; // list of code-instances which call this method-instance; `invoke` records (invokesig, caller) pairs + // list of code-instances which call this method-instance; `invoke` records (invokesig, caller) pairs + jl_array_t *backedges; _Atomic(struct _jl_code_instance_t*) cache; uint8_t cache_with_orig; // !cache_with_specTypes // flags for this method instance // bit 0: generated by an explicit `precompile(...)` // bit 1: dispatched + // bit 2: The ->backedges field is currently being walked higher up the stack - entries may be deleted, but not moved + // bit 3: The ->backedges field was modified and should be compacted when clearing bit 2 _Atomic(uint8_t) flags; }; #define JL_MI_FLAGS_MASK_PRECOMPILED 0x01 diff --git a/src/julia_internal.h b/src/julia_internal.h index cf9ed6f08f4af..0ecc243dcc493 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -726,9 +726,29 @@ JL_DLLEXPORT void jl_resolve_definition_effects_in_ir(jl_array_t *stmts, jl_modu JL_DLLEXPORT int jl_maybe_add_binding_backedge(jl_binding_t *b, jl_value_t *edge, jl_method_t *in_method); JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge); +static const uint8_t MI_FLAG_BACKEDGES_INUSE = 0b0100; +static const uint8_t MI_FLAG_BACKEDGES_DIRTY = 0b1000; +static const uint8_t MI_FLAG_BACKEDGES_ALL = 0b1100; + +STATIC_INLINE jl_array_t *jl_mi_get_backedges_mutate(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t *flags) { + *flags = jl_atomic_load_relaxed(&mi->flags) & (MI_FLAG_BACKEDGES_ALL); + jl_array_t *ret = mi->backedges; + if (ret) + jl_atomic_fetch_or_relaxed(&mi->flags, MI_FLAG_BACKEDGES_INUSE); + return ret; +} + +STATIC_INLINE jl_array_t *jl_mi_get_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT) { + assert(!(jl_atomic_load_relaxed(&mi->flags) & MI_FLAG_BACKEDGES_ALL)); + jl_array_t *ret = mi->backedges; + return ret; +} + int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT; int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller); +int clear_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller); void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller); +void jl_mi_done_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t old_flags); JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root); void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots); diff --git a/src/method.c b/src/method.c index abf039d9bce08..b3ed63e810f77 100644 --- a/src/method.c +++ b/src/method.c @@ -1028,7 +1028,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT { jl_value_t *item = jl_array_ptr_ref(list, i); - if (jl_is_code_instance(item)) { + if (!item || jl_is_code_instance(item)) { // Not an `invoke` call, it's just the CodeInstance if (invokesig != NULL) *invokesig = NULL; @@ -1053,6 +1053,14 @@ int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instan return i; } +int clear_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller) +{ + if (invokesig) + jl_array_ptr_set(list, i++, NULL); + jl_array_ptr_set(list, i++, NULL); + return i; +} + void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller) { if (invokesig) @@ -1061,6 +1069,33 @@ void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *call return; } +void jl_mi_done_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t old_flags) { + uint8_t flags_now = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(mi, &flags_now); + if (backedges && !old_flags) { + if (flags_now & MI_FLAG_BACKEDGES_DIRTY) { + size_t n = jl_array_nrows(backedges); + size_t i = 0; + size_t insb = 0; + while (i < n) { + jl_value_t *invokesig; + jl_code_instance_t *caller; + i = get_next_edge(backedges, i, &invokesig, &caller); + if (!caller) + continue; + insb = set_next_edge(backedges, insb, invokesig, caller); + } + if (insb == n) { + // All were deleted + mi->backedges = NULL; + } else { + jl_array_del_end(backedges, n - insb); + } + } + jl_atomic_fetch_and_relaxed(&mi->flags, ~MI_FLAG_BACKEDGES_ALL); + } +} + // method definition ---------------------------------------------------------- jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name, diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 9bfd4c355efe6..95a64964eccba 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -223,11 +223,14 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_push(stack, (void*)mi); int depth = stack->len; *bp = (void*)((char*)HT_NOTFOUND + 4 + depth); // preliminarily mark as in-progress - size_t i = 0, n = jl_array_nrows(mi->backedges); + jl_array_t *backedges = jl_mi_get_backedges(mi); + size_t i = 0, n = jl_array_nrows(backedges); int cycle = depth; while (i < n) { jl_code_instance_t *be; - i = get_next_edge(mi->backedges, i, NULL, &be); + i = get_next_edge(backedges, i, NULL, &be); + if (!be) + continue; JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack, query_cache); if (child_found == 1 || child_found == 2) { From 77f264f07adb0cd372237b2431eac72c6c672c0b Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Tue, 1 Apr 2025 22:34:34 +0200 Subject: [PATCH 035/662] Documentation: `public` needs global scope (#57916) --- doc/src/devdocs/ast.md | 8 ++++++++ doc/src/manual/modules.md | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index 1330940862b99..0bd8075a40229 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -119,10 +119,18 @@ parses as `(macrocall (|.| Core '@doc) (line) "some docs" (= (call f x) (block x | `import Base: x` | `(import (: (. Base) (. x)))` | | `import Base: x, y` | `(import (: (. Base) (. x) (. y)))` | | `export a, b` | `(export a b)` | +| `public a, b` | `(public a b)` | `using` has the same representation as `import`, but with expression head `:using` instead of `:import`. +To programmatically create a `public` statement, you can use `Expr(:public, :a, :b)` or, +closer to regular code, `Meta.parse("public a, b")`. This approach is necessary due to +[current limitations on `public`](@ref Export-lists). The `public` keyword is only +recognized at the syntactic top level within a file (`parse_stmts`) or module. This +restriction was implemented to prevent breaking existing code that used `public` as an +identifier when it was introduced in Julia 1.11. + ### Numbers Julia supports more number types than many scheme implementations, so not all numbers are represented diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index a6c66bbd14f77..00b55c23af905 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -114,6 +114,12 @@ and above. To maintain compatibility with Julia 1.10 and below, use the `@compat VERSION >= v"1.11.0-DEV.469" && eval(Meta.parse("public a, b, c")) ``` +`export` is a keyword wherever it occurs whereas the `public` keyword is currently limited to the +syntactic top level within a file or module. This limitation exists for compatibility reasons, +as `public` was introduced as a new keyword in Julia 1.11 while `export` has existed since Julia +1.0. However, this restriction on `public` may be lifted in future releases, so do not use `public` +as an identifier. + ### Standalone `using` and `import` For interactive use, the most common way of loading a module is `using ModuleName`. This [loads](@ref From 9f8ae16aaaba893dcc925d11d414441138a9a9b0 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Wed, 2 Apr 2025 08:15:31 +0530 Subject: [PATCH 036/662] Declare `AbstractOneTo` public (#57971) Since `AbstractOneTo` is meant to be subtyped by packages, this needs to be public. This was missed out in https://github.com/JuliaLang/julia/pull/56902. --- base/public.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/public.jl b/base/public.jl index 8d2a65f9a150c..f5748744eb4a3 100644 --- a/base/public.jl +++ b/base/public.jl @@ -10,6 +10,7 @@ public # Types AbstractLock, + AbstractOneTo, AbstractPipe, AsyncCondition, CodeUnits, From 4b95442c234b5edca1a5503f23a356c9004b4d9d Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 2 Apr 2025 12:41:46 +0900 Subject: [PATCH 037/662] inference: model `Core._svec_ref` (#57973) --- Compiler/src/tfuncs.jl | 27 ++++++++++++++++++++++----- Compiler/test/effects.jl | 10 ++++++++++ src/jltypes.c | 2 +- test/reflection.jl | 2 ++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index e987ad3fd2b27..f39601c1e729d 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -573,6 +573,16 @@ end add_tfunc(nfields, 1, 1, nfields_tfunc, 1) add_tfunc(Core._expr, 1, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->Expr), 100) add_tfunc(svec, 0, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->SimpleVector), 20) +@nospecs function _svec_ref_tfunc(𝕃::AbstractLattice, s, i) + if isa(s, Const) && isa(i, Const) + s, i = s.val, i.val + if isa(s, SimpleVector) && isa(i, Int) + return 1 ≤ i ≤ length(s) ? Const(s[i]) : Bottom + end + end + return Any +end +add_tfunc(Core._svec_ref, 2, 2, _svec_ref_tfunc, 1) @nospecs function typevar_tfunc(𝕃::AbstractLattice, n, lb_arg, ub_arg) lb = Union{} ub = Any @@ -2316,6 +2326,9 @@ function _builtin_nothrow(𝕃::AbstractLattice, @nospecialize(f::Builtin), argt elseif f === Core.compilerbarrier na == 2 || return false return compilerbarrier_nothrow(argtypes[1], nothing) + elseif f === Core._svec_ref + na == 2 || return false + return _svec_ref_tfunc(𝕃, argtypes[1], argtypes[2]) isa Const end return false end @@ -2346,7 +2359,9 @@ const _CONSISTENT_BUILTINS = Any[ throw, Core.throw_methoderror, setfield!, - donotdelete + donotdelete, + memoryrefnew, + memoryrefoffset, ] # known to be effect-free (but not necessarily nothrow) @@ -2371,6 +2386,7 @@ const _EFFECT_FREE_BUILTINS = [ Core.throw_methoderror, getglobal, compilerbarrier, + Core._svec_ref, ] const _INACCESSIBLEMEM_BUILTINS = Any[ @@ -2404,6 +2420,7 @@ const _ARGMEM_BUILTINS = Any[ replacefield!, setfield!, swapfield!, + Core._svec_ref, ] const _INCONSISTENT_INTRINSICS = Any[ @@ -2546,7 +2563,7 @@ const _EFFECTS_KNOWN_BUILTINS = Any[ # Core._primitivetype, # Core._setsuper!, # Core._structtype, - # Core._svec_ref, + Core._svec_ref, # Core._typebody!, Core._typevar, apply_type, @@ -2650,9 +2667,7 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argty else if contains_is(_CONSISTENT_BUILTINS, f) consistent = ALWAYS_TRUE - elseif f === memoryrefnew || f === memoryrefoffset - consistent = ALWAYS_TRUE - elseif f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned + elseif f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned || f === Core._svec_ref consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY elseif f === Core._typevar || f === Core.memorynew consistent = CONSISTENT_IF_NOTRETURNED @@ -2838,6 +2853,8 @@ _istypemin(@nospecialize x) = !_iszero(x) && Intrinsics.neg_int(x) === x function builtin_exct(𝕃::AbstractLattice, @nospecialize(f::Builtin), argtypes::Vector{Any}, @nospecialize(rt)) if isa(f, IntrinsicFunction) return intrinsic_exct(𝕃, f, argtypes) + elseif f === Core._svec_ref + return BoundsError end return Any end diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index 49a9628ba97c5..b3d8d1a56ea16 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -1467,3 +1467,13 @@ end @test Base.infer_effects(Core.invoke_in_world, Tuple{Vararg{Any}}) == Compiler.Effects() @test Base.infer_effects(invokelatest, Tuple{Vararg{Any}}) == Compiler.Effects() @test Base.infer_effects(invoke, Tuple{Vararg{Any}}) == Compiler.Effects() + +# Core._svec_ref effects modeling (required for external abstract interpreter that doesn't run optimization) +let effects = Base.infer_effects((Core.SimpleVector,Int); optimize=false) do svec, i + Core._svec_ref(svec, i) + end + @test !Compiler.is_consistent(effects) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_nothrow(effects) + @test Compiler.is_terminates(effects) +end diff --git a/src/jltypes.c b/src/jltypes.c index 3853f07013e94..09394b6b0e02b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3885,7 +3885,7 @@ void jl_init_types(void) JL_GC_DISABLED // override ismutationfree for builtin types that are mutable for identity jl_string_type->ismutationfree = jl_string_type->isidentityfree = 1; jl_symbol_type->ismutationfree = jl_symbol_type->isidentityfree = 1; - jl_simplevector_type->ismutationfree = jl_simplevector_type->isidentityfree = 1; + jl_simplevector_type->isidentityfree = 1; jl_typename_type->ismutationfree = 1; jl_datatype_type->ismutationfree = 1; jl_uniontype_type->ismutationfree = 1; diff --git a/test/reflection.jl b/test/reflection.jl index 345e219b41a3e..b9d1eaa1c86f9 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -1211,6 +1211,8 @@ end @test Base.ismutationfree(Type{Union{}}) +@test !Base.ismutationfree(Core.SimpleVector) + module TestNames public publicized From 2fd6db2e2b96057dbfa15ee651958e03ca5ce0d9 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 2 Apr 2025 07:21:33 -0400 Subject: [PATCH 038/662] Add `Base.@acquire` for non-closure version of `Base.acquire(f, s::Base.Semaphore)` (#56845) --- NEWS.md | 2 ++ base/lock.jl | 30 ++++++++++++++++++++++++++++++ base/public.jl | 1 + test/misc.jl | 18 ++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/NEWS.md b/NEWS.md index 7e6aae67f0def..9c6e35a6c8127 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,8 @@ Julia v1.13 Release Notes New language features --------------------- + - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) + Language changes ---------------- diff --git a/base/lock.jl b/base/lock.jl index 40bc9e08bd9b0..24c09ec289018 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -561,6 +561,36 @@ function acquire(f, s::Semaphore) end end +""" + Base.@acquire s::Semaphore expr + +Macro version of `Base.acquire(f, s::Semaphore)` but with `expr` instead of `f` function. +Expands to: +```julia +Base.acquire(s) +try + expr +finally + Base.release(s) +end +``` +This is similar to using [`acquire`](@ref) with a `do` block, but avoids creating a closure. + +!!! compat "Julia 1.13" + `Base.@acquire` was added in Julia 1.13 +""" +macro acquire(s, expr) + quote + local temp = $(esc(s)) + Base.acquire(temp) + try + $(esc(expr)) + finally + Base.release(temp) + end + end +end + """ release(s::Semaphore) diff --git a/base/public.jl b/base/public.jl index f5748744eb4a3..ca2cf0b146e0f 100644 --- a/base/public.jl +++ b/base/public.jl @@ -28,6 +28,7 @@ public # Semaphores Semaphore, acquire, + @acquire, release, # arrays diff --git a/test/misc.jl b/test/misc.jl index 6c8d76fa1cd1a..64b057d1ec4fc 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -237,6 +237,24 @@ end @test all(<=(sem_size), history) @test all(>=(0), history) @test history[end] == 0 + + # macro form + clock = Threads.Atomic{Int}(1) + occupied = Threads.Atomic{Int}(0) + history = fill!(Vector{Int}(undef, 2n), -1) + @sync for _ in 1:n + @async begin + @test Base.@acquire s begin + history[Threads.atomic_add!(clock, 1)] = Threads.atomic_add!(occupied, 1) + 1 + sleep(rand(0:0.01:0.1)) + history[Threads.atomic_add!(clock, 1)] = Threads.atomic_sub!(occupied, 1) - 1 + return :resultvalue + end === :resultvalue + end + end + @test all(<=(sem_size), history) + @test all(>=(0), history) + @test history[end] == 0 end # task switching From eae3a8a88c5af38885501678850173167dfe0bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Wed, 2 Apr 2025 16:29:35 +0200 Subject: [PATCH 039/662] CompilerDevTools: use `transform_result_for_cache` instead of `optimize` (#57640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As a follow-up to #57193 now that #57375 is merged. --------- Co-authored-by: Cédric Belmant --- Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index f430a4c84662f..7f6ee154e29dd 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -45,8 +45,8 @@ function lookup_method_instance(f, args...) @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} end -function Compiler.optimize(interp::SplitCacheInterp, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult) - @invoke Compiler.optimize(interp::Compiler.AbstractInterpreter, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult) +function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) + opt = result.src::Compiler.OptimizationState ir = opt.result.ir::Compiler.IRCode override = GlobalRef(@__MODULE__(), :with_new_compiler) for inst in ir.stmts @@ -61,6 +61,7 @@ function Compiler.optimize(interp::SplitCacheInterp, opt::Compiler.OptimizationS insert!(stmt.args, 1, override) insert!(stmt.args, 3, interp.owner) end + @invoke Compiler.transform_result_for_cache(interp::Compiler.AbstractInterpreter, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) end with_new_compiler(f, args...; owner::SplitCacheOwner = SplitCacheOwner()) = with_new_compiler(f, owner, args...) From f211a7746a7ea2de55af9b041f7101f09842b9dc Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 2 Apr 2025 12:36:22 -0400 Subject: [PATCH 040/662] Use juliaup name and id to avoid msstore bug (#57978) See https://github.com/JuliaLang/juliaup/issues/1191 Also https://github.com/JuliaLang/www.julialang.org/pull/2275 cc. @davidanthoff --------- Co-authored-by: Ajinkya Kokandakar --- doc/src/manual/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/installation.md b/doc/src/manual/installation.md index f45aba2c37a28..60a52e8cb6e19 100644 --- a/doc/src/manual/installation.md +++ b/doc/src/manual/installation.md @@ -18,7 +18,7 @@ On Windows Julia can be installed directly from the Windows store exactly the same version by executing ``` -winget install julia -s msstore +winget install --name Julia --id 9NJNWW8PVKMN -e -s msstore ``` in any shell. From c5d97f8c24be957b7b21df77377e65e457e2b5ab Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 3 Apr 2025 21:19:16 +0200 Subject: [PATCH 041/662] `Base.incomplete_tag`: deduplicate code and typeassert (#57984) The code deduplication should enable `a` to be inferred as `String` in the branch after the `a isa String` check. Also added some `::Symbol` (concrete) typeasserts just in case, because this is recursive code so more demanding for inference. Should improve the resistance to invalidation of the sysimage. --- base/client.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/client.jl b/base/client.jl index 9b7d80f51c219..853402c916d12 100644 --- a/base/client.jl +++ b/base/client.jl @@ -223,10 +223,13 @@ function incomplete_tag(ex::Expr) return :none elseif isempty(ex.args) return :other - elseif ex.args[1] isa String - return fl_incomplete_tag(ex.args[1]) else - return incomplete_tag(ex.args[1]) + a = ex.args[1] + if a isa String + return fl_incomplete_tag(a)::Symbol + else + return incomplete_tag(a)::Symbol + end end end incomplete_tag(exc::Meta.ParseError) = incomplete_tag(exc.detail) From 91505bb6a11456038e456d29cc925c0a5f6205b8 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Fri, 4 Apr 2025 05:41:16 +0530 Subject: [PATCH 042/662] Index a `CartesianIndex` with `begin`/`end` (#57985) Currently, it is possible to index into a `CartesianIndex` using an `Int` index. ```julia julia> I = CartesianIndex(4,3) CartesianIndex(4, 3) julia> I[1] 4 julia> I[2] 3 ``` This PR adds the ability to use `begin`/`end` in the indexing: ```julia julia> I[begin] 4 julia> I[end] 3 ``` The advantage of this (particularly indexing with `end`) is that one doesn't need to know the number of dimensions. This makes writing generic code that accepts either a vector or a matrix easier. --- base/multidimensional.jl | 4 +++- test/cartesian.jl | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f15c72a746698..5e3fdefbde9ae 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -5,7 +5,7 @@ module IteratorsMD import .Base: eltype, length, size, first, last, in, getindex, setindex!, min, max, zero, oneunit, isless, eachindex, convert, show, iterate, promote_rule, to_indices, copy, - isassigned + isassigned, lastindex, firstindex import .Base: +, -, *, (:) import .Base: simd_outer_range, simd_inner_length, simd_index, setindex @@ -103,6 +103,8 @@ module IteratorsMD # indexing getindex(index::CartesianIndex, i::Integer) = index.I[i] + firstindex(index::CartesianIndex) = firstindex(index.I) + lastindex(index::CartesianIndex) = lastindex(index.I) Base.get(A::AbstractArray, I::CartesianIndex, default) = get(A, I.I, default) eltype(::Type{T}) where {T<:CartesianIndex} = eltype(fieldtype(T, :I)) diff --git a/test/cartesian.jl b/test/cartesian.jl index 5db07b5316407..f9a0c8f976956 100644 --- a/test/cartesian.jl +++ b/test/cartesian.jl @@ -582,3 +582,9 @@ end c = CartesianIndex(3, 3) @test sprint(show, c) == "CartesianIndex(3, 3)" end + +@testset "CartesianIndex indexing with begin/end" begin + I = CartesianIndex(3,4) + @test I[begin] == I[1] + @test I[end] == I[2] +end From 1c3878c0740e9835ce604a5daca2c983e47e7389 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 3 Apr 2025 21:10:04 -0400 Subject: [PATCH 043/662] Merge adjacent implicit binding partitions (#57995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After: ``` julia> convert(Core.Binding, GlobalRef(Base, :Intrinsics)) Binding Base.Intrinsics 617:∞ - implicit `using` resolved to constant Core.Intrinsics 0:616 - undefined binding - guard entry julia> convert(Core.Binding, GlobalRef(Base, :Math)) Binding Base.Math 22128:∞ - constant binding to Base.Math 0:22127 - backdated constant binding to Base.Math ``` There is a bit of trickiness here. In particular, the question is, "when do we check" whether the partition next to the one we currently looked at happens to have the same implicit resolution as our current one. The most obvious answer is that we should do it on access, but in practice that would require essentially scanning back and considering every possible world age state at every lookup. This is undesirable - the lookup is not crazy expensive, but it can add up and most world ages we never touch, so it is also wasteful. This instead implements a different approach where we only perform the resolution for world ages that somebody actually asked about, but can then subsequently merge partitions if we do find that they are identical. The logic for that is a bit involved, since we need to be careful to keep the datastructure valid at every point, but does address the issue. Fixes #57923 --- src/jltypes.c | 4 +- src/julia.h | 6 +- src/module.c | 138 ++++++++++++++++++++++++++++++++++------------ src/rtutils.c | 4 +- src/staticdata.c | 12 ++-- src/toplevel.c | 4 +- test/rebinding.jl | 65 ++++++++++++++++++++++ 7 files changed, 186 insertions(+), 47 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 09394b6b0e02b..ebef8e8df1747 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3266,8 +3266,10 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(5, jl_any_type, jl_ulong_type, jl_ulong_type, jl_any_type/*jl_binding_partition_type*/, jl_ulong_type), jl_emptysvec, 0, 1, 0); - const static uint32_t binding_partition_atomicfields[] = { 0b01101 }; // Set fields 1, 3, 4 as atomic + const static uint32_t binding_partition_atomicfields[] = { 0b01111 }; // Set fields 1, 2, 3, 4 as atomic jl_binding_partition_type->name->atomicfields = binding_partition_atomicfields; + const static uint32_t binding_partition_constfields[] = { 0x100001 }; // Set fields 1, 5 as constant + jl_binding_partition_type->name->constfields = binding_partition_constfields; jl_binding_type = jl_new_datatype(jl_symbol("Binding"), core, jl_any_type, jl_emptysvec, diff --git a/src/julia.h b/src/julia.h index 63d29ea18aff8..e5e3428466ba1 100644 --- a/src/julia.h +++ b/src/julia.h @@ -760,7 +760,7 @@ typedef struct JL_ALIGNED_ATTR(8) _jl_binding_partition_t { * } restriction; */ jl_value_t *restriction; - size_t min_world; + _Atomic(size_t) min_world; _Atomic(size_t) max_world; _Atomic(struct _jl_binding_partition_t *) next; size_t kind; @@ -1942,8 +1942,8 @@ JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void); JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b JL_PROPAGATES_ROOT); JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_latest_resolved_and_const_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name); JL_DLLEXPORT jl_method_t *jl_method_def(jl_svec_t *argdata, jl_methtable_t *mt, jl_code_info_t *f, jl_module_t *module); JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo, size_t world, jl_code_instance_t **cache); diff --git a/src/module.c b/src/module.c index 7d77e933f8193..53466c27e8ac9 100644 --- a/src/module.c +++ b/src/module.c @@ -20,7 +20,7 @@ static jl_binding_partition_t *new_binding_partition(void) jl_binding_partition_t *bpart = (jl_binding_partition_t*)jl_gc_alloc(jl_current_task->ptls, sizeof(jl_binding_partition_t), jl_binding_partition_type); bpart->restriction = NULL; bpart->kind = (size_t)PARTITION_KIND_GUARD; - bpart->min_world = 0; + jl_atomic_store_relaxed(&bpart->min_world, 0); jl_atomic_store_relaxed(&bpart->max_world, (size_t)-1); jl_atomic_store_relaxed(&bpart->next, NULL); return bpart; @@ -40,9 +40,12 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition__(jl_binding_t *b { // Iterate through the list of binding partitions, keeping track of where to insert a new one for an implicit // resolution if necessary. - while (gap->replace && world < gap->replace->min_world) { + while (gap->replace) { + size_t replace_min_world = jl_atomic_load_relaxed(&gap->replace->min_world); + if (world >= replace_min_world) + break; gap->insert = &gap->replace->next; - gap->max_world = gap->replace->min_world - 1; + gap->max_world = replace_min_world - 1; gap->parent = (jl_value_t*)gap->replace; gap->replace = jl_atomic_load_relaxed(gap->insert); } @@ -126,13 +129,70 @@ static void update_implicit_resolution(struct implicit_search_resolution *to_upd static jl_binding_partition_t *jl_implicit_import_resolved(jl_binding_t *b, struct implicit_search_gap gap, struct implicit_search_resolution resolution) { + size_t new_kind = resolution.ultimate_kind | gap.inherited_flags; + size_t new_max_world = gap.max_world < resolution.max_world ? gap.max_world : resolution.max_world; + size_t new_min_world = gap.min_world > resolution.min_world ? gap.min_world : resolution.min_world; + jl_binding_partition_t *next = gap.replace; + if (jl_is_binding_partition(gap.parent)) { + // Check if we can merge this into the previous binding partition + jl_binding_partition_t *prev = (jl_binding_partition_t *)gap.parent; + size_t expected_prev_min_world = new_max_world + 1; + if (prev->restriction == resolution.binding_or_const && prev->kind == new_kind) { + if (!jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, new_min_world)) { + if (expected_prev_min_world <= new_min_world) { + return prev; + } + else if (expected_prev_min_world <= new_max_world) { + // Concurrent modification by another thread - bail. + return NULL; + } + // There remains a gap - proceed + } else { + if (next) { + size_t next_min_world = jl_atomic_load_relaxed(&next->min_world); + expected_prev_min_world = new_min_world; + for (;;) { + // We've updated the previous partition - check if we've closed a gap + size_t next_max_world = jl_atomic_load_relaxed(&next->max_world); + if (next_max_world == expected_prev_min_world-1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { + if (jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, next_min_world)) { + jl_binding_partition_t *nextnext = jl_atomic_load_relaxed(&next->next); + if (!jl_atomic_cmpswap(&prev->next, &next, nextnext)) { + // `next` may have been merged into its subsequent partition - we need to retry + assert(next); + continue; + } + // N.B.: This can lose modifications to next->{min_world, next}. + // However, those modifications could only have been for another implicit + // partition, so we are ok to lose them and recompute them later if necessary. + } + assert(expected_prev_min_world <= new_min_world); + } + break; + } + } + return prev; + } + } + } jl_binding_partition_t *new_bpart = new_binding_partition(); - jl_atomic_store_relaxed(&new_bpart->max_world, gap.max_world < resolution.max_world ? gap.max_world : resolution.max_world); - new_bpart->min_world = gap.min_world > resolution.min_world ? gap.min_world : resolution.min_world; - new_bpart->kind = resolution.ultimate_kind | gap.inherited_flags; + jl_atomic_store_relaxed(&new_bpart->max_world, new_max_world); + new_bpart->kind = new_kind; new_bpart->restriction = resolution.binding_or_const; jl_gc_wb_fresh(new_bpart, new_bpart->restriction); - jl_atomic_store_relaxed(&new_bpart->next, gap.replace); + + if (next) { + // See if we can merge the next partition into this one + size_t next_max_world = jl_atomic_load_relaxed(&next->max_world); + if (next_max_world == new_min_world - 1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { + // See above for potentially losing modifications to next. + new_min_world = jl_atomic_load_acquire(&next->min_world); + next = jl_atomic_load_relaxed(&next->next); + } + } + + jl_atomic_store_relaxed(&new_bpart->min_world, new_min_world); + jl_atomic_store_relaxed(&new_bpart->next, next); if (!jl_atomic_cmpswap(gap.insert, &gap.replace, new_bpart)) return NULL; jl_gc_wb(gap.parent, new_bpart); @@ -170,13 +230,11 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo struct _jl_module_using data = *module_usings_getidx(m, i); JL_UNLOCK(&m->lock); if (data.min_world > world) { - if (max_world > data.min_world) - max_world = data.min_world - 1; + max_world = WORLDMIN(max_world, data.min_world - 1); continue; } if (data.max_world < world) { - if (min_world < data.max_world) - min_world = data.max_world + 1; + min_world = WORLDMAX(min_world, data.max_world + 1); continue; } @@ -198,7 +256,7 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo while (tempbpart && jl_bkind_is_some_explicit_import(jl_binding_kind(tempbpart))) { max_world = WORLDMIN(max_world, jl_atomic_load_relaxed(&tempbpart->max_world)); - min_world = WORLDMAX(min_world, tempbpart->min_world); + min_world = WORLDMAX(min_world, jl_atomic_load_relaxed(&tempbpart->min_world)); tempb = (jl_binding_t*)tempbpart->restriction; tempbpart = jl_get_binding_partition_if_present(tempb, world, &gap); @@ -206,7 +264,7 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo int tempbpart_valid = tempbpart && (trust_cache || !jl_bkind_is_some_implicit(jl_binding_kind(tempbpart))); size_t tembppart_max_world = tempbpart_valid ? jl_atomic_load_relaxed(&tempbpart->max_world) : gap.max_world; - size_t tembppart_min_world = tempbpart ? WORLDMAX(tempbpart->min_world, gap.min_world) : gap.min_world; + size_t tembppart_min_world = tempbpart ? WORLDMAX(jl_atomic_load_relaxed(&tempbpart->min_world), gap.min_world) : gap.min_world; max_world = WORLDMIN(max_world, tembppart_max_world); min_world = WORLDMAX(min_world, tembppart_min_world); @@ -279,8 +337,15 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b assert(bpart == jl_atomic_load_relaxed(&b->partitions)); assert(bpart); struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_max_world+1, 0); - if (resolution.min_world == bpart->min_world) { - assert(bpart->restriction == resolution.binding_or_const && jl_binding_kind(bpart) == resolution.ultimate_kind); + int resolution_unchanged = bpart->restriction == resolution.binding_or_const && jl_binding_kind(bpart) == resolution.ultimate_kind; + size_t bpart_min_world = jl_atomic_load_relaxed(&bpart->min_world); + if (resolution.min_world == bpart_min_world) { + // The resolution has the same world bounds - it must be unchanged + assert(resolution_unchanged); + return bpart; + } else if (resolution_unchanged) { + // If the resolution is unchanged, we can still keep the bpart + assert(resolution.min_world > bpart_min_world); return bpart; } assert(resolution.min_world == new_max_world+1 && "Missed an invalidation or bad resolution bounds"); @@ -299,7 +364,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart) { struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_relaxed(&jl_world_counter), 0); - bpart->min_world = resolution.min_world; + jl_atomic_store_relaxed(&bpart->min_world, resolution.min_world); jl_atomic_store_relaxed(&bpart->max_world, resolution.max_world); bpart->restriction = resolution.binding_or_const; bpart->kind = resolution.ultimate_kind; @@ -336,7 +401,8 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b, jl_binding_partition_t *prev, size_t world) JL_GLOBALLY_ROOTED { // Helper for getting a binding partition for an older world after we've already looked up the partition for a newer world assert(b); - assert(prev->min_world > world); + // TODO: Is it possible for a concurrent lookup to have expanded this bpart, making this false? + assert(jl_atomic_load_relaxed(&prev->min_world) > world); return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, NULL); } @@ -362,10 +428,11 @@ JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b while (validated_min_world > min_world) { bpart = bpart ? jl_get_binding_partition_with_hint(b, bpart, validated_min_world - 1) : jl_get_binding_partition(b, validated_min_world - 1); - while (validated_min_world > min_world && validated_min_world > bpart->min_world) { + size_t bpart_min_world = jl_atomic_load_relaxed(&bpart->min_world); + while (validated_min_world > min_world && validated_min_world > bpart_min_world) { jl_binding_t *curb = b; jl_binding_partition_t *curbpart = bpart; - size_t cur_min_world = bpart->min_world; + size_t cur_min_world = bpart_min_world; size_t cur_max_world = validated_min_world - 1; jl_walk_binding_inplace_worlds(&curb, &curbpart, &cur_min_world, &cur_max_world, &maybe_depwarn, cur_max_world); enum jl_partition_kind kind = jl_binding_kind(curbpart); @@ -494,7 +561,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_errorf("cannot declare %s.%s constant; it was already declared global", jl_symbol_name(mod->name), jl_symbol_name(var)); } - if (bpart->min_world == new_world) { + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) { bpart->kind = constant_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; if (val) @@ -515,9 +582,10 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( need_backdate = 0; break; } - if (prev_bpart->min_world == 0) + size_t prev_bpart_min_world = jl_atomic_load_relaxed(&prev_bpart->min_world); + if (prev_bpart_min_world == 0) break; - prev_bpart = jl_get_binding_partition(b, prev_bpart->min_world - 1); + prev_bpart = jl_get_binding_partition(b, prev_bpart_min_world - 1); } } // If backdate is required, replace each existing partition by a new one. @@ -530,7 +598,8 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( while (1) { backdate_bpart->kind = (size_t)PARTITION_KIND_BACKDATED_CONST | (prev_bpart->kind & 0xf0); backdate_bpart->restriction = val; - backdate_bpart->min_world = prev_bpart->min_world; + jl_atomic_store_relaxed(&backdate_bpart->min_world, + jl_atomic_load_relaxed(&prev_bpart->min_world)); jl_gc_wb_fresh(backdate_bpart, val); jl_atomic_store_relaxed(&backdate_bpart->max_world, jl_atomic_load_relaxed(&prev_bpart->max_world)); @@ -861,17 +930,19 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b) return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_latest_resolved_and_const_debug_only(jl_binding_t *b) { // Unlike jl_get_binding_value_if_const this doesn't try to allocate new binding partitions if they - // don't already exist, making this JL_NOTSAFEPOINT. + // don't already exist, making this JL_NOTSAFEPOINT. However, as a result, this may fail to return + // a value - even if one does exist. It should only be used for reflection/debugging when the integrity + // of the runtime is not guaranteed. if (!b) return NULL; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (bpart->min_world > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (jl_atomic_load_relaxed(&bpart->min_world) > jl_current_task->world_age || jl_current_task->world_age > max_world) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -882,17 +953,16 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_t *b) { - // Unlike jl_get_binding_value this doesn't try to allocate new binding partitions if they - // don't already exist, making this JL_NOTSAFEPOINT. + // See note above. Use for debug/reflection purposes only. if (!b) return NULL; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (bpart->min_world > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (jl_atomic_load_relaxed(&bpart->min_world) > jl_current_task->world_age || jl_current_task->world_age > max_world) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -1486,7 +1556,7 @@ JL_DLLEXPORT void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sy // jl_declare_constant_val3(NULL, m, var, (jl_value_t*)jl_any_type, kind, 0); jl_binding_t *bp = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(bp, 0); - assert(bpart->min_world == 0); + assert(jl_atomic_load_relaxed(&bpart->min_world) == 0); jl_atomic_store_relaxed(&bpart->max_world, ~(size_t)0); // jl_check_new_binding_implicit likely incorrectly truncated it if (exported) jl_atomic_fetch_or_relaxed(&bp->flags, BINDING_FLAG_PUBLICP); @@ -1500,7 +1570,7 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var // this function is dangerous and unsound. do not use. jl_binding_t *bp = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); - bpart->min_world = 0; + jl_atomic_store_relaxed(&bpart->min_world, 0); jl_atomic_store_release(&bpart->max_world, ~(size_t)0); bpart->kind = PARTITION_KIND_CONST | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; @@ -1583,7 +1653,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, assert(jl_atomic_load_relaxed(&b->partitions) == old_bpart); jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); - new_bpart->min_world = new_world; + jl_atomic_store_relaxed(&new_bpart->min_world, new_world); if ((kind & PARTITION_MASK_KIND) == PARTITION_FAKE_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_world, 0); @@ -1635,7 +1705,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding(jl_binding_t *b, size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; jl_binding_partition_t *bpart = jl_replace_binding_locked(b, old_bpart, restriction_val, kind, new_world); - if (bpart && bpart->min_world == new_world) + if (bpart && jl_atomic_load_relaxed(&bpart->min_world) == new_world) jl_atomic_store_release(&jl_world_counter, new_world); JL_UNLOCK(&world_counter_lock); diff --git a/src/rtutils.c b/src/rtutils.c index 6515b80c5d2b5..3283388cb1d9b 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -579,7 +579,7 @@ JL_DLLEXPORT jl_value_t *jl_stderr_obj(void) JL_NOTSAFEPOINT if (jl_base_module == NULL) return NULL; jl_binding_t *stderr_obj = jl_get_module_binding(jl_base_module, jl_symbol("stderr"), 0); - return stderr_obj ? jl_get_binding_value_if_resolved(stderr_obj) : NULL; + return stderr_obj ? jl_get_binding_value_if_resolved_debug_only(stderr_obj) : NULL; } // toys for debugging --------------------------------------------------------- @@ -674,7 +674,7 @@ static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; if (globname && dv->name->module) { jl_binding_t *b = jl_get_module_binding(dv->name->module, globname, 0); - jl_value_t *bv = jl_get_binding_value_if_resolved_and_const(b); + jl_value_t *bv = jl_get_binding_value_if_latest_resolved_and_const_debug_only(b); // The `||` makes this function work for both function instances and function types. if (bv && (bv == v || jl_typeof(bv) == v)) return 1; diff --git a/src/staticdata.c b/src/staticdata.c index b007fc04eeb4b..c96faf5a71a54 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1749,7 +1749,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED write_uint(f, 0); } } else { - write_uint(f, bpart->min_world); + write_uint(f, jl_atomic_load_relaxed(&bpart->min_world)); write_uint(f, max_world); } write_pointerfield(s, (jl_value_t*)jl_atomic_load_relaxed(&bpart->next)); @@ -3618,7 +3618,7 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t // and allocate a fresh bpart? jl_update_loaded_bpart(b, bpart); bpart->kind |= (raw_kind & PARTITION_MASK_FLAG); - if (bpart->min_world > jl_require_world) + if (jl_atomic_load_relaxed(&bpart->min_world) > jl_require_world) goto invalidated; } if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) @@ -3629,7 +3629,8 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t jl_binding_partition_t *latest_imported_bpart = jl_atomic_load_relaxed(&imported_binding->partitions); if (!latest_imported_bpart) return 1; - if (latest_imported_bpart->min_world <= bpart->min_world) { + if (jl_atomic_load_relaxed(&latest_imported_bpart->min_world) <= + jl_atomic_load_relaxed(&bpart->min_world)) { add_backedge: // Imported binding is still valid if ((kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) && @@ -3640,8 +3641,9 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t } else { // Binding partition was invalidated - assert(bpart->min_world == jl_require_world); - bpart->min_world = latest_imported_bpart->min_world; + assert(jl_atomic_load_relaxed(&bpart->min_world) == jl_require_world); + jl_atomic_store_relaxed(&bpart->min_world, + jl_atomic_load_relaxed(&latest_imported_bpart->min_world)); } invalidated: // We need to go through and re-validate any bindings in the same image that diff --git a/src/toplevel.c b/src/toplevel.c index cbf220f0016f1..f1fff694926ba 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -309,7 +309,7 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in goto check_type; } check_safe_newbinding(gm, gs); - if (bpart->min_world == new_world) { + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) { bpart->kind = new_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = global_type; if (global_type) @@ -730,7 +730,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2( JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_binding_partition_t *bpart = jl_declare_constant_val3(b, mod, var, val, constant_kind, new_world); - if (bpart->min_world == new_world) + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) jl_atomic_store_release(&jl_world_counter, new_world); JL_UNLOCK(&world_counter_lock); return bpart; diff --git a/test/rebinding.jl b/test/rebinding.jl index ab9696c7f0222..1c2a4d7a0b91c 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -318,3 +318,68 @@ module UndefinedTransitions @test Base.Compiler.is_nothrow(Base.Compiler.decode_effects(ci.ipo_purity_bits)) end end + +# Identical implicit partitions should be merge (#57923) +for binding in (convert(Core.Binding, GlobalRef(Base, :Math)), + convert(Core.Binding, GlobalRef(Base, :Intrinsics))) + # Test that these both only have two partitions + @test isdefined(binding, :partitions) + @test isdefined(binding.partitions, :next) + @test !isdefined(binding.partitions.next, :next) +end + +# Test various scenarios for implicit partition merging +module MergeStress + for i = 1:5 + @eval module $(Symbol("M$i")) + export x, y + const x = 1 + const y = 2 + end + end + const before = Base.get_world_counter() + using .M1 + const afterM1 = Base.get_world_counter() + using .M2 + const afterM2 = Base.get_world_counter() + using .M3 + const afterM3 = Base.get_world_counter() + using .M4 + const afterM4 = Base.get_world_counter() + using .M5 + const afterM5 = Base.get_world_counter() +end + +function count_partitions(b::Core.Binding) + n = 0 + isdefined(b, :partitions) || return n + bpart = b.partitions + while true + n += 1 + isdefined(bpart, :next) || break + bpart = bpart.next + end + return n +end +using Base: invoke_in_world + +const xbinding = convert(Core.Binding, GlobalRef(MergeStress, :x)) +function access_and_count(point) + invoke_in_world(getglobal(MergeStress, point), getglobal, MergeStress, :x) + count_partitions(xbinding) +end + +@test count_partitions(xbinding) == 0 +@test access_and_count(:afterM1) == 1 +# M2 is the first change to the `usings` table after M1. The partitions +# can and should be merged +@test access_and_count(:afterM2) == 1 + +# There is a gap between M2 and M5 - the partitions should not be merged +@test access_and_count(:afterM5) == 2 + +# M4 and M5 are adjacent, these partitions should also be merged (in the opposite direction) +@test access_and_count(:afterM4) == 2 + +# M3 connects all, so we should have a single partition +@test access_and_count(:afterM3) == 1 From a3c48d742779fe3b866911b7477afe88d082c643 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 3 Apr 2025 22:13:22 -0400 Subject: [PATCH 044/662] fix generated_body_to_codeinfo to avoid `string` (which is not always defined) (#57925) Fixes a discrepancy between the code in C before #57230 and in Julia afterwards, making sure to sequence these method definitions correctly. Not sure how to write a reliable test since it is specific to when this generated function is defined relative to the helpers used by this thunk, but the issue/fix is visible with: ``` $ ./julia -e 'code_lowered(ntuple, (Returns{Nothing}, Val{1000000}))' ``` Fix #57301 --- base/Base_compiler.jl | 44 ++++++++++++++++++++++++------------------- base/essentials.jl | 2 ++ base/expr.jl | 3 ++- base/pointer.jl | 2 -- test/staged.jl | 2 ++ 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index d4a106564a076..e22a7d980f06c 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -157,6 +157,31 @@ if false println(io::IO, x...) = Core.println(io, x...) end +## Load essential files and libraries +include("essentials.jl") + +# Because lowering inserts direct references, it is mandatory for this binding +# to exist before we start inferring code. +function string end +import Core: String + +# For OS specific stuff +# We need to strcat things here, before strings are really defined +function strcat(x::String, y::String) + out = ccall(:jl_alloc_string, Ref{String}, (Int,), Core.sizeof(x) + Core.sizeof(y)) + gc_x = @_gc_preserve_begin(x) + gc_y = @_gc_preserve_begin(y) + gc_out = @_gc_preserve_begin(out) + out_ptr = unsafe_convert(Ptr{UInt8}, out) + unsafe_copyto!(out_ptr, unsafe_convert(Ptr{UInt8}, x), Core.sizeof(x)) + unsafe_copyto!(out_ptr + Core.sizeof(x), unsafe_convert(Ptr{UInt8}, y), Core.sizeof(y)) + @_gc_preserve_end(gc_x) + @_gc_preserve_end(gc_y) + @_gc_preserve_end(gc_out) + return out +end + + """ time_ns()::UInt64 @@ -171,8 +196,6 @@ const _DOCS_ALIASING_WARNING = """ Behavior can be unexpected when any mutated argument shares memory with any other argument. """ -## Load essential files and libraries -include("essentials.jl") include("ctypes.jl") include("gcutils.jl") include("generator.jl") @@ -283,7 +306,6 @@ include("rounding.jl") include("float.jl") # Lazy strings -import Core: String include("strings/lazy.jl") function cld end @@ -320,22 +342,6 @@ using .Order include("coreir.jl") include("invalidation.jl") -# Because lowering inserts direct references, it is mandatory for this binding -# to exist before we start inferring code. -function string end - -# For OS specific stuff -# We need to strcat things here, before strings are really defined -function strcat(x::String, y::String) - out = ccall(:jl_alloc_string, Ref{String}, (Csize_t,), Core.sizeof(x) + Core.sizeof(y)) - GC.@preserve x y out begin - out_ptr = unsafe_convert(Ptr{UInt8}, out) - unsafe_copyto!(out_ptr, unsafe_convert(Ptr{UInt8}, x), Core.sizeof(x)) - unsafe_copyto!(out_ptr + Core.sizeof(x), unsafe_convert(Ptr{UInt8}, y), Core.sizeof(y)) - end - return out -end - BUILDROOT::String = "" DATAROOT::String = "" const DL_LOAD_PATH = String[] diff --git a/base/essentials.jl b/base/essentials.jl index 029205666b2bc..9f70a90bdac7d 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -723,6 +723,8 @@ cconvert(::Type{<:Ptr}, x) = x # but defer the conversion to Ptr to unsafe_conve unsafe_convert(::Type{T}, x::T) where {T} = x # unsafe_convert (like convert) defaults to assuming the convert occurred unsafe_convert(::Type{T}, x::T) where {T<:Ptr} = x # to resolve ambiguity with the next method unsafe_convert(::Type{P}, x::Ptr) where {P<:Ptr} = convert(P, x) +unsafe_convert(::Type{Ptr{UInt8}}, s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) +unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, (Any,), s) """ reinterpret(::Type{Out}, x::In) diff --git a/base/expr.jl b/base/expr.jl index 634e2ddb7b4de..91b37af17c231 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1672,7 +1672,8 @@ function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) ci = ccall(:jl_expand, Any, (Any, Any), ex, defmod) if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error - error("syntax: $(ci.args[1])") + msg = ci.args[1] + error(msg isa String ? strcat("syntax: ", msg) : msg) end error("The function body AST defined by this @generated function is not pure. This likely means it contains a closure, a comprehension or a generator.") end diff --git a/base/pointer.jl b/base/pointer.jl index fdbcc7bb427b9..72c567eaf2a85 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -59,8 +59,6 @@ cconvert(::Type{Ptr{UInt8}}, s::AbstractString) = String(s) cconvert(::Type{Ptr{Int8}}, s::AbstractString) = String(s) unsafe_convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), x) unsafe_convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x) -unsafe_convert(::Type{Ptr{UInt8}}, s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) -unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, (Any,), s) cconvert(::Type{<:Ptr}, a::Array) = getfield(a, :ref) unsafe_convert(::Type{Ptr{S}}, a::AbstractArray{T}) where {S,T} = convert(Ptr{S}, unsafe_convert(Ptr{T}, a)) diff --git a/test/staged.jl b/test/staged.jl index d416b0f9a22f0..4fb5d03331711 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -477,3 +477,5 @@ module GeneratedScope57417 end @test g() == 1 end + +@test_throws "syntax: expression too large" code_lowered(ntuple, (Returns{Nothing}, Val{1000000})) From 3627a85749e4aa8197edb56dd93d1b21fd54dbe8 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Fri, 4 Apr 2025 04:13:40 +0200 Subject: [PATCH 045/662] fix `nextpow`, `prevpow` for types without `typemax` (#49669) Without this change `prevpow` and `nextpow` fail for, e.g., `BigInt`; incorrectly throwing a `MethodError`. For example, calls like `prevpow(3, big"10")` or `nextpow(3, big"10")` fail. The issue is that the code incorrectly assumes the existence of a `typemax` method. Another issue was a missing promote for the arguments of a `mul_with_overflow` call. Fixes #57906 --- base/intfuncs.jl | 8 +++++--- test/intfuncs.jl | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index e14f48170f0e8..fccd8fcb9accc 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -570,7 +570,8 @@ function nextpow(a::Real, x::Real) n = ceil(Integer,log(a, x)) # round-off error of log can go either direction, so need some checks p = a^(n-1) - x > typemax(p) && throw(DomainError(x,"argument is beyond the range of type of the base")) + hastypemax(typeof(p)) && x > typemax(p) && + throw(DomainError(x,"argument is beyond the range of type of the base")) p >= x && return p wp = a^n wp > p || throw(OverflowError("result is beyond the range of type of the base")) @@ -611,9 +612,10 @@ function prevpow(a::T, x::Real) where T <: Real n = floor(Integer,log(a, x)) # round-off error of log can go either direction, so need some checks p = a^n - x > typemax(p) && throw(DomainError(x,"argument is beyond the range of type of the base")) + hastypemax(typeof(p)) && x > typemax(p) && + throw(DomainError(x,"argument is beyond the range of type of the base")) if a isa Integer - wp, overflow = mul_with_overflow(a, p) + wp, overflow = mul_with_overflow(promote(a, p)...) wp <= x && !overflow && return wp else wp = a^(n+1) diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 38f29344d2f30..adcfee2a050dd 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -326,6 +326,14 @@ end end @testset "nextpow/prevpow" begin + fs = (prevpow, nextpow) + types = (Int8, BigInt, BigFloat) + for f ∈ fs, P ∈ types, R ∈ types, p ∈ 1:20, r ∈ 2:5 + q = P(p) + n = R(r) + @test f(r, p) == f(n, q) + end + @test nextpow(2, 3) == 4 @test nextpow(2, 4) == 4 @test nextpow(2, 7) == 8 @@ -339,7 +347,14 @@ end @test prevpow(10, 101.0) === 100 @test prevpow(10.0, 101) === 100.0 @test_throws DomainError prevpow(0, 3) - @test_throws DomainError prevpow(0, 3) + @test_throws DomainError prevpow(3, 0) + + # "argument is beyond the range of type of the base" + @test_throws DomainError prevpow(Int8(3), 243) + @test_throws DomainError nextpow(Int8(3), 243) + + # "result is beyond the range of type of the base" + @test_throws OverflowError nextpow(Int8(3), 82) end @testset "ndigits/ndigits0z" begin From 15e0debb62eb0445352919681315ed78c07e7919 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 4 Apr 2025 05:46:27 -0500 Subject: [PATCH 046/662] Add matches for invoke invalidations (#57999) Edge invalidations of the form ```julia Package InvalidA: module InvalidA f(::Integer) = 1 invokesfs(x) = invoke(f, Tuple{Signed}, x) end Package InvalidB: module InvalidB using InvalidA InvalidA.invokesfs(1) # precompile end ``` used as ```julia using PkgA InvalidA.f(::Signed) = 4 using PkgB ``` did not formerly attribute a "cause": ``` ... Tuple{typeof(InvalidA.f), Signed} "insert_backedges_callee" CodeInstance for MethodInstance for InvalidA.invokesfs(::Int64) nothing ... ``` This fills in the new method that replaced the previous dispatch. --------- Co-authored-by: Jameson Nash --- base/staticdata.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index 94526cc4f7bd3..d51e7892bbe56 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -162,8 +162,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi else meth = callee::Method end - min_valid2, max_valid2 = verify_invokesig(edge, meth, world) - matches = nothing + min_valid2, max_valid2, matches = verify_invokesig(edge, meth, world) j += 2 end if minworld < min_valid2 @@ -295,6 +294,7 @@ end function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt) @assert invokesig isa Type local minworld::UInt, maxworld::UInt + matched = nothing if invokesig === expected.sig # the invoke match is `expected` for `expected->sig`, unless `expected` is invalid minworld = expected.primary_world @@ -314,12 +314,15 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world if matched === nothing maxworld = 0 - elseif matched.method != expected - maxworld = 0 + else + matched = Any[matched.method] + if matched[] !== expected + maxworld = 0 + end end end end - return minworld, maxworld + return minworld, maxworld, matched end end # module StaticData From 2634a665f5f6b7f18804c94555e4df10171c36e7 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Apr 2025 07:28:28 -0400 Subject: [PATCH 047/662] Profile: make `@Compiler` test robust (#58002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://buildkite.com/julialang/julia-master/builds/46446#0195f712-1844-4e81-8b16-27b953fedcd3/899-1778 ```   | Error in testset Profile:   | Test Failed at /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/stdlib/v1.13/Profile/test/runtests.jl:231   | Expression: occursin("@Compiler" * slash, str)   | Evaluated: occursin("@Compiler/", "Overhead ╎ [+additional indent] Count File:Line Function\n=========================================================\n ╎9 @juliasrc/task.c:1249 start_task\n ╎ 9 @juliasrc/julia.h:2353 jl_apply\n ╎ 9 @juliasrc/gf.c:3693 ijl_apply_generic\n ╎ 9 @juliasrc/gf.c:3493 _jl_invoke\n ╎ 9 [unknown stackframe]\n ╎ 9 @Distributed/src/process_messages.jl:287 (::Distributed.var\"#handle_msg##2#handle_msg##3\"{Distributed.CallMsg{:call_fetch}, Distributed.MsgHeader, Sockets.TCPSocket})()\n ╎ ╎ 9 @Distributed/src/process_messages.jl:70 run_work_thunk(thunk::Distributed.var\"#handle_msg##4#handle_msg##5\"{Distributed.CallMsg{:call_fetch}}, print_error::Bool)\n ╎ ╎ 9 @Distributed/src/process_messages.jl:287 (::Distributed.var\"#handle_msg##4#handle_msg##5\"{Distributed.CallMsg{:call_fetch}})()\n ╎ ╎ 9 @juliasrc/builtins.c:841 jl_f__apply_iterate\n ╎ ╎ 9 @juliasrc/julia.h:2353 jl_apply\n ╎ ╎ 9 @juliasrc/gf.c:3693 ijl_apply_generic\n ╎ ╎ ╎ 9 @juliasrc/gf.c:3493 _jl_invoke\n ╎ ╎ ╎ 9 @Base/Base_compiler.jl:223 kwcall(::@NamedTuple{seed::UInt128}, ::typeof(invokelatest), ::Function, ::String, ::Vararg{String})\n ╎ ╎ ╎ 9 @juliasrc/builtins.c:841 jl_f__apply_iterate\n ╎ ╎ ╎ 9 @juliasrc/julia.h:2353 jl_apply\n ╎ ╎ ╎ 9 @juliasrc/gf.c:3693 ijl_apply_generic\n ╎ ╎ ╎ ╎ 9 @juliasrc/gf.c:3493 _jl_invoke\n ╎ ╎ ╎ ╎ 9 @juliasrc/builtins.c:853 jl_f_invokelatest\n ╎ ╎ ╎ ╎ 9 @juliasrc/julia.h:2353 jl_apply\n ╎ ╎ ╎ ╎ 9 @juliasrc/gf.c:3693 ijl_apply_generic\n ╎ ╎ ╎ ╎ 9 @juliasrc/gf.c:3493 _jl_invoke\n ╎ ╎ ╎ ╎ ╎ 9 [unknown stackframe]\n ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:7 kwcall(::@NamedTuple{seed::UInt128}, ::typeof(runtests), name::String, path::String)\n ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:7 runtests\n ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:13 runtests(name::String, path::String, isolate::Bool; seed::UInt128)\n ╎ ╎ ╎ ╎ ╎ 9 @Base/env.jl:265 withenv(f::var\"#4#5\"{UInt128, String, String, Bool, Bool}, keyvals::Pair{String, Bool})\n ╎ ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:27 (::var\"#4#5\"{UInt128, String, String, Bool, Bool})()\n ╎ ╎ ╎ ╎ ╎ ╎ 9 @Base/timing.jl:621 macro expansion\n ╎ ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:29 macro expansion\n ╎ ╎ ╎ ╎ ╎ ╎ 9 @Test/src/Test.jl:1835 macro expansion\n ╎ ╎ ╎ ╎ ╎ ╎ 9 /cache/build/tester-amdci5-10/julialang/julia-master/julia-7c9af464cc/share/julia/test/testdefs.jl:37 macro expansion\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @Base/Base.jl:303 include(mod::Module, _path::String)\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @Base/loading.jl:2925 _include(mapexpr::Function, mod::Module, _path::String)\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/gf.c:3693 ijl_apply_generic\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/gf.c:3493 _jl_invoke\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @Base/loading.jl:2865 include_string(mapexpr::typeof(identity), mod::Module, code::String, filename::String)\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @Base/boot.jl:489 eval(m::Module, e::Any)\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/toplevel.c:1095 ijl_toplevel_eval_in\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/toplevel.c:1050 ijl_toplevel_eval\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/toplevel.c:978 jl_toplevel_eval_flex\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/toplevel.c:1038 jl_toplevel_eval_flex\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:897 jl_interpret_toplevel_thunk\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:557 eval_body\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:557 eval_body\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:557 eval_body\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:692 eval_body\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:193 eval_stmt_value\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:242 eval_value\n ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ ╎ 9 @juliasrc/interpreter.c:124 do_call\n ╎ ╎ ╎ ╎ ... ``` --- stdlib/Profile/test/runtests.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index b487d8963f156..0ae773ec8f284 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -220,9 +220,19 @@ end import InteractiveUtils +@generated function compile_takes_1_second(x) + t = time_ns() + while time_ns() < t + 1e9 + # busy wait for 1 second + end + return :(x) +end @testset "Module short names" begin Profile.clear() - @profile InteractiveUtils.peakflops() + @profile begin + @eval compile_takes_1_second(1) # to increase chance of profiling hitting compilation code + InteractiveUtils.peakflops() + end io = IOBuffer() ioc = IOContext(io, :displaysize=>(1000,1000)) Profile.print(ioc, C=true) From 9af96508e9715e22154fc7b5a7283ad41d23765a Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Apr 2025 09:30:32 -0400 Subject: [PATCH 048/662] Logging: Improve threadsafety (#57591) Closes https://github.com/JuliaLang/julia/issues/57376 Closes https://github.com/JuliaLang/julia/issues/34037 - Adds a lock in `SimpleLogger` and `ConsoleLogger` for use on maxlog tracking and stream writes to improve threadsafety. Closely similar to https://github.com/JuliaLang/julia/pull/54497 - Turns the internal `_min_enabled_level` into a `Threads.Atomic`. There are [some direct interactions](https://juliahub.com/ui/Search?type=code&q=_min_enabled_level&w=true) to this internal in the ecosystem, but they should still work ``` julia> Base.CoreLogging._min_enabled_level[] = Logging.Info+1 LogLevel(1) ``` - Brings tests over from https://github.com/JuliaLang/julia/pull/57448 Performance seems highly similar: ### Master ``` julia> @time for i in 1:10000 @info "foo" maxlog=10000000 end [ Info: foo ... 0.481446 seconds (1.33 M allocations: 89.226 MiB, 0.49% gc time) ``` ### This PR ``` 0.477235 seconds (1.31 M allocations: 79.002 MiB, 1.77% gc time) ``` --- base/logging/ConsoleLogger.jl | 21 +++++++++----- base/logging/logging.jl | 36 ++++++++++++++++-------- stdlib/Logging/test/runtests.jl | 43 +++++++++++++++++++++++++++++ stdlib/Logging/test/threads_exec.jl | 13 +++++++++ stdlib/Test/src/logging.jl | 2 +- 5 files changed, 95 insertions(+), 20 deletions(-) create mode 100644 stdlib/Logging/test/threads_exec.jl diff --git a/base/logging/ConsoleLogger.jl b/base/logging/ConsoleLogger.jl index 818b2272b773c..8766d0ae56331 100644 --- a/base/logging/ConsoleLogger.jl +++ b/base/logging/ConsoleLogger.jl @@ -9,6 +9,9 @@ interactive work with the Julia REPL. Log levels less than `min_level` are filtered out. +This Logger is thread-safe, with locks for both orchestration of message +limits i.e. `maxlog`, and writes to the stream. + Message formatting can be controlled by setting keyword arguments: * `meta_formatter` is a function which takes the log event metadata @@ -24,6 +27,7 @@ Message formatting can be controlled by setting keyword arguments: """ struct ConsoleLogger <: AbstractLogger stream::IO + lock::ReentrantLock # do not log within this lock min_level::LogLevel meta_formatter show_limited::Bool @@ -33,19 +37,19 @@ end function ConsoleLogger(stream::IO, min_level=Info; meta_formatter=default_metafmt, show_limited=true, right_justify=0) - ConsoleLogger(stream, min_level, meta_formatter, + ConsoleLogger(stream, ReentrantLock(), min_level, meta_formatter, show_limited, right_justify, Dict{Any,Int}()) end function ConsoleLogger(min_level=Info; meta_formatter=default_metafmt, show_limited=true, right_justify=0) - ConsoleLogger(closed_stream, min_level, meta_formatter, + ConsoleLogger(closed_stream, ReentrantLock(), min_level, meta_formatter, show_limited, right_justify, Dict{Any,Int}()) end shouldlog(logger::ConsoleLogger, level, _module, group, id) = - get(logger.message_limits, id, 1) > 0 + @lock logger.lock get(logger.message_limits, id, 1) > 0 min_enabled_level(logger::ConsoleLogger) = logger.min_level @@ -109,9 +113,11 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module hasmaxlog = haskey(kwargs, :maxlog) ? 1 : 0 maxlog = get(kwargs, :maxlog, nothing) if maxlog isa Core.BuiltinInts - remaining = get!(logger.message_limits, id, Int(maxlog)::Int) - logger.message_limits[id] = remaining - 1 - remaining > 0 || return + @lock logger.lock begin + remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return + logger.message_limits[id] = remaining - 1 + end end # Generate a text representation of the message and all key value pairs, @@ -184,6 +190,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module println(iob) end - write(stream, take!(buf)) + b = take!(buf) + @lock logger.lock write(stream, b) nothing end diff --git a/base/logging/logging.jl b/base/logging/logging.jl index a9e41777f29b8..b43b7d9faa594 100644 --- a/base/logging/logging.jl +++ b/base/logging/logging.jl @@ -132,6 +132,7 @@ isless(a::LogLevel, b::LogLevel) = isless(a.level, b.level) +(level::LogLevel, inc::Integer) = LogLevel(level.level+inc) -(level::LogLevel, inc::Integer) = LogLevel(level.level-inc) convert(::Type{LogLevel}, level::Integer) = LogLevel(level) +convert(::Type{Int32}, level::LogLevel) = level.level """ BelowMinLevel @@ -171,7 +172,8 @@ Alias for [`LogLevel(1_000_001)`](@ref LogLevel). const AboveMaxLevel = LogLevel( 1000001) # Global log limiting mechanism for super fast but inflexible global log limiting. -const _min_enabled_level = Ref{LogLevel}(Debug) +# Atomic ensures that the value is always consistent across threads. +const _min_enabled_level = Threads.Atomic{Int32}(Debug) function show(io::IO, level::LogLevel) if level == BelowMinLevel print(io, "BelowMinLevel") @@ -394,7 +396,7 @@ function logmsg_code(_module, file, line, level, message, exs...) level = $level # simplify std_level code emitted, if we know it is one of our global constants std_level = $(level isa Symbol ? :level : :(level isa $LogLevel ? level : convert($LogLevel, level)::$LogLevel)) - if std_level >= $(_min_enabled_level)[] + if std_level.level >= $(_min_enabled_level)[] group = $(log_data._group) _module = $(log_data._module) logger = $(current_logger_for_env)(std_level, group, _module) @@ -541,7 +543,8 @@ with_logstate(f::Function, logstate) = @with(CURRENT_LOGSTATE => logstate, f()) Disable all log messages at log levels equal to or less than `level`. This is a *global* setting, intended to make debug logging extremely cheap when -disabled. +disabled. Note that this cannot be used to enable logging that is currently disabled +by other mechanisms. # Examples ```julia @@ -663,17 +666,21 @@ close(closed_stream) Simplistic logger for logging all messages with level greater than or equal to `min_level` to `stream`. If stream is closed then messages with log level greater or equal to `Warn` will be logged to `stderr` and below to `stdout`. + +This Logger is thread-safe, with a lock taken around orchestration of message +limits i.e. `maxlog`, and writes to the stream. """ struct SimpleLogger <: AbstractLogger stream::IO + lock::ReentrantLock min_level::LogLevel message_limits::Dict{Any,Int} end -SimpleLogger(stream::IO, level=Info) = SimpleLogger(stream, level, Dict{Any,Int}()) +SimpleLogger(stream::IO, level=Info) = SimpleLogger(stream, ReentrantLock(), level, Dict{Any,Int}()) SimpleLogger(level=Info) = SimpleLogger(closed_stream, level) shouldlog(logger::SimpleLogger, level, _module, group, id) = - get(logger.message_limits, id, 1) > 0 + @lock logger.lock get(logger.message_limits, id, 1) > 0 min_enabled_level(logger::SimpleLogger) = logger.min_level @@ -684,15 +691,14 @@ function handle_message(logger::SimpleLogger, level::LogLevel, message, _module, @nospecialize maxlog = get(kwargs, :maxlog, nothing) if maxlog isa Core.BuiltinInts - remaining = get!(logger.message_limits, id, Int(maxlog)::Int) - logger.message_limits[id] = remaining - 1 - remaining > 0 || return + @lock logger.lock begin + remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return + logger.message_limits[id] = remaining - 1 + end end buf = IOBuffer() stream::IO = logger.stream - if !(isopen(stream)::Bool) - stream = stderr - end iob = IOContext(buf, stream) levelstr = level == Warn ? "Warning" : string(level) msglines = eachsplit(chomp(convert(String, string(message))::String), '\n') @@ -706,7 +712,13 @@ function handle_message(logger::SimpleLogger, level::LogLevel, message, _module, println(iob, "│ ", key, " = ", val) end println(iob, "└ @ ", _module, " ", filepath, ":", line) - write(stream, take!(buf)) + b = take!(buf) + @lock logger.lock begin + if !(isopen(stream)::Bool) + stream = stderr + end + write(stream, b) + end nothing end diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 2fedbde557078..3e92b7d9e2697 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -306,4 +306,47 @@ end @test isempty(undoc) end +@testset "Logging when multithreaded" begin + n = 10000 + cmd = `$(Base.julia_cmd()) -t4 --color=no $(joinpath(@__DIR__, "threads_exec.jl")) $n` + fname = tempname() + @testset "Thread safety" begin + f = open(fname, "w") + @test success(run(pipeline(cmd, stderr=f))) + close(f) + end + + @testset "No tearing in log printing" begin + # Check for print tearing by verifying that each log entry starts and ends correctly + f = open(fname, "r") + entry_start = r"┌ (Info|Warning|Error): iteration" + entry_end = r"└ " + + open_entries = 0 + total_entries = 0 + for line in eachline(fname) + starts = count(entry_start, line) + starts > 1 && error("Interleaved logs: Multiple log entries started on one line") + if starts == 1 + startswith(line, entry_start) || error("Interleaved logs: Log entry started in the middle of a line") + open_entries += 1 + total_entries += 1 + end + + ends = count(entry_end, line) + starts == 1 && ends == 1 && error("Interleaved logs: Log entry started and and another ended on one line") + ends > 1 && error("Interleaved logs: Multiple log entries ended on one line") + if ends == 1 + startswith(line, entry_end) || error("Interleaved logs: Log entry ended in the middle of a line") + open_entries -= 1 + end + # Ensure no mismatched log entries + open_entries >= 0 || error("Interleaved logs") + end + + @test open_entries == 0 # Ensure all entries closed properly + @test total_entries == n * 3 # Ensure all logs were printed (3 because @debug is hidden) + end +end + end diff --git a/stdlib/Logging/test/threads_exec.jl b/stdlib/Logging/test/threads_exec.jl new file mode 100644 index 0000000000000..497a22b1c7b22 --- /dev/null +++ b/stdlib/Logging/test/threads_exec.jl @@ -0,0 +1,13 @@ +using Logging + +function test_threads_exec(n) + Threads.@threads for i in 1:n + @debug "iteration" maxlog=1 _id=Symbol("$(i)_debug") i Threads.threadid() + @info "iteration" maxlog=1 _id=Symbol("$(i)_info") i Threads.threadid() + @warn "iteration" maxlog=1 _id=Symbol("$(i)_warn") i Threads.threadid() + @error "iteration" maxlog=1 _id=Symbol("$(i)_error") i Threads.threadid() + end +end + +n = parse(Int, ARGS[1]) +test_threads_exec(n) diff --git a/stdlib/Test/src/logging.jl b/stdlib/Test/src/logging.jl index b224d79e47cd9..a3a94a642f250 100644 --- a/stdlib/Test/src/logging.jl +++ b/stdlib/Test/src/logging.jl @@ -107,8 +107,8 @@ function Logging.handle_message(logger::TestLogger, level, msg, _module, if maxlog isa Core.BuiltinInts @lock logger.lock begin remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return logger.message_limits[id] = remaining - 1 - remaining > 0 || return end end end From 75d5588d46bf7445626837f05e7a284ad85c7d30 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 4 Apr 2025 14:41:27 -0700 Subject: [PATCH 049/662] Add missing PARTITION_KIND_CONST_IMPORT case to print_partition (#58006) --- base/show.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/show.jl b/base/show.jl index fa666bea22985..34a08393c9853 100644 --- a/base/show.jl +++ b/base/show.jl @@ -3414,6 +3414,9 @@ function print_partition(io::IO, partition::Core.BindingPartition) elseif kind == PARTITION_KIND_CONST print(io, "constant binding to ") print(io, partition_restriction(partition)) + elseif kind == PARTITION_KIND_CONST_IMPORT + print(io, "constant binding (declared with `import`) to ") + print(io, partition_restriction(partition)) elseif kind == PARTITION_KIND_UNDEF_CONST print(io, "undefined const binding") elseif kind == PARTITION_KIND_GUARD From 0a4ca16e1bb4bc65c631e13b2352ecc816770d04 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Sun, 6 Apr 2025 12:36:26 +0800 Subject: [PATCH 050/662] typeintersect: fix triangular vars handling outside constructor. Fix #57852. --- src/subtype.c | 2 +- test/subtype.jl | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index 1086d11d2fe8c..5d5034e08252b 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2801,7 +2801,7 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int jl_value_t *ub = R ? intersect_aside(a, bb->ub, e, bb->depth0) : intersect_aside(bb->ub, a, e, bb->depth0); if (ub == jl_bottom_type) return jl_bottom_type; - if (bb->constraintkind == 1 || e->triangular) { + if (bb->constraintkind == 1 || (e->triangular && param == 1)) { if (e->triangular && check_unsat_bound(ub, b, e)) return jl_bottom_type; set_bound(&bb->ub, ub, b, e); diff --git a/test/subtype.jl b/test/subtype.jl index 979746bd626dc..075b5149442e0 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2087,8 +2087,7 @@ let A = Tuple{Any, Type{Ref{_A}} where _A}, I = typeintersect(A, B) @test I != Union{} @test Tuple{Type{Ref{Integer}}, Type{Ref{Integer}}} <: I - # TODO: this intersection result seems too wide (I == B) ? - @test_broken !<:(Tuple{Type{Int}, Type{Int}}, I) + @test !<:(Tuple{Type{Int}, Type{Int}}, I) end @testintersect(Tuple{Type{T}, T} where T<:(Tuple{Vararg{_A, _B}} where _B where _A), @@ -2653,10 +2652,10 @@ let S = Type{T53371{A, B, C, D, E}} where {A, B<:R53371{A}, C<:R53371{A}, D<:R53 end #issue 54356 -let S = Tuple{Val{Val{Union{Val{A2}, A2}}}, Val{Val{Union{Val{A2}, Val{A4}, A4}}}} where {A2, A4<:Union{Val{A2}, A2}}, - T = Tuple{Vararg{Val{V}}} where {V} - @testintersect(S, T, !Union{}) -end +# let S = Tuple{Val{Val{Union{Val{A2}, A2}}}, Val{Val{Union{Val{A2}, Val{A4}, A4}}}} where {A2, A4<:Union{Val{A2}, A2}}, +# T = Tuple{Vararg{Val{V}}} where {V} +# @testintersect(S, T, !Union{}) +# end #issue 54356 abstract type A54356{T<:Real} end @@ -2757,3 +2756,15 @@ end Pair{N, T} where {N,NTuple{N,Int}<:T<:Tuple{Int,Vararg{Int}}}, !Union{} ) + +#issue 57852 +@testintersect( + Tuple{Type{T}, Type{<:F}, Type{<:F}} where {T, F<:Union{String, T}}, + Tuple{Type{Complex{T}} where T, Type{Complex{T}} where T, Type{String}}, + Tuple{Type{Complex{T}}, Type{Complex{T}}, Type{String}} where T +) +@testintersect( + Tuple{Type{T}, Type{<:Union{F, Nothing}}, Type{<:Union{F, Nothing}}} where {T, F<:Union{String, T}}, + Tuple{Type{Complex{T}} where T, Type{Complex{T}} where T, Type{String}}, + Tuple{Type{Complex{T}}, Type{Complex{T}}, Type{String}} where T +) From 2b8a05cf2853141bd438ed8ff0e639e0752d4226 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Sun, 6 Apr 2025 14:38:48 +0800 Subject: [PATCH 051/662] typeintersect: add more fast path. skip intersection/subtyping under circular env. --- src/subtype.c | 13 ++++++------- test/subtype.jl | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index 5d5034e08252b..c1c811d4e0ad5 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1393,7 +1393,7 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param) x = pick_union_element(x, e, 0); } if (jl_is_uniontype(y)) { - if (x == ((jl_uniontype_t*)y)->a || x == ((jl_uniontype_t*)y)->b) + if (obviously_in_union(y, x)) return 1; if (jl_is_unionall(x)) return subtype_unionall(y, (jl_unionall_t*)x, e, 0, param); @@ -2539,9 +2539,6 @@ static jl_value_t *intersect_aside(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, static jl_value_t *intersect_union(jl_value_t *x, jl_uniontype_t *u, jl_stenv_t *e, int8_t R, int param) { - // band-aid for #56040 - if (!jl_is_uniontype(x) && obviously_in_union((jl_value_t *)u, x)) - return x; int no_free = !jl_has_free_typevars(x) && !jl_has_free_typevars((jl_value_t*)u); if (param == 2 || no_free) { jl_value_t *a=NULL, *b=NULL; @@ -2678,7 +2675,7 @@ static void set_bound(jl_value_t **bound, jl_value_t *val, jl_tvar_t *v, jl_sten // subtype, treating all vars as existential static int subtype_in_env_existential(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) { - if (x == jl_bottom_type || y == (jl_value_t*)jl_any_type) + if (x == jl_bottom_type || y == (jl_value_t*)jl_any_type || obviously_in_union(y, x)) return 1; int8_t *rs = (int8_t*)alloca(current_env_length(e)); jl_varbinding_t *v = e->vars; @@ -4116,12 +4113,14 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa if (jl_subtype(y, x)) return y; } if (jl_is_uniontype(x)) { - if (y == ((jl_uniontype_t*)x)->a || y == ((jl_uniontype_t*)x)->b) + if (obviously_in_union(x, y)) return y; + if (jl_is_uniontype(y) && obviously_in_union(y, x)) + return x; return intersect_union(y, (jl_uniontype_t*)x, e, 0, param); } if (jl_is_uniontype(y)) { - if (x == ((jl_uniontype_t*)y)->a || x == ((jl_uniontype_t*)y)->b) + if (obviously_in_union(y, x)) return x; if (jl_is_unionall(x) && (jl_has_free_typevars(x) || jl_has_free_typevars(y))) return intersect_unionall(y, (jl_unionall_t*)x, e, 0, param); diff --git a/test/subtype.jl b/test/subtype.jl index 075b5149442e0..d186262a6e1ba 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2652,10 +2652,10 @@ let S = Type{T53371{A, B, C, D, E}} where {A, B<:R53371{A}, C<:R53371{A}, D<:R53 end #issue 54356 -# let S = Tuple{Val{Val{Union{Val{A2}, A2}}}, Val{Val{Union{Val{A2}, Val{A4}, A4}}}} where {A2, A4<:Union{Val{A2}, A2}}, -# T = Tuple{Vararg{Val{V}}} where {V} -# @testintersect(S, T, !Union{}) -# end +let S = Tuple{Val{Val{Union{Val{A2}, A2}}}, Val{Val{Union{Val{A2}, Val{A4}, A4}}}} where {A2, A4<:Union{Val{A2}, A2}}, + T = Tuple{Vararg{Val{V}}} where {V} + @testintersect(S, T, !Union{}) +end #issue 54356 abstract type A54356{T<:Real} end From 55bf45d38d8bd0aa70cadc4022de625729acfc34 Mon Sep 17 00:00:00 2001 From: angerpointnerd <102872423+angerpointnerd@users.noreply.github.com> Date: Sun, 6 Apr 2025 20:16:33 +0200 Subject: [PATCH 052/662] Documentation: Add notes about subtypes of concrete types (#58017) Based on this [discussion on Discourse](https://discourse.julialang.org/t/how-to-interpret-type-t-where-t-in-julias-type-system/37402/21?u=sevi) This is my shot at adding three short sentences that hopefully clarify the restrictions of subtyping concrete types. There seems to be some recurring confusion by Julia users (including myself) about how to imagine Julia's type hierarchy. I think the language in the manual page is sufficiently clear already (i.e. there is no mention of a "type tree" and it is also mentioned that only _concrete subtypes of concrete types_ are strictly not possible), but adding the proposed sentences could help readers to keep in mind that e.g. `Type{...}` and `Union{}` are counterexamples to a strict "type tree" image. I tried to be conservative with the changes though, because I'm not sure how far this should be discussed here and whether my current mental model of Julia's type graph is actually correct. --- Co-authored-by: S.Angerpointner Co-authored-by: Lilith Orion Hafner --- doc/src/manual/types.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/src/manual/types.md b/doc/src/manual/types.md index f13e2e6865d0f..926b33e17460a 100644 --- a/doc/src/manual/types.md +++ b/doc/src/manual/types.md @@ -31,8 +31,11 @@ each other: all concrete types are final and may only have abstract types as the While this might at first seem unduly restrictive, it has many beneficial consequences with surprisingly few drawbacks. It turns out that being able to inherit behavior is much more important than being able to inherit structure, and inheriting both causes significant difficulties in traditional -object-oriented languages. Other high-level aspects of Julia's type system that should be mentioned -up front are: +object-oriented languages. While concrete types do have abstract subtypes, there are only two examples of this +([`Union{}`](@ref man-abstract-types) and [`Type{T}`](@ref man-typet-type))) and additional subtypes +of concrete types cannot be declared. + +Other high-level aspects of Julia's type system that should be mentioned up front are: * There is no division between object and non-object values: all values in Julia are true objects having a type that belongs to a single, fully connected type graph, all nodes of which are equally @@ -182,7 +185,7 @@ When no supertype is given, the default supertype is `Any` -- a predefined abstr all objects are instances of and all types are subtypes of. In type theory, `Any` is commonly called "top" because it is at the apex of the type graph. Julia also has a predefined abstract "bottom" type, at the nadir of the type graph, which is written as `Union{}`. It is the exact -opposite of `Any`: no object is an instance of `Union{}` and all types are supertypes of `Union{}`. +opposite of `Any`: no object is an instance of `Union{}` and all types (including concrete types) are supertypes of `Union{}`. Let's consider some of the abstract types that make up Julia's numerical hierarchy: @@ -1306,6 +1309,9 @@ julia> WrapType(Float64) # sharpened constructor, note more precise Type{Float64 WrapType{Type{Float64}}(Float64) ``` +This behavior of `Type{Float64}` is an example of an abstract type subtyping a +concrete type (here `DataType`). + ## Type Aliases Sometimes it is convenient to introduce a new name for an already expressible type. From b627f5bb2592cb7120487ca40c149eb933b32dba Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 6 Apr 2025 16:36:30 -0400 Subject: [PATCH 053/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=206f309f17f=20to=205432fdd30=20(#58022)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 | 1 + .../Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 | 1 + .../Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 | 1 - .../Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 create mode 100644 deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 diff --git a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 new file mode 100644 index 0000000000000..bcaf046283425 --- /dev/null +++ b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 @@ -0,0 +1 @@ +e097e71eea7524a17bda8ba14569a983 diff --git a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 new file mode 100644 index 0000000000000..1c659df733de4 --- /dev/null +++ b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 @@ -0,0 +1 @@ +56f6baeda00597ab8a9d9827e6a07aa41662e7914c6ba6fe8c6c21c2008a6552db9778ad95dbaad8d0309e6caa25995916adafcd0243b0f26e1e5217ae608a01 diff --git a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 deleted file mode 100644 index f1a567935cfc6..0000000000000 --- a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -cdbbe19efa29dcaa89fe167e15de449d diff --git a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 b/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 deleted file mode 100644 index fc895ae726858..0000000000000 --- a/deps/checksums/Pkg-6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7ddce211c686a4c91ac7806dd134e4d0a3abcfe8d0a5c6568697a26e4dd0164c60fb2611421e5e7821819145cdbda85ecd1eec0724409346affcb457a41f5aee diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index d356689c391f6..8d13ca7f3e8dc 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17 +PKG_SHA1 = 5432fdd30f06707ee227237224b0b72168c2c2f5 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 4a3eba3b9125bfe825bf71f71404097f4f280176 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 7 Apr 2025 14:06:54 +0200 Subject: [PATCH 054/662] Avoid most boxes in precompilation code (#57986) - Use `local` where variable names accidentally overlapped and caused boxes - Use `@lock` to avoid closures - Move out some recursive closures to top-level normal function to avoid boxing them - Use explicit boxes (`Ref`) instead of relying on the typeassert on the `Core.Box` because it it hard to determine if a `Core.Box` is benign or not. - Only assign to `io` once to avoid boxing it. - Type assert some `Tuple{Int, Int}` on `IO`. The two remaining boxes after this is `t_print` and `monitor_std`. --- base/precompilation.jl | 212 +++++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 104 deletions(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 3ab9fcad5aee6..c24026aa2a8ef 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -301,7 +301,7 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere else string(p.current, "/", p.max) end - termwidth = @something termwidth displaysize(io)[2] + termwidth = @something termwidth (displaysize(io)::Tuple{Int,Int})[2] max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) n_filled = floor(Int, max_progress_width * perc / 100) partial_filled = (max_progress_width * perc / 100) - n_filled @@ -409,6 +409,53 @@ function excluded_circular_deps_explanation(io::IOContext{IO}, ext_to_parent::Di return msg end + +function scan_pkg!(stack, could_be_cycle, cycles, pkg, dmap) + if haskey(could_be_cycle, pkg) + return could_be_cycle[pkg] + else + return scan_deps!(stack, could_be_cycle, cycles, pkg, dmap) + end +end +function scan_deps!(stack, could_be_cycle, cycles, pkg, dmap) + push!(stack, pkg) + cycle = nothing + for dep in dmap[pkg] + if dep in stack + # Created fresh cycle + cycle′ = stack[findlast(==(dep), stack):end] + if cycle === nothing || length(cycle′) < length(cycle) + cycle = cycle′ # try to report smallest cycle possible + end + elseif scan_pkg!(stack, could_be_cycle, cycles, dep, dmap) + # Reaches an existing cycle + could_be_cycle[pkg] = true + pop!(stack) + return true + end + end + pop!(stack) + if cycle !== nothing + push!(cycles, cycle) + could_be_cycle[pkg] = true + return true + end + could_be_cycle[pkg] = false + return false +end + +# restrict to dependencies of given packages +function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) + for _dep in direct_deps[dep] + if !(_dep in alldeps) + push!(alldeps, _dep) + collect_all_deps(direct_deps, _dep, alldeps) + end + end + return alldeps +end + + function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, strict::Bool = false, @@ -434,7 +481,7 @@ function _precompilepkgs(pkgs::Vector{String}, timing::Bool, _from_loading::Bool, configs::Vector{Config}, - io::IOContext{IO}, + _io::IOContext{IO}, fancyprint::Bool, manifest::Bool, ignore_loaded::Bool) @@ -452,10 +499,9 @@ function _precompilepkgs(pkgs::Vector{String}, num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) parallel_limiter = Base.Semaphore(num_tasks) - if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false) - # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests - io = IOContext{IO}(devnull) - end + # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests + io = (_from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false)) ? IOContext{IO}(devnull) : _io + nconfigs = length(configs) hascolor = get(io, :color, false)::Bool @@ -552,7 +598,7 @@ function _precompilepkgs(pkgs::Vector{String}, end end - indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() + local indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() for package in keys(direct_deps) # Initialize a set to keep track of all dependencies for 'package' all_deps = Set{Base.PkgId}() @@ -619,46 +665,14 @@ function _precompilepkgs(pkgs::Vector{String}, could_be_cycle = Dict{Base.PkgId, Bool}() # temporary stack for the SCC-like algorithm below stack = Base.PkgId[] - function scan_pkg!(pkg, dmap) - if haskey(could_be_cycle, pkg) - return could_be_cycle[pkg] - else - return scan_deps!(pkg, dmap) - end - end - function scan_deps!(pkg, dmap) - push!(stack, pkg) - cycle = nothing - for dep in dmap[pkg] - if dep in stack - # Created fresh cycle - cycle′ = stack[findlast(==(dep), stack):end] - if cycle === nothing || length(cycle′) < length(cycle) - cycle = cycle′ # try to report smallest cycle possible - end - elseif scan_pkg!(dep, dmap) - # Reaches an existing cycle - could_be_cycle[pkg] = true - pop!(stack) - return true - end - end - pop!(stack) - if cycle !== nothing - push!(cycles, cycle) - could_be_cycle[pkg] = true - return true - end - could_be_cycle[pkg] = false - return false - end + # set of packages that depend on a cycle (either because they are # a part of a cycle themselves or because they transitively depend # on a package in some cycle) circular_deps = Base.PkgId[] for pkg in keys(direct_deps) @assert isempty(stack) - if scan_pkg!(pkg, direct_deps) + if scan_pkg!(stack, could_be_cycle, cycles, pkg, direct_deps) push!(circular_deps, pkg) for pkg_config in keys(was_processed) # notify all to allow skipping @@ -675,16 +689,6 @@ function _precompilepkgs(pkgs::Vector{String}, if isempty(pkgs) pkgs = [pkg.name for pkg in project_deps] end - # restrict to dependencies of given packages - function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) - for _dep in direct_deps[dep] - if !(_dep in alldeps) - push!(alldeps, _dep) - collect_all_deps(direct_deps, _dep, alldeps) - end - end - return alldeps - end keep = Set{Base.PkgId}() for dep in direct_deps dep_pkgid = first(dep) @@ -711,13 +715,13 @@ function _precompilepkgs(pkgs::Vector{String}, end end - target = nothing + target = Ref{Union{Nothing, String}}(nothing) if nconfigs == 1 if !isempty(only(configs)[1]) - target = "for configuration $(join(only(configs)[1], " "))" + target[] = "for configuration $(join(only(configs)[1], " "))" end else - target = "for $nconfigs compilation configurations..." + target[] = "for $nconfigs compilation configurations..." end @debug "precompile: packages filtered" @@ -727,7 +731,7 @@ function _precompilepkgs(pkgs::Vector{String}, print_lock = io.io isa Base.LibuvStream ? io.io.lock::ReentrantLock : ReentrantLock() first_started = Base.Event() - printloop_should_exit::Bool = !fancyprint # exit print loop immediately if not fancy printing + printloop_should_exit = Ref{Bool}(!fancyprint) # exit print loop immediately if not fancy printing interrupted_or_done = Base.Event() ansi_moveup(n::Int) = string("\e[", n, "A") @@ -736,19 +740,19 @@ function _precompilepkgs(pkgs::Vector{String}, ansi_cleartoendofline = "\e[0K" ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" - n_done::Int = 0 - n_already_precomp::Int = 0 - n_loaded::Int = 0 - interrupted = false + n_done = Ref(0) + n_already_precomp = Ref(0) + n_loaded = Ref(0) + interrupted = Ref(false) - function handle_interrupt(err, in_printloop = false) + function handle_interrupt(err, in_printloop::Bool) notify(interrupted_or_done) in_printloop || wait(t_print) # wait to let the print loop cease first if err isa InterruptException - lock(print_lock) do + @lock print_lock begin println(io, " Interrupted: Exiting precompilation...", ansi_cleartoendofline) end - interrupted = true + interrupted[] = true return true else return false @@ -757,34 +761,34 @@ function _precompilepkgs(pkgs::Vector{String}, std_outputs = Dict{PkgConfig,IOBuffer}() taskwaiting = Set{PkgConfig}() pkgspidlocked = Dict{PkgConfig,String}() - pkg_liveprinted = nothing + pkg_liveprinted = Ref{Union{Nothing, PkgId}}(nothing) function monitor_std(pkg_config, pipe; single_requested_pkg=false) pkg, config = pkg_config try liveprinting = false while !eof(pipe) - str = readline(pipe, keep=true) + local str = readline(pipe, keep=true) if single_requested_pkg && (liveprinting || !isempty(str)) - lock(print_lock) do + @lock print_lock begin if !liveprinting printpkgstyle(io, :Info, "Given $(pkg.name) was explicitly requested, output will be shown live $ansi_cleartoendofline", color = Base.info_color()) liveprinting = true - pkg_liveprinted = pkg + pkg_liveprinted[] = pkg end print(io, ansi_cleartoendofline, str) end end write(get!(IOBuffer, std_outputs, pkg_config), str) if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str) - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) end push!(taskwaiting, pkg_config) end if !fancyprint && in(pkg_config, taskwaiting) - lock(print_lock) do + @lock print_lock begin print(io, str) end end @@ -799,9 +803,9 @@ function _precompilepkgs(pkgs::Vector{String}, try wait(first_started) (isempty(pkg_queue) || interrupted_or_done.set) && return - lock(print_lock) do - if target !== nothing - printpkgstyle(io, :Precompiling, target) + @lock print_lock begin + if target[] !== nothing + printpkgstyle(io, :Precompiling, target[]) end if fancyprint print(io, ansi_disablecursor) @@ -813,12 +817,12 @@ function _precompilepkgs(pkgs::Vector{String}, last_length = 0 bar = MiniProgressBar(; indent=0, header = "Precompiling packages ", color = :green, percentage=false, always_reprint=true) n_total = length(direct_deps) * length(configs) - bar.max = n_total - n_already_precomp + bar.max = n_total - n_already_precomp[] final_loop = false n_print_rows = 0 - while !printloop_should_exit - lock(print_lock) do - term_size = displaysize(io) + while !printloop_should_exit[] + @lock print_lock begin + term_size = displaysize(io)::Tuple{Int, Int} num_deps_show = max(term_size[1] - 3, 2) # show at least 2 deps pkg_queue_show = if !interrupted_or_done.set && length(pkg_queue) > num_deps_show last(pkg_queue, num_deps_show) @@ -829,11 +833,11 @@ function _precompilepkgs(pkgs::Vector{String}, if i > 1 print(iostr, ansi_cleartoend) end - bar.current = n_done - n_already_precomp - bar.max = n_total - n_already_precomp + bar.current = n_done[] - n_already_precomp[] + bar.max = n_total - n_already_precomp[] # when sizing to the terminal width subtract a little to give some tolerance to resizing the # window between print cycles - termwidth = displaysize(io)[2] - 4 + termwidth = (displaysize(io)::Tuple{Int,Int})[2] - 4 if !final_loop s = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) print(iostr, Base._truncate_at_width_or_chars(true, s, termwidth), "\n") @@ -877,10 +881,10 @@ function _precompilepkgs(pkgs::Vector{String}, last_length = length(pkg_queue_show) n_print_rows = count("\n", str_) print(io, str_) - printloop_should_exit = interrupted_or_done.set && final_loop + printloop_should_exit[] = interrupted_or_done.set && final_loop final_loop = interrupted_or_done.set # ensures one more loop to tidy last task after finish i += 1 - printloop_should_exit || print(io, ansi_moveup(n_print_rows), ansi_movecol1) + printloop_should_exit[] || print(io, ansi_moveup(n_print_rows), ansi_movecol1) end wait(t) end @@ -931,9 +935,9 @@ function _precompilepkgs(pkgs::Vector{String}, t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) name = describe_pkg(pkg, is_project_dep, flags, cacheflags) - lock(print_lock) do + @lock print_lock begin if !fancyprint && isempty(pkg_queue) - printpkgstyle(io, :Precompiling, something(target, "packages...")) + printpkgstyle(io, :Precompiling, something(target[], "packages...")) end end push!(pkg_queue, pkg_config) @@ -960,16 +964,16 @@ function _precompilepkgs(pkgs::Vector{String}, end if ret isa Base.PrecompilableError push!(precomperr_deps, pkg_config) - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) end else - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) end was_recompiled[pkg_config] = true end - loaded && (n_loaded += 1) + loaded && (n_loaded[] += 1) catch err # @show err close(std_pipe.in) # close pipe to end the std output monitor @@ -978,7 +982,7 @@ function _precompilepkgs(pkgs::Vector{String}, errmsg = String(take!(get(IOBuffer, std_outputs, pkg_config))) delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report failed_deps[pkg_config] = (strict || is_project_dep) ? string(sprint(showerror, err), "\n", strip(errmsg)) : "" - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) end else @@ -990,15 +994,15 @@ function _precompilepkgs(pkgs::Vector{String}, Base.release(parallel_limiter) end else - is_stale || (n_already_precomp += 1) + is_stale || (n_already_precomp[] += 1) end - n_done += 1 + n_done[] += 1 notify(was_processed[pkg_config]) catch err_outer # For debugging: # println("Task failed $err_outer") # Base.display_error(ErrorException(""), Base.catch_backtrace())# logging doesn't show here - handle_interrupt(err_outer) || rethrow() + handle_interrupt(err_outer, false) || rethrow() notify(was_processed[pkg_config]) finally filter!(!istaskdone, tasks) @@ -1013,13 +1017,13 @@ function _precompilepkgs(pkgs::Vector{String}, try wait(interrupted_or_done) catch err - handle_interrupt(err) || rethrow() + handle_interrupt(err, false) || rethrow() finally Base.LOADING_CACHE[] = nothing end notify(first_started) # in cases of no-op or !fancyprint fancyprint && wait(t_print) - quick_exit = !all(istaskdone, tasks) || interrupted # if some not finished internal error is likely + quick_exit = !all(istaskdone, tasks) || interrupted[] # if some not finished internal error is likely seconds_elapsed = round(Int, (time_ns() - time_start) / 1e9) ndeps = count(values(was_recompiled)) if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) @@ -1031,18 +1035,18 @@ function _precompilepkgs(pkgs::Vector{String}, end plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" print(iostr, " $(ndeps) $(plural) successfully precompiled in $(seconds_elapsed) seconds") - if n_already_precomp > 0 || !isempty(circular_deps) - n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) + if n_already_precomp[] > 0 || !isempty(circular_deps) + n_already_precomp[] > 0 && (print(iostr, ". $(n_already_precomp[]) already precompiled")) !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) print(iostr, ".") end - if n_loaded > 0 - plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded == 1 ? "dependency" : "dependencies" - plural2 = n_loaded == 1 ? "a different version is" : "different versions are" - plural3 = n_loaded == 1 ? "" : "s" - plural4 = n_loaded == 1 ? "this package" : "these packages" + if n_loaded[] > 0 + local plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded[] == 1 ? "dependency" : "dependencies" + local plural2 = n_loaded[] == 1 ? "a different version is" : "different versions are" + local plural3 = n_loaded[] == 1 ? "" : "s" + local plural4 = n_loaded[] == 1 ? "this package" : "these packages" print(iostr, "\n ", - color_string(string(n_loaded), Base.warn_color()), + color_string(string(n_loaded[]), Base.warn_color()), " $(plural1) precompiled but ", color_string("$(plural2) currently loaded", Base.warn_color()), ". Restart julia to access the new version$(plural3). \ @@ -1062,12 +1066,12 @@ function _precompilepkgs(pkgs::Vector{String}, let std_outputs = Tuple{PkgConfig,SubString{String}}[(pkg_config, strip(String(take!(io)))) for (pkg_config,io) in std_outputs] filter!(kv -> !isempty(last(kv)), std_outputs) if !isempty(std_outputs) - plural1 = length(std_outputs) == 1 ? "y" : "ies" - plural2 = length(std_outputs) == 1 ? "" : "s" + local plural1 = length(std_outputs) == 1 ? "y" : "ies" + local plural2 = length(std_outputs) == 1 ? "" : "s" print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") for (pkg_config, err) in std_outputs pkg, config = pkg_config - err = if pkg == pkg_liveprinted + err = if pkg == pkg_liveprinted[] "[Output was shown above]" else join(split(err, "\n"), color_string("\n│ ", Base.warn_color())) @@ -1079,7 +1083,7 @@ function _precompilepkgs(pkgs::Vector{String}, end end let str=str - lock(print_lock) do + @lock print_lock begin println(io, str) end end @@ -1157,7 +1161,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo else "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" end - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg_config])", Base.info_color(), hascolor)) end Base.release(parallel_limiter) # release so other work can be done while waiting From 9d00b575771f2aab357a4537fa8125081bf0e570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 7 Apr 2025 16:26:47 +0200 Subject: [PATCH 055/662] Run Compiler tests in parallel for `Pkg.test`, continued (#57987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: KristofferC Co-authored-by: Cédric Belmant --- Compiler/.gitignore | 1 + Compiler/test/AbstractInterpreter.jl | 1 + Compiler/test/EscapeAnalysis.jl | 1 + Compiler/test/compact.jl | 1 + Compiler/test/contextual.jl | 1 + Compiler/test/effects.jl | 2 ++ Compiler/test/inference.jl | 1 + Compiler/test/inline.jl | 1 + Compiler/test/invalidation.jl | 1 + Compiler/test/irpasses.jl | 1 + Compiler/test/newinterp.jl | 2 ++ Compiler/test/runtests.jl | 9 ++++++--- Compiler/test/setup_Compiler.jl | 7 +++++++ Compiler/test/special_loading.jl | 11 +++++++---- Compiler/test/ssair.jl | 1 + Compiler/test/tarjan.jl | 1 + 16 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 Compiler/.gitignore diff --git a/Compiler/.gitignore b/Compiler/.gitignore new file mode 100644 index 0000000000000..ba39cc531edeb --- /dev/null +++ b/Compiler/.gitignore @@ -0,0 +1 @@ +Manifest.toml diff --git a/Compiler/test/AbstractInterpreter.jl b/Compiler/test/AbstractInterpreter.jl index 5cb7c6fddf15f..6ca5154c571e7 100644 --- a/Compiler/test/AbstractInterpreter.jl +++ b/Compiler/test/AbstractInterpreter.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") include("newinterp.jl") diff --git a/Compiler/test/EscapeAnalysis.jl b/Compiler/test/EscapeAnalysis.jl index 60364769c95a8..07855d3362881 100644 --- a/Compiler/test/EscapeAnalysis.jl +++ b/Compiler/test/EscapeAnalysis.jl @@ -1,5 +1,6 @@ module test_EA +include("setup_Compiler.jl") include("irutils.jl") const EscapeAnalysis = Compiler.EscapeAnalysis diff --git a/Compiler/test/compact.jl b/Compiler/test/compact.jl index b01e209d5ce9b..5b19dc68811fc 100644 --- a/Compiler/test/compact.jl +++ b/Compiler/test/compact.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") using .Compiler: IncrementalCompact, insert_node_here!, finish, diff --git a/Compiler/test/contextual.jl b/Compiler/test/contextual.jl index a9c63ab34c0c0..4550501f5546e 100644 --- a/Compiler/test/contextual.jl +++ b/Compiler/test/contextual.jl @@ -2,6 +2,7 @@ # N.B.: This file is also run from interpreter.jl, so needs to be standalone-executable using Test + include("setup_Compiler.jl") # Cassette diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index b3d8d1a56ea16..18cfc0e8d5388 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -1,6 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test + +include("setup_Compiler.jl") include("irutils.jl") # Test that the Core._apply_iterate bail path taints effects diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index c94059f59f6e7..ded2ea7ba0030 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -4,6 +4,7 @@ module inference using Test +include("setup_Compiler.jl") include("irutils.jl") # tests for Compiler correctness and precision diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index c5a7ab197e16e..fd3f950d2925a 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -4,6 +4,7 @@ using Test using Base.Meta using Core: ReturnNode +include("setup_Compiler.jl") include("irutils.jl") include("newinterp.jl") diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index 32f462f06f4e9..29058ffa5cfc6 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -3,6 +3,7 @@ # setup # ----- +include("setup_Compiler.jl") include("irutils.jl") using Test diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 7912f1c9c7f9e..c42db500ed3f7 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -4,6 +4,7 @@ using Test using Base.Meta using Core.IR +include("setup_Compiler.jl") include("irutils.jl") # domsort diff --git a/Compiler/test/newinterp.jl b/Compiler/test/newinterp.jl index 5ebcf332895fa..1e0c50192fb5b 100644 --- a/Compiler/test/newinterp.jl +++ b/Compiler/test/newinterp.jl @@ -2,6 +2,8 @@ # TODO set up a version who defines new interpreter with persistent cache? +include("setup_Compiler.jl") + """ @newinterp NewInterpreter [ephemeral_cache::Bool=false] diff --git a/Compiler/test/runtests.jl b/Compiler/test/runtests.jl index 6a38fce678ba0..8a35d8e71102f 100644 --- a/Compiler/test/runtests.jl +++ b/Compiler/test/runtests.jl @@ -1,8 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Compiler -using InteractiveUtils: @activate -@activate Compiler +Base.runtests(["Compiler"]; propagate_project=true) + +#= +# To run serially +using Test, Compiler @testset "Compiler.jl" begin for file in readlines(joinpath(@__DIR__, "testgroups")) file == "special_loading" && continue # Only applicable to Base.Compiler @@ -10,3 +12,4 @@ using InteractiveUtils: @activate @eval @testset $testfile include($testfile) end end +=# diff --git a/Compiler/test/setup_Compiler.jl b/Compiler/test/setup_Compiler.jl index a28a3f918aaf9..83cc42e2d0936 100644 --- a/Compiler/test/setup_Compiler.jl +++ b/Compiler/test/setup_Compiler.jl @@ -1,5 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using InteractiveUtils: @activate + +if Base.identify_package("Compiler") !== nothing && !isdefined(Main, :__custom_compiler_active) + Base.eval(Main, :(__custom_compiler_active=true)) + @activate Compiler +end + if !@isdefined(Compiler) if Base.REFLECTION_COMPILER[] === nothing using Base.Compiler: Compiler diff --git a/Compiler/test/special_loading.jl b/Compiler/test/special_loading.jl index ba012446dc61f..ba8cbc635eae8 100644 --- a/Compiler/test/special_loading.jl +++ b/Compiler/test/special_loading.jl @@ -1,9 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mktempdir() do dir - withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do - cd(joinpath(@__DIR__, "CompilerLoadingTest")) do - @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) +# Only run when testing Base compiler +if Base.identify_package("Compiler") === nothing + mktempdir() do dir + withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do + cd(joinpath(@__DIR__, "CompilerLoadingTest")) do + @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) + end end end end diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 3026202466599..d710c81069850 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -1,5 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +include("setup_Compiler.jl") include("irutils.jl") using Test diff --git a/Compiler/test/tarjan.jl b/Compiler/test/tarjan.jl index aa04bd94a6f6a..8fe940463b558 100644 --- a/Compiler/test/tarjan.jl +++ b/Compiler/test/tarjan.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") using .Compiler: CFGReachability, DomTree, CFG, BasicBlock, StmtRange, dominates, From e542818aa9f6deca44b07d81e94a33b953f65b70 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Mon, 7 Apr 2025 18:26:08 -0400 Subject: [PATCH 056/662] optimizer: fix various `optimize_until` misuses (#58035) I fixed a bunch of places where the `optimize_until` path was specified incorrectly. Also, I changed `"Inlining"` to `"inlining"` to match the lowercase style of the other path names. --- Compiler/src/optimize.jl | 2 +- Compiler/test/inline.jl | 4 ++-- Compiler/test/irpasses.jl | 2 +- Compiler/test/ssair.jl | 2 ++ stdlib/InteractiveUtils/test/runtests.jl | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 99283c30b2744..3fdd0ee4a7030 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -1044,7 +1044,7 @@ function run_passes_ipo_safe( @pass "slot2reg" ir = slot2reg(ir, ci, sv) # TODO: Domsorting can produce an updated domtree - no need to recompute here @pass "compact 1" ir = compact!(ir) - @pass "Inlining" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) + @pass "inlining" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) # @timeit "verify 2" verify_ir(ir) @pass "compact 2" ir = compact!(ir) @pass "SROA" ir = sroa_pass!(ir, sv.inlining) diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index fd3f950d2925a..3160d4ca74e6a 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -2218,7 +2218,7 @@ struct Issue52644 end issue52644(::DataType) = :DataType issue52644(::UnionAll) = :UnionAll -let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t +let ir = Base.code_ircode((Issue52644,); optimize_until="inlining") do t issue52644(t.tuple) end |> only |> first ir.argtypes[1] = Tuple{} @@ -2227,7 +2227,7 @@ let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t @test irfunc(Issue52644(Tuple{<:Integer})) === :UnionAll end issue52644_single(x::DataType) = :DataType -let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t +let ir = Base.code_ircode((Issue52644,); optimize_until="inlining") do t issue52644_single(t.tuple) end |> only |> first ir.argtypes[1] = Tuple{} diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index c42db500ed3f7..ed697ab738e11 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -921,7 +921,7 @@ let # Test that CFG simplify doesn't try to merge every block in a loop into end # `cfg_simplify!` shouldn't error in a presence of `try/catch` block -let ir = Base.code_ircode(; optimize_until="slot2ssa") do +let ir = Base.code_ircode(; optimize_until="slot2reg") do v = try catch end diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index d710c81069850..4d94e5b7dc4ca 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -601,6 +601,7 @@ import .Compiler: NewInstruction, insert_node! let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b a^b end |> only |> first + ir = Compiler.compact!(ir) nstmts = length(ir.stmts) invoke_idx = findfirst(@nospecialize(stmt)->Meta.isexpr(stmt, :invoke), ir.stmts.stmt) @test invoke !== nothing @@ -662,6 +663,7 @@ end let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b a^b end |> only |> first + ir = Compiler.compact!(ir) invoke_idx = findfirst(@nospecialize(stmt)->Meta.isexpr(stmt, :invoke), ir.stmts.stmt) @test invoke_idx !== nothing invoke_expr = ir.stmts.stmt[invoke_idx] diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 739ed5fac9ef2..084ac973b913d 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -826,7 +826,7 @@ end @test Base.infer_return_type(sin, (Int,)) == InteractiveUtils.@infer_return_type sin(42) @test Base.infer_exception_type(sin, (Int,)) == InteractiveUtils.@infer_exception_type sin(42) @test first(InteractiveUtils.@code_ircode sin(42)) isa Core.Compiler.IRCode -@test first(InteractiveUtils.@code_ircode optimize_until="Inlining" sin(42)) isa Core.Compiler.IRCode +@test first(InteractiveUtils.@code_ircode optimize_until="inlining" sin(42)) isa Core.Compiler.IRCode @testset "Docstrings" begin @test isempty(Docs.undocumented_names(InteractiveUtils)) From 019aa63fdeeabb0d42c435af2ade796938b3631a Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:07:25 +0200 Subject: [PATCH 057/662] `Base`: delete unnecessary `hastypemax` methods (#58003) The hardcoded methods are covered by the fallback method correctly. The reason these methods existed is probably that the fallback was not foldable at some point. Now it is foldable, so delete the unnecessary methods. Also add tests. --- base/gmp.jl | 4 +--- base/intfuncs.jl | 2 -- test/intfuncs.jl | 13 +++++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index 97488551f60f6..465b4b6a9772d 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -10,7 +10,7 @@ import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor, trailing_zeros, trailing_ones, count_ones, count_zeros, tryparse_internal, bin, oct, dec, hex, isequal, invmod, _prevpow2, _nextpow2, ndigits0zpb, widen, signed, unsafe_trunc, trunc, iszero, isone, big, flipsign, signbit, - sign, hastypemax, isodd, iseven, digits!, hash, hash_integer, top_set_bit, + sign, isodd, iseven, digits!, hash, hash_integer, top_set_bit, clamp, unsafe_takestring import Core: Signed, Float16, Float32, Float64 @@ -285,8 +285,6 @@ signed(x::BigInt) = x BigInt(x::BigInt) = x Signed(x::BigInt) = x -hastypemax(::Type{BigInt}) = false - function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, endpos::Int, base_::Integer, raise::Bool) # don't make a copy in the common case where we are parsing a whole String bstr = startpos == firstindex(s) && endpos == lastindex(s) ? String(s) : String(SubString(s,startpos,endpos)) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index fccd8fcb9accc..1c73698658aa9 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -1068,8 +1068,6 @@ end Return `true` if and only if the extrema `typemax(T)` and `typemin(T)` are defined. """ -hastypemax(::Base.BitIntegerType) = true -hastypemax(::Type{Bool}) = true hastypemax(::Type{T}) where {T} = applicable(typemax, T) && applicable(typemin, T) """ diff --git a/test/intfuncs.jl b/test/intfuncs.jl index adcfee2a050dd..a62092ad6b849 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -638,6 +638,19 @@ end @test Base.infer_effects(gcdx, (Int,Int)) |> Core.Compiler.is_foldable @test Base.infer_effects(invmod, (Int,Int)) |> Core.Compiler.is_foldable @test Base.infer_effects(binomial, (Int,Int)) |> Core.Compiler.is_foldable +@testset "concrete-foldability: `hastypemax`" begin + @test Base.infer_effects(Base.hastypemax, (Type,)) |> Core.Compiler.is_foldable + @test Base.infer_effects(Base.hastypemax, (DataType,)) |> Core.Compiler.is_foldable + for t in (Bool, Int, BigInt) + @test Base.infer_effects(Base.hastypemax, (Type{t},)) |> Core.Compiler.is_foldable + end +end + +@testset "`hastypemax`" begin + @test Base.hastypemax(Bool) + @test Base.hastypemax(Int) + @test !Base.hastypemax(BigInt) +end @testset "literal power" begin @testset for T in Base.uniontypes(Base.HWReal) From e6ebb38337c3fea60f65cc2cc35a5972d1abd184 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Tue, 8 Apr 2025 19:32:32 +0200 Subject: [PATCH 058/662] Random: get rid of `random_seed()` (#58042) For seeding `MersenneTwister()` randomly, we were generating a random seed via `RandomDevice()`, but if it failed, we used other means. This was implemented in `random_seed()`, which dates back from a long time ago, maybe when the ancestor of `RandomDevice()` had a reasonable chance to fail. But nowadays, it never fails in practice, and even the new default RNG uses it fearlessly. So RIP `random_seed()`. --- stdlib/Random/src/RNGs.jl | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 48d474f4ba41f..b99430adbdb46 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -278,20 +278,7 @@ end ### seeding -#### random_seed() & hash_seed() - -# random_seed tries to produce a random seed of type UInt128 from system entropy -function random_seed() - try - # as MersenneTwister prints its seed when `show`ed, 128 bits is a good compromise for - # almost surely always getting distinct seeds, while having them printed reasonably tersely - return rand(RandomDevice(), UInt128) - catch ex - ex isa IOError || rethrow() - @warn "Entropy pool not available to seed RNG; using ad-hoc entropy sources." - return Libc.rand() - end -end +#### hash_seed() function hash_seed(seed::Integer) ctx = SHA.SHA2_256_CTX() @@ -370,11 +357,13 @@ function initstate!(r::MersenneTwister, data::StridedVector, seed) return r end -# when a seed is not provided, we generate one via `RandomDevice()` in `random_seed()` rather +# When a seed is not provided, we generate one via `RandomDevice()` rather # than calling directly `initstate!` with `rand(RandomDevice(), UInt32, whatever)` because the # seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a -# small overhead compared to `initstate!`, so this simple solution is fine -seed!(r::MersenneTwister, ::Nothing) = seed!(r, random_seed()) +# small overhead compared to `initstate!`, so this simple solution is fine. +# A random seed with 128 bits is a good compromise for almost surely always getting distinct +# seeds, while having them printed reasonably tersely. +seed!(r::MersenneTwister, ::Nothing) = seed!(r, rand(RandomDevice(), UInt128)) seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed) From edbad8b55d6dda096c755a8cb816c6c82198d748 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:17:02 -0400 Subject: [PATCH 059/662] inference: re-enable inference overload for basic statements (#58027) After JuliaLang/julia#55575, inference for basic statements got inlined into `typeinf_local`. But external abstract interpreters like JET.jl need to overload this inference to customize this behavior. So this commit extracts the inlined inference logic back out into a separate `abstract_eval_basic_statement` method. --- Compiler/src/abstractinterpretation.jl | 197 +++++++++++++++---------- 1 file changed, 115 insertions(+), 82 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 34bf08731b239..97c68d48f34ef 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3706,6 +3706,107 @@ function abstract_eval_ssavalue(s::SSAValue, ssavaluetypes::Vector{Any}) return typ end +struct AbstractEvalBasicStatementResult + rt + exct + effects::Union{Nothing,Effects} + changes::Union{Nothing,StateUpdate} + refinements # ::Union{Nothing,SlotRefinement,Vector{Any}} + currsaw_latestworld::Bool + function AbstractEvalBasicStatementResult(rt, exct, effects::Union{Nothing,Effects}, + changes::Union{Nothing,StateUpdate}, refinements, currsaw_latestworld::Bool) + @nospecialize rt exct refinements + return new(rt, exct, effects, changes, refinements, currsaw_latestworld) + end +end + +function abstract_eval_basic_statement(interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState, + result::Union{Nothing,Future{RTEffects}}=nothing) + rt = nothing + exct = Bottom + changes = nothing + refinements = nothing + effects = nothing + currsaw_latestworld = sstate.saw_latestworld + if result !== nothing + @goto injectresult + end + if isa(stmt, NewvarNode) + changes = StateUpdate(stmt.slot, VarState(Bottom, true)) + elseif isa(stmt, PhiNode) + add_curr_ssaflag!(frame, IR_FLAGS_REMOVABLE) + # Implement convergence for PhiNodes. In particular, PhiNodes need to tmerge over + # the incoming values from all iterations, but `abstract_eval_phi` will only tmerge + # over the first and last iterations. By tmerging in the current old_rt, we ensure that + # we will not lose an intermediate value. + rt = abstract_eval_phi(interp, stmt, sstate, frame) + old_rt = frame.ssavaluetypes[frame.currpc] + rt = old_rt === NOT_FOUND ? rt : tmerge(typeinf_lattice(interp), old_rt, rt) + else + lhs = nothing + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + stmt = stmt.args[2] + end + if !isa(stmt, Expr) + (; rt, exct, effects, refinements) = abstract_eval_special_value(interp, stmt, sstate, frame) + else + hd = stmt.head + if hd === :method + fname = stmt.args[1] + if isa(fname, SlotNumber) + changes = StateUpdate(fname, VarState(Any, false)) + end + elseif (hd === :code_coverage_effect || + # :boundscheck can be narrowed to Bool + (hd !== :boundscheck && is_meta_expr(stmt))) + rt = Nothing + elseif hd === :latestworld + currsaw_latestworld = true + rt = Nothing + else + result = abstract_eval_statement_expr(interp, stmt, sstate, frame)::Future{RTEffects} + if !isready(result) || !isempty(frame.tasks) + return result + + @label injectresult + # reload local variables + lhs = nothing + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + stmt = stmt.args[2] + end + end + result = result[] + (; rt, exct, effects, refinements) = result + if effects.noub === NOUB_IF_NOINBOUNDS + if has_curr_ssaflag(frame, IR_FLAG_INBOUNDS) + effects = Effects(effects; noub=ALWAYS_FALSE) + elseif !propagate_inbounds(frame) + # The callee read our inbounds flag, but unless we propagate inbounds, + # we ourselves don't read our parent's inbounds. + effects = Effects(effects; noub=ALWAYS_TRUE) + end + end + @assert !isa(rt, TypeVar) "unhandled TypeVar" + rt = maybe_singleton_const(rt) + if !isempty(frame.pclimitations) + if rt isa Const || rt === Union{} + empty!(frame.pclimitations) + else + rt = LimitedAccuracy(rt, frame.pclimitations) + frame.pclimitations = IdSet{InferenceState}() + end + end + end + end + if lhs !== nothing && rt !== Bottom + changes = StateUpdate(lhs::SlotNumber, VarState(rt, false)) + end + end + return AbstractEvalBasicStatementResult(rt, exct, effects, changes, refinements, currsaw_latestworld) +end + struct BestguessInfo{Interp<:AbstractInterpreter} interp::Interp bestguess @@ -3986,14 +4087,16 @@ end # make as much progress on `frame` as possible (without handling cycles) struct CurrentState - result::Future + result::Future{RTEffects} currstate::VarTable currsaw_latestworld::Bool bbstart::Int bbend::Int - CurrentState(result::Future, currstate::VarTable, currsaw_latestworld::Bool, bbstart::Int, bbend::Int) = new(result, currstate, currsaw_latestworld, bbstart, bbend) + CurrentState(result::Future{RTEffects}, currstate::VarTable, currsaw_latestworld::Bool, bbstart::Int, bbend::Int) = + new(result, currstate, currsaw_latestworld, bbstart, bbend) CurrentState() = new() end + function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextresult::CurrentState) @assert !is_inferred(frame) W = frame.ip @@ -4012,7 +4115,9 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr bbend = nextresult.bbend currstate = nextresult.currstate currsaw_latestworld = nextresult.currsaw_latestworld - @goto injectresult + stmt = frame.src.code[currpc] + result = abstract_eval_basic_statement(interp, stmt, StatementState(currstate, currsaw_latestworld), frame, nextresult.result) + @goto injected_result end if currbb != 1 @@ -4165,87 +4270,15 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr end # Process non control-flow statements @assert isempty(frame.tasks) - rt = nothing - exct = Bottom - changes = nothing - refinements = nothing - effects = nothing - if isa(stmt, NewvarNode) - changes = StateUpdate(stmt.slot, VarState(Bottom, true)) - elseif isa(stmt, PhiNode) - add_curr_ssaflag!(frame, IR_FLAGS_REMOVABLE) - # Implement convergence for PhiNodes. In particular, PhiNodes need to tmerge over - # the incoming values from all iterations, but `abstract_eval_phi` will only tmerge - # over the first and last iterations. By tmerging in the current old_rt, we ensure that - # we will not lose an intermediate value. - rt = abstract_eval_phi(interp, stmt, StatementState(currstate, currsaw_latestworld), frame) - old_rt = frame.ssavaluetypes[currpc] - rt = old_rt === NOT_FOUND ? rt : tmerge(typeinf_lattice(interp), old_rt, rt) + sstate = StatementState(currstate, currsaw_latestworld) + result = abstract_eval_basic_statement(interp, stmt, sstate, frame) + if result isa Future{RTEffects} + return CurrentState(result, currstate, currsaw_latestworld, bbstart, bbend) else - lhs = nothing - if isexpr(stmt, :(=)) - lhs = stmt.args[1] - stmt = stmt.args[2] - end - if !isa(stmt, Expr) - (; rt, exct, effects, refinements) = abstract_eval_special_value(interp, stmt, StatementState(currstate, currsaw_latestworld), frame) - else - hd = stmt.head - if hd === :method - fname = stmt.args[1] - if isa(fname, SlotNumber) - changes = StateUpdate(fname, VarState(Any, false)) - end - elseif (hd === :code_coverage_effect || ( - hd !== :boundscheck && # :boundscheck can be narrowed to Bool - is_meta_expr(stmt))) - rt = Nothing - elseif hd === :latestworld - currsaw_latestworld = true - rt = Nothing - else - result = abstract_eval_statement_expr(interp, stmt, StatementState(currstate, currsaw_latestworld), frame)::Future - if !isready(result) || !isempty(frame.tasks) - return CurrentState(result, currstate, currsaw_latestworld, bbstart, bbend) - @label injectresult - # reload local variables - stmt = frame.src.code[currpc] - changes = nothing - lhs = nothing - if isexpr(stmt, :(=)) - lhs = stmt.args[1] - stmt = stmt.args[2] - end - result = nextresult.result::Future{RTEffects} - end - result = result[] - (; rt, exct, effects, refinements) = result - if effects.noub === NOUB_IF_NOINBOUNDS - if has_curr_ssaflag(frame, IR_FLAG_INBOUNDS) - effects = Effects(effects; noub=ALWAYS_FALSE) - elseif !propagate_inbounds(frame) - # The callee read our inbounds flag, but unless we propagate inbounds, - # we ourselves don't read our parent's inbounds. - effects = Effects(effects; noub=ALWAYS_TRUE) - end - end - @assert !isa(rt, TypeVar) "unhandled TypeVar" - rt = maybe_singleton_const(rt) - if !isempty(frame.pclimitations) - if rt isa Const || rt === Union{} - empty!(frame.pclimitations) - else - rt = LimitedAccuracy(rt, frame.pclimitations) - frame.pclimitations = IdSet{InferenceState}() - end - end - end - end - effects === nothing || merge_override_effects!(interp, effects, frame) - if lhs !== nothing && rt !== Bottom - changes = StateUpdate(lhs::SlotNumber, VarState(rt, false)) - end + @label injected_result + (; rt, exct, effects, changes, refinements, currsaw_latestworld) = result end + effects === nothing || merge_override_effects!(interp, effects, frame) if !has_curr_ssaflag(frame, IR_FLAG_NOTHROW) if exct !== Union{} update_exc_bestguess!(interp, exct, frame) From 0cf5a4dd63872b21a1817c6d4eb1bb63af5bbe89 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 8 Apr 2025 15:45:54 -0400 Subject: [PATCH 060/662] Make delete_binding operate in the latest world (#58043) Fixes #58016 --- src/module.c | 18 +++++++++++------- test/rebinding.jl | 9 +++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/module.c b/src/module.c index 53466c27e8ac9..4c685ca574523 100644 --- a/src/module.c +++ b/src/module.c @@ -1727,16 +1727,20 @@ JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_binding_kind(bpart) == PARTITION_KIND_GUARD) { - // Already guard + for (;;) { + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_atomic_load_acquire(&jl_world_counter)); + + if (jl_binding_kind(bpart) == PARTITION_KIND_GUARD) { + // Already guard + return; + } + + if (!jl_replace_binding(b, bpart, NULL, PARTITION_KIND_GUARD)) + continue; + return; } - - for (;;) - if (jl_replace_binding(b, bpart, NULL, PARTITION_KIND_GUARD)) - break; } JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) diff --git a/test/rebinding.jl b/test/rebinding.jl index 1c2a4d7a0b91c..56bde53b7b746 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -383,3 +383,12 @@ end # M3 connects all, so we should have a single partition @test access_and_count(:afterM3) == 1 + +# Test that delete_binding in an outdated world age works +module BindingTestModule; end +function create_and_delete_binding() + Core.eval(BindingTestModule, :(const x = 1)) + Base.delete_binding(BindingTestModule, :x) +end +create_and_delete_binding() +@test Base.binding_kind(BindingTestModule, :x) == Base.PARTITION_KIND_GUARD From 4480f42a63c4915a9d0514194a1f69aa9f2c719a Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 9 Apr 2025 08:36:18 -0400 Subject: [PATCH 061/662] inference: simplify `abstract_eval_globalref` (#58026) By moving the optional `assume_bindings_static` refinement logic into `abstract_eval_partition_load` and removing the redirect via `abstract_eval_globalref_partition`. Also adds test cases with `assume_bindings_static=true`. --- Compiler/src/abstractinterpretation.jl | 46 ++++++++++++-------------- Compiler/test/inference.jl | 22 +++++++++--- base/invalidation.jl | 6 ++-- 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 97c68d48f34ef..1c6bfd93935ae 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2574,9 +2574,9 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) gr = GlobalRef(M, s) - (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition + (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, binding, partition partition_T = nothing - partition_rte = abstract_eval_partition_load(interp, partition) + partition_rte = abstract_eval_partition_load(interp, binding, partition) if binding_kind(partition) == PARTITION_KIND_GLOBAL partition_T = partition_restriction(partition) end @@ -3558,13 +3558,11 @@ function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::Global return partition end -abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, ::Core.Binding, partition::Core.BindingPartition) = - abstract_eval_partition_load(interp, partition) -function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, partition::Core.BindingPartition) +function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, binding::Core.Binding, partition::Core.BindingPartition) kind = binding_kind(partition) isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) - if is_some_guard(kind) || kind == PARTITION_KIND_UNDEF_CONST + if is_some_guard(kind) if interp !== nothing && InferenceParams(interp).assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else @@ -3590,12 +3588,23 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing # Could be replaced by a backdated const which has an effect, so we can't assume it won't. # Besides, we would prefer not to merge the world range for this into the world range for # _GLOBAL, because that would pessimize codegen. - local_getglobal_effects = Effects(local_getglobal_effects, effect_free=ALWAYS_FALSE) + effects = Effects(local_getglobal_effects, effect_free=ALWAYS_FALSE) rt = Any else rt = partition_restriction(partition) + effects = local_getglobal_effects + end + if (interp !== nothing && InferenceParams(interp).assume_bindings_static && + kind in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) && + isdefined(binding, :value)) + exct = Union{} + effects = Effects(generic_getglobal_effects; nothrow=true) + else + # We do not assume in general that assigned global bindings remain assigned. + # The existence of pkgimages allows them to revert in practice. + exct = UndefVarError end - return RTEffects(rt, UndefVarError, local_getglobal_effects) + return RTEffects(rt, exct, effects) end function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp, g::GlobalRef, wwr::WorldWithRange) @@ -3643,28 +3652,15 @@ scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) -function abstract_eval_globalref_partition(interp, binding::Core.Binding, partition::Core.BindingPartition) - # For inference purposes, we don't particularly care which global binding we end up loading, we only - # care about its type. However, we would still like to terminate the world range for the particular - # binding we end up reaching such that codegen can emit a simpler pointer load. - Pair{RTEffects, Union{Nothing, Core.Binding}}( - abstract_eval_partition_load(interp, partition), - binding_kind(partition) in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) ? binding : nothing) -end - function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) if saw_latestworld return RTEffects(Any, Any, generic_getglobal_effects) end - (valid_worlds, (ret, binding_if_global)) = scan_leaf_partitions(abstract_eval_globalref_partition, interp, g, sv.world) + # For inference purposes, we don't particularly care which global binding we end up loading, we only + # care about its type. However, we would still like to terminate the world range for the particular + # binding we end up reaching such that codegen can emit a simpler pointer load. + (valid_worlds, ret) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, sv.world) update_valid_age!(sv, valid_worlds) - if ret.rt !== Union{} && ret.exct === UndefVarError && binding_if_global !== nothing && InferenceParams(interp).assume_bindings_static - if isdefined(binding_if_global, :value) - ret = RTEffects(ret.rt, Union{}, Effects(generic_getglobal_effects, nothrow=true)) - end - # We do not assume in general that assigned global bindings remain assigned. - # The existence of pkgimages allows them to revert in practice. - end return ret end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index ded2ea7ba0030..249a614fccc5b 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6348,14 +6348,28 @@ end === Int swapglobal!(@__MODULE__, :swapglobal!_xxx, x) end === Union{} +@newinterp AssumeBindingsStaticInterp +Compiler.InferenceParams(::AssumeBindingsStaticInterp) = Compiler.InferenceParams(; assume_bindings_static=true) + eval(Expr(:const, :swapglobal!_must_throw)) -@newinterp SwapGlobalInterp -Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true) function func_swapglobal!_must_throw(x) swapglobal!(@__MODULE__, :swapglobal!_must_throw, x) end -@test Base.infer_return_type(func_swapglobal!_must_throw, (Int,); interp=SwapGlobalInterp()) === Union{} -@test !Compiler.is_effect_free(Base.infer_effects(func_swapglobal!_must_throw, (Int,); interp=SwapGlobalInterp()) ) +@test Base.infer_return_type(func_swapglobal!_must_throw, (Int,); interp=AssumeBindingsStaticInterp()) === Union{} +@test !Compiler.is_effect_free(Base.infer_effects(func_swapglobal!_must_throw, (Int,); interp=AssumeBindingsStaticInterp()) ) + +global global_decl_defined +global_decl_defined = 42 +@test Base.infer_effects(; interp=AssumeBindingsStaticInterp()) do + global global_decl_defined + return global_decl_defined +end |> Compiler.is_nothrow +global global_decl_defined2::Int +global_decl_defined2 = 42 +@test Base.infer_effects(; interp=AssumeBindingsStaticInterp()) do + global global_decl_defined2 + return global_decl_defined2 +end |> Compiler.is_nothrow @eval get_exception() = $(Expr(:the_exception)) @test Base.infer_return_type() do diff --git a/base/invalidation.jl b/base/invalidation.jl index 14b88e71b9def..e974bcd226de8 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -124,12 +124,12 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world) (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1) - # abstract_eval_globalref_partition is the maximum amount of information that inference + # `abstract_eval_partition_load` is the maximum amount of information that inference # reads from a binding partition. If this information does not change - we do not need to # invalidate any code that inference created, because we know that the result will not change. need_to_invalidate_code = - Compiler.abstract_eval_globalref_partition(nothing, ib, ibpart) !== - Compiler.abstract_eval_globalref_partition(nothing, nb, nbpart) + Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !== + Compiler.abstract_eval_partition_load(nothing, nb, nbpart) need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== export_affecting_partition_flags(new_bpart) From 215dca2b90b6d059b5578fb5e46d86dde8a78a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pi=C5=A1l?= <70214112+romanrexc@users.noreply.github.com> Date: Wed, 9 Apr 2025 18:57:09 +0200 Subject: [PATCH 062/662] pkgimage.mk: Avoid using environment variable as Makefile variable. (#57753) --- pkgimage.mk | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgimage.mk b/pkgimage.mk index 78b2618be549f..9217573ab623f 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -4,9 +4,10 @@ JULIAHOME := $(SRCDIR) include $(JULIAHOME)/Make.inc include $(JULIAHOME)/stdlib/stdlib.mk +DEPOTDIR := $(build_prefix)/share/julia # set some influential environment variables -export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_prefix)/share/julia)) +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(DEPOTDIR))) export JULIA_LOAD_PATH := @stdlib$(PATHSEP)$(shell echo $(call cygpath_w,$(JULIAHOME)/stdlib)) unexport JULIA_PROJECT := unexport JULIA_BINDIR := @@ -18,13 +19,13 @@ release: $(BUILDDIR)/stdlib/release.image debug: $(BUILDDIR)/stdlib/debug.image all: release debug -$(JULIA_DEPOT_PATH)/compiled: +$(DEPOTDIR)/compiled: mkdir -p $@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') -$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(JULIA_DEPOT_PATH)/compiled +$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled @$(call PRINT_JULIA, JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') touch $@ @@ -33,5 +34,5 @@ $(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) $(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) clean: - rm -rf $(JULIA_DEPOT_PATH)/compiled + rm -rf $(DEPOTDIR)/compiled rm -f $(BUILDDIR)/stdlib/*.image From 9d4e31f7eb9441f1d78f07d9fb181dbac59ce7a8 Mon Sep 17 00:00:00 2001 From: Chengyu Han Date: Thu, 10 Apr 2025 18:08:37 +0800 Subject: [PATCH 063/662] deps: Update MPFR to v4.2.2 (#58039) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GNU MPFR version 4.2.2 was released on 20 March 2025. Fix #57945 --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- deps/checksums/mpfr | 76 ++++++++++++++++---------------- deps/mpfr.version | 3 +- stdlib/MPFR_jll/Project.toml | 2 +- stdlib/MPFR_jll/test/runtests.jl | 2 +- stdlib/Manifest.toml | 2 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/deps/checksums/mpfr b/deps/checksums/mpfr index 7b3b57978bd01..7f0de6099713c 100644 --- a/deps/checksums/mpfr +++ b/deps/checksums/mpfr @@ -1,38 +1,38 @@ -MPFR.v4.2.1+2.aarch64-apple-darwin.tar.gz/md5/1f5bba3e8e540720e239da75e5ae79eb -MPFR.v4.2.1+2.aarch64-apple-darwin.tar.gz/sha512/7de26c625e540a5b88e280ec2cb8712d4514732d80a0c6342d2b2cabc6bc17c05f6c614b8e38800c93a4af5438c554733d3fa2002ef70072dfb44c08d3f03d26 -MPFR.v4.2.1+2.aarch64-linux-gnu.tar.gz/md5/112ddd4e5cddf36b005394f9cd81b8e5 -MPFR.v4.2.1+2.aarch64-linux-gnu.tar.gz/sha512/dc125f625e8c74ce18c052ef759ccbcfc2f3a932f2810a306bdddf70d5f37f3546200690fd08fb76742022322a7c1b9aa907b4aec6edb318060f0648ff426cbc -MPFR.v4.2.1+2.aarch64-linux-musl.tar.gz/md5/a0919ef7cc35bb663d05e27da2bcb9a7 -MPFR.v4.2.1+2.aarch64-linux-musl.tar.gz/sha512/8acbaaca766c2ce225ac8df88c103a57fc52119d1fd54e9fc7d1f9d725c4ca9f74a0090e86eea0c140482a1abaf5b6086c453824a7516e9aef3ede5058f1767c -MPFR.v4.2.1+2.aarch64-unknown-freebsd.tar.gz/md5/61e1dcc7e323b976854a4e8164316d37 -MPFR.v4.2.1+2.aarch64-unknown-freebsd.tar.gz/sha512/f3a5493f88b290d15aff9bf79b15158d19bea05af7210b2967368e0b2f98cd291f77e62f39ee0c7ad4e9d2ef6ebdba4bf2fea24c723791f71f7b9b1ef989a67d -MPFR.v4.2.1+2.armv6l-linux-gnueabihf.tar.gz/md5/629aad4ac45ba23becd8a26df188638c -MPFR.v4.2.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/bb05a8bf127eb16608a82037546f48462cb6168e1adcdb2c60dc3bd08f62cff30cf603abcab87bb336305d37dbb7b0480ea8f6664191879bdcd487738a33dd99 -MPFR.v4.2.1+2.armv6l-linux-musleabihf.tar.gz/md5/0c3c026051b096d98c8d476dd44db334 -MPFR.v4.2.1+2.armv6l-linux-musleabihf.tar.gz/sha512/9e791fe9748c87068c167517883cc905fe51ea38d2db89562a7a0959cfd83b268eed2897e5eaaf90c0b0b08a4efd8039bdeece64e83b17bf1d676570d13c2b98 -MPFR.v4.2.1+2.armv7l-linux-gnueabihf.tar.gz/md5/a2433a717e49ad95c3e430a538d01134 -MPFR.v4.2.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/abde21a943d4af312e0d44b1ff1d4aefa10b2f38c74ff0e04c0c2b8561750ef5d164679564ffe1b551821d83ebcafbe99467230b37fe4591c593a24dfb070c6a -MPFR.v4.2.1+2.armv7l-linux-musleabihf.tar.gz/md5/4c892b4cbf1926d5d2b6a88330015c8f -MPFR.v4.2.1+2.armv7l-linux-musleabihf.tar.gz/sha512/24825bb1268ef2ea42894ec9ff6589308abae430dd8e43a2ca0d368f1e718fd3cdf6d9bc4bc383346970ba845d2ef1721c4848ee0c783d09addc5505131db3e6 -MPFR.v4.2.1+2.i686-linux-gnu.tar.gz/md5/0b1e0268dcaeb3aa0f7f0a6451c6b841 -MPFR.v4.2.1+2.i686-linux-gnu.tar.gz/sha512/f0ef142c7b86e8f92b78a7ff0607da70bf8f3970b118fa77438cbb0acbea604dc0c7566b52ff1f85b179aac7661b31e4aee049f2c5ff799c95b385ba9cde2a25 -MPFR.v4.2.1+2.i686-linux-musl.tar.gz/md5/2fc9a938e76e7bdc0b73d7e8bfc8b8ee -MPFR.v4.2.1+2.i686-linux-musl.tar.gz/sha512/4aed3884ad569b7695b9383db9d9dbb279ffe5349f7757b867ff860fa600b47faa4c169f4a60409666ce45fc6e6f269c18cef2df6fa0585f056d7e07e55005b8 -MPFR.v4.2.1+2.i686-w64-mingw32.tar.gz/md5/d13c44bb28d721107639c8555db5e157 -MPFR.v4.2.1+2.i686-w64-mingw32.tar.gz/sha512/1b5562d2df322c28bd06bb4ba8c9039cf90ed62affcf7f2b0d7ae8925d503c76a0d3d2f9b65c8c55575f245a4df8fbc4c7c63e93e7b973188f203a7fbda4eac5 -MPFR.v4.2.1+2.powerpc64le-linux-gnu.tar.gz/md5/52b3912b2c5f59ab3dcd7c3e06ca41b5 -MPFR.v4.2.1+2.powerpc64le-linux-gnu.tar.gz/sha512/533cf1f93c4464b4bed1d56ea79946fc2d20f3a7825d6b0383ed98cec99f85713e7bca549fd8948adb69aedc14e5d14a54238b3e67ef103e1b049b0cfb6cc1c9 -MPFR.v4.2.1+2.riscv64-linux-gnu.tar.gz/md5/aef7709c8457ee2db2622c39f1da16b7 -MPFR.v4.2.1+2.riscv64-linux-gnu.tar.gz/sha512/7a9c88563e3e7ab22a3aaa45690ed89c3e7eb22333a3d45c5e04ad2660c91ad2c97f10cd6c1aa1ccfdbf97186f9fd7f92330a41ec0be026e2ff84c5ba91f2652 -MPFR.v4.2.1+2.x86_64-apple-darwin.tar.gz/md5/12afc9778e39a5b6d9ea0161e2c80a95 -MPFR.v4.2.1+2.x86_64-apple-darwin.tar.gz/sha512/a9070423a898fa865740753ae7513d3cc0b500bd9b6b5c6aa672833dcac429efd806eff48501b51afcba5db0d31e79dac243b11b2f8847a1551576c6131506f5 -MPFR.v4.2.1+2.x86_64-linux-gnu.tar.gz/md5/46c6a5f40243795bdff51bd68a89c82e -MPFR.v4.2.1+2.x86_64-linux-gnu.tar.gz/sha512/df8209d69ae55dd54491055078f113f4ac8be7bc68e1c0eb62944e6c9c04ed3e9a55c4a5f28ec68eb69f558d9f4d1b975f36de572fbd0ef7720568efc8042327 -MPFR.v4.2.1+2.x86_64-linux-musl.tar.gz/md5/045236ee0d558d2eda42df76c3397f69 -MPFR.v4.2.1+2.x86_64-linux-musl.tar.gz/sha512/52b68a673160af7cd09b191f3c28e17d5af7516b5baa86c0df9cb63a116772a15b5358f3db5f0b254b5752c652f8959454667cc1726ea4ff30946e3bbdb90ab4 -MPFR.v4.2.1+2.x86_64-unknown-freebsd.tar.gz/md5/da3da71bc7572eca5bc3d3895abf73c2 -MPFR.v4.2.1+2.x86_64-unknown-freebsd.tar.gz/sha512/4270b83ebe72d431f8fd9127b2b8d3bd75c2e52c563d390a4ca8d40c0514f5996fce57746d07b7d3bcbf93bfe78d420f815fde5eda4d84a5bcb7b7cf0e092504 -MPFR.v4.2.1+2.x86_64-w64-mingw32.tar.gz/md5/2a6f5ccb8d45591a845ad43916beb85a -MPFR.v4.2.1+2.x86_64-w64-mingw32.tar.gz/sha512/db9ecc9d8247fe4421c4cc9c6ab540e17a7445056b7a1062d4e334b353783a1c067062fd8e6f0517d8bd8782c9bb75abcce8ab8247be707ba066dc90b7fc12ff -mpfr-4.2.1.tar.bz2/md5/7765afa036e4ce7fb0e02bce0fef894b -mpfr-4.2.1.tar.bz2/sha512/c81842532ecc663348deb7400d911ad71933d3b525a2f9e5adcd04265c9c0fdd1f22eca229f482703ac7f222ef209fc9e339dd1fa47d72ae57f7f70b2336a76f +MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/md5/01a13215fd646c761e469f36f693fdc8 +MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/sha512/da473776ac8c687ab34792235ee5e1e08dc6a2e29b73620bd6dac93db32397037ae502b8ac3a35e020f722dae7da007a060e5e11e3287c4cdb846bf7e5168297 +MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/md5/58ca9f3e08a388c3e40692e623f3884e +MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/sha512/c6846d982ce1211791b466ed6fed2aad9e5f9a4866c48db99eb288dcbb1480660772010869fdea66d6453c8c140c92e367cfe55f6087fe41ea040fbd77eafe34 +MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/md5/2ff7e1400f27d049e3274a6277322860 +MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/sha512/388f7050288be9d30c4a2e772c0859e414b0cf6dbc845eec0eb6aeda53595df94a4e3001d02fa04c173fcf74e00c2552a8880b62ebf5adf443da2a95497be891 +MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/md5/d1e6c477ab9678d1cd1dfa7e00366e69 +MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/sha512/897174756651d01272d86bb147f5dda9f84f8f1bf1fe02b8505e141df3cc38523019f85cbe538fcc6ea8073d7743fc6428a06271107b059de80cd8f959c52daa +MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/md5/5213b0ef1b191c529e3335e05b918003 +MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/bbcdb90f80d8cb826cd055eb41f051890c7847fc0887389b61bd24c051d35873af36672e5f1956cc3fb23b8e3ee50ee069c185fc2faabe302787d70210bd5b07 +MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/md5/9a9d9207a6b52b6e84b1b2b1c631e0f2 +MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/sha512/fd40d16a40b1db2b441339e5c8cb3f8a1810d2889713b0504f9bfd5451f4f4c2dd0ca35a4b2922feca9cf50e4a9b3bf8cf2c088655dd85a23c33ee67c12e0a72 +MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/md5/44532dd5607ced01a8ba0856c3bfdbc3 +MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/469fc030f458bd52f6bdffc442ceaaf8659f0f1e40d581eb1303fd4753d2c665fcb75bc6c54d04eb53d77b1945d67f48a5ea5614f2ee82cc7fd27e89859b45f4 +MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/md5/fbd13b054b8d27be6bc836283f7846bf +MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/sha512/926dc03f99a6827c833614d17c5ef4f80fb862bdf4397db9aaf8ae9b3a66e8b9121cfa044b18db46f5774abbd7e9c129363183ccb2ae3192084711e7ff9d6382 +MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/md5/da6fbb90dc20830af9325cfaf3544e4c +MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/sha512/d235884e1d1bef406b1e5ceb9c34aab68c1a8040b2022964105238ef8cdfd4af7aebe474fef80849689ca88d9168697fd55e8d6ab92b6641a1f37c431d5e3ff3 +MPFR.v4.2.2+0.i686-linux-musl.tar.gz/md5/fc885092e1469a06aaaaf24168e8fafe +MPFR.v4.2.2+0.i686-linux-musl.tar.gz/sha512/5307926e1222b302e48e2f5c08479b920279d15b95937a245e16ac1dfd5c6206cb64fe4b6ca4cb7d6be847d8cc01a04d2661a630b978dc2dbd60605d222b8b21 +MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/md5/55f129d5b5b849b3bc018e68ccf14914 +MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/sha512/9a24e4616e05f5c1fb53e7a12167f7a55d05ec1895124d6ee23b2efd548f49e4c7995c16d240ec803f352d586ae4667027ee0bdeefa520e0c1f581fcc338dc44 +MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/md5/6f47e4cde45ddf0cb2ea4f31ef9c9e04 +MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/sha512/4fd8fbe166e719c636e430d4d5c938231fa9126b29eacbc678d2eb50d3d4b95cf6ccef155ce401c6d33b9730c2f89c0c77ec8fb39254483c2e4004639c503c1c +MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/md5/c4736705ff2a55cf8206c3af84bfc417 +MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/sha512/e1e77d64ee88de2990fbc791d7307afe859cfbdc1ac67e7bdfa633627b5542ce2e3ee0cd9fe4036abfaf60509277a43f263e2155665ac2c5e38b8627e470f399 +MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/md5/c3e983178a1e9600f42714d4cc1ecdf6 +MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/sha512/c5c6cebcdfc5b7b84e9e217a81d99e5af78d163949745d570af5689210b3eedeb9de3c11991b1b36d8fdbee17b550a4072af951d19c3f863cf24cda7d9c12950 +MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/md5/61fc7c7aa676d0a07e1709b433a8e423 +MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/sha512/74bdefa72c51c82ca709e3494cd664a6593173bbfbe0198f18f4c0add06ce4c1217e4dd49e99cb151d71c85cd696ae2147aed29ed2cf3f1ca0e5b40582abb571 +MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/md5/207ee8ad2293ba36d3d7bb845ab346e0 +MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/sha512/63325e6595861a324f3c299d8c51b1d665197217c8fc9a5ae627b624037394f050bb08a9acd14e9809f982942c066f1185dded0fa493f360bcd3baae17a05f92 +MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/md5/74e5a5ce0ea84959ccec7b7f7ab22c66 +MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/sha512/411dbb339218669af6181fdf1e17f926abb9830ae54a8f9ef1b7df53021e8da01a41fda13067731afaf9b803324d5f82c060ef5b5b91045625188458b99dcc75 +MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/md5/2de84b494ea832147be4f9bfa786cd19 +MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/sha512/5f86aef6ab4fd7517cb23ad9a32ae21954a3ce1f27f5cbd28abe038271e20197b7c241055092a4aa6d5391f012bdee10465c58b53acd64bb5b99fd754c75ad29 +mpfr-4.2.2.tar.bz2/md5/afe8268360bc8702fbc8297d351c8b5e +mpfr-4.2.2.tar.bz2/sha512/0176e50808dcc07afbf5bc3e38bf9b7b21918e5f194aa0bfd860d99b00c470630aef149776c4be814a61c44269c3a5b9a4b0b1c0fcd4c9feb1459d8466452da8 diff --git a/deps/mpfr.version b/deps/mpfr.version index ec109e181ecdc..70585f95a6385 100644 --- a/deps/mpfr.version +++ b/deps/mpfr.version @@ -1,5 +1,6 @@ +# -*- makefile -*- ## jll artifact MPFR_JLL_NAME := MPFR ## source build -MPFR_VER := 4.2.1 +MPFR_VER := 4.2.2 diff --git a/stdlib/MPFR_jll/Project.toml b/stdlib/MPFR_jll/Project.toml index 50de38f169ff0..9958383f4e65b 100644 --- a/stdlib/MPFR_jll/Project.toml +++ b/stdlib/MPFR_jll/Project.toml @@ -1,6 +1,6 @@ name = "MPFR_jll" uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" -version = "4.2.1+2" +version = "4.2.2+0" [deps] GMP_jll = "781609d7-10c4-51f6-84f2-b8444358ff6d" diff --git a/stdlib/MPFR_jll/test/runtests.jl b/stdlib/MPFR_jll/test/runtests.jl index fc931b462fa9c..1dbbbb298e737 100644 --- a/stdlib/MPFR_jll/test/runtests.jl +++ b/stdlib/MPFR_jll/test/runtests.jl @@ -4,5 +4,5 @@ using Test, Libdl, MPFR_jll @testset "MPFR_jll" begin vn = VersionNumber(unsafe_string(ccall((:mpfr_get_version,libmpfr), Cstring, ()))) - @test vn == v"4.2.1" + @test vn == v"4.2.2" end diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 418931e34ab80..ca3863d974f6f 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -136,7 +136,7 @@ version = "1.11.0" [[deps.MPFR_jll]] deps = ["Artifacts", "GMP_jll", "Libdl"] uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" -version = "4.2.1+1" +version = "4.2.2+0" [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] From a0740d0984d00da8010002079224e707cdcd8ee4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 10 Apr 2025 10:30:21 -0400 Subject: [PATCH 064/662] debuginfo: improve robustness of code slightly (#57989) I hit this iswindows assert locally while frobbing stuff, so I wanted to remove it and clean up the code slightly to be more careful here about verifying what it got from the file system. --- src/debuginfo.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 6db6a07c917b1..53c07c978f72a 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -568,7 +568,7 @@ void JITDebugInfoRegistry::libc_frames_t::libc_deregister_frame(const char *Entr } #endif -static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) JL_NOTSAFEPOINT +static bool getObjUUID(const object::MachOObjectFile *obj, uint8_t uuid[16]) JL_NOTSAFEPOINT { for (auto Load : obj->load_commands()) { @@ -830,9 +830,6 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA std::string debuginfopath; uint8_t uuid[16], uuid2[16]; if (isdarwin) { - // Hide Darwin symbols (e.g. CoreFoundation) from non-Darwin systems. -#ifdef _OS_DARWIN_ - size_t msize = (size_t)(((uint64_t)-1) - fbase); std::unique_ptr membuf = MemoryBuffer::getMemBuffer( StringRef((const char *)fbase, msize), "", false); @@ -843,14 +840,18 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA return entry; } - llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile*) - origerrorobj.get().get(); + const object::MachOObjectFile *morigobj = dyn_cast( + origerrorobj.get().get()); // First find the uuid of the object file (we'll use this to make sure we find the // correct debug symbol file). - if (!getObjUUID(morigobj, uuid)) + if (!morigobj || !getObjUUID(morigobj, uuid)) return entry; + // Hide Darwin symbols (e.g. CoreFoundation) from non-Darwin systems. +#ifndef _OS_DARWIN_ + return entry; +#else // On macOS, debug symbols are not contained in the dynamic library. // Use DBGCopyFullDSYMURLForUUID from the private DebugSymbols framework // to make use of spotlight to find the dSYM file. If that fails, lookup @@ -906,6 +907,7 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA if (dsfmwkbundle) { CFRelease(dsfmwkbundle); } +#endif if (objpath.empty()) { // Fall back to simple path relative to the dynamic library. @@ -915,7 +917,6 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA debuginfopath += fname.substr(sep + 1); objpath = debuginfopath; } -#endif } else { // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. @@ -974,15 +975,17 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA if (isdarwin) { // verify the UUID matches - if (!getObjUUID((llvm::object::MachOObjectFile*)debugobj, uuid2) || - memcmp(uuid, uuid2, sizeof(uuid)) != 0) { + if (!isa(debugobj) || + !getObjUUID(cast(debugobj), uuid2) || + memcmp(uuid, uuid2, sizeof(uuid)) != 0) { return entry; } } int64_t slide = 0; if (auto *OF = dyn_cast(debugobj)) { - assert(iswindows); + if (!iswindows) // the COFF parser accepts some garbage inputs (like empty files) that the other parsers correctly reject, so we can end up here even when we should not + return entry; slide = OF->getImageBase() - fbase; } else { From 1117df66f2f199e7224697d110d023add4231579 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 10 Apr 2025 11:48:16 -0400 Subject: [PATCH 065/662] Remove try-finally scope from `@time_imports` `@trace_compile` `@trace_dispatch` (#58011) Matches `@time` `@profile` etc. --- base/timing.jl | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/base/timing.jl b/base/timing.jl index 61fa73f2eff62..2dabe964f08c6 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -643,33 +643,36 @@ end # here so it's possible to time/trace all imports, including InteractiveUtils and its deps macro time_imports(ex) quote - try - Base.Threads.atomic_add!(Base.TIMING_IMPORTS, 1) - $(esc(ex)) - finally + Base.Threads.atomic_add!(Base.TIMING_IMPORTS, 1) + @__tryfinally( + # try + $(esc(ex)), + # finally Base.Threads.atomic_sub!(Base.TIMING_IMPORTS, 1) - end + ) end end macro trace_compile(ex) quote - try - ccall(:jl_force_trace_compile_timing_enable, Cvoid, ()) - $(esc(ex)) - finally + ccall(:jl_force_trace_compile_timing_enable, Cvoid, ()) + @__tryfinally( + # try + $(esc(ex)), + # finally ccall(:jl_force_trace_compile_timing_disable, Cvoid, ()) - end + ) end end macro trace_dispatch(ex) quote - try - ccall(:jl_force_trace_dispatch_enable, Cvoid, ()) - $(esc(ex)) - finally + ccall(:jl_force_trace_dispatch_enable, Cvoid, ()) + @__tryfinally( + # try + $(esc(ex)), + # finally ccall(:jl_force_trace_dispatch_disable, Cvoid, ()) - end + ) end end From 90eb96fe45804b6cd76885e9798f5ca6acf41552 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 10 Apr 2025 18:37:19 -0400 Subject: [PATCH 066/662] fix #58013, error for too few arguments in `invokelatest` (#58056) Also a couple builtins had the wrong function name in their error messages. fix #58013 --- src/builtins.c | 7 ++++--- test/worlds.jl | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 67900dc10584f..a2cae857f26b4 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -846,6 +846,7 @@ JL_CALLABLE(jl_f__apply_iterate) // this is like a regular call, but always runs in the newest world JL_CALLABLE(jl_f_invokelatest) { + JL_NARGSV(invokelatest, 1); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; if (!ct->ptls->in_pure_callback) @@ -859,10 +860,10 @@ JL_CALLABLE(jl_f_invokelatest) // If world > jl_atomic_load_acquire(&jl_world_counter), run in the latest world. JL_CALLABLE(jl_f_invoke_in_world) { - JL_NARGSV(_apply_in_world, 2); + JL_NARGSV(invoke_in_world, 2); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; - JL_TYPECHK(_apply_in_world, ulong, args[0]); + JL_TYPECHK(invoke_in_world, ulong, args[0]); size_t world = jl_unbox_ulong(args[0]); if (!ct->ptls->in_pure_callback) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); @@ -877,7 +878,7 @@ JL_CALLABLE(jl_f_invoke_in_world) JL_CALLABLE(jl_f__call_in_world_total) { JL_NARGSV(_call_in_world_total, 2); - JL_TYPECHK(_apply_in_world, ulong, args[0]); + JL_TYPECHK(_call_in_world_total, ulong, args[0]); jl_task_t *ct = jl_current_task; int last_in = ct->ptls->in_pure_callback; jl_value_t *ret = NULL; diff --git a/test/worlds.jl b/test/worlds.jl index 686708c5efd27..4520ab257d078 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -5,6 +5,9 @@ using Base: get_world_counter, tls_world_age @test typemax(UInt) > get_world_counter() == tls_world_age() > 0 +# issue #58013 +@test_throws ArgumentError invokelatest() + # test simple method replacement begin g265a() = f265a(0) From 4b19ab4a448b516234f7aad5c16570ee293a95d6 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 10 Apr 2025 20:03:01 -0400 Subject: [PATCH 067/662] [CompilerDevTools] Use `with_new_compiler` directly (#58069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise this fails IR validation potentially, because the bounds of that globalref are not being taken into account in the world ages, e.g. (In julia-debug): ``` julia> CompilerDevTools.with_new_compiler(Base.fdio, Int32(1)) Unbound or partitioned GlobalRef not allowed in value position ERROR: IR verification failed. Code location: lock.jl:174 Method instance: MethodInstance for Base._trylock(::ReentrantLock, ::Task) Stacktrace: [1] error(::String, ::String, ::String, ::Symbol, ::String, ::Int32, ::String, ::String, ::Core.MethodInstance) @ Base ./error.jl:54 [2] (::Compiler.var"#raise_error#verify_ir##0"{Compiler.IRCode, Core.MethodInstance})() @ Compiler ./../usr/share/julia/Compiler/src/ssair/verify.jl:125 [3] check_op(ir::Compiler.IRCode, domtree::Compiler.GenericDomTree{…}, op::Any, use_bb::Int64, use_idx::Int64, printed_use_idx::Int64, print::Bool, isforeigncall::Bool, arg_idx::Int64, allow_frontend_forms::Bool, raise_error::Any) @ Compiler ./../usr/share/julia/Compiler/src/ssair/verify.jl:71 [4] verify_ir(ir::Compiler.IRCode, print::Bool, allow_frontend_forms::Bool, 𝕃ₒ::Compiler.PartialsLattice{…}, mi::Core.MethodInstance) @ Compiler ./../usr/share/julia/Compiler/src/ssair/verify.jl:429 [5] run_passes_ipo_safe(ci::Core.CodeInfo, sv::Compiler.OptimizationState{CompilerDevTools.SplitCacheInterp}, optimize_until::Nothing) @ Compiler ./../usr/share/julia/Compiler/src/optimize.jl:1027 [6] run_passes_ipo_safe @ ./../usr/share/julia/Compiler/src/optimize.jl:1011 [inlined] [7] optimize(interp::CompilerDevTools.SplitCacheInterp, opt::Compiler.OptimizationState{…}, caller::Compiler.InferenceResult) @ Compiler ./../usr/share/julia/Compiler/src/optimize.jl:985 [8] finish_nocycle(::CompilerDevTools.SplitCacheInterp, frame::Compiler.InferenceState, time_before::UInt64) @ Compiler ./../usr/share/julia/Compiler/src/typeinfer.jl:213 [9] typeinf(interp::CompilerDevTools.SplitCacheInterp, frame::Compiler.InferenceState) @ Compiler ./../usr/share/julia/Compiler/src/abstractinterpretation.jl:4435 [10] typeinf_ext(interp::CompilerDevTools.SplitCacheInterp, mi::Core.MethodInstance, source_mode::UInt8) @ Compiler ./../usr/share/julia/Compiler/src/typeinfer.jl:1311 [11] typeinf_ext_toplevel(interp::CompilerDevTools.SplitCacheInterp, mi::Core.MethodInstance, source_mode::UInt8) @ Compiler ./../usr/share/julia/Compiler/src/typeinfer.jl:1403 [12] typeinf(owner::CompilerDevTools.SplitCacheOwner, mi::Core.MethodInstance, source_mode::UInt8) @ CompilerDevTools ~/julia/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl:35 [13] with_new_compiler(f::Function, owner::CompilerDevTools.SplitCacheOwner, args::Int32) @ CompilerDevTools ~/julia/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl:71 [14] with_new_compiler(f::Function, args::Int32) @ CompilerDevTools ~/julia/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl:67 ``` --- Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index 7f6ee154e29dd..269de99ebe8c7 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -48,7 +48,7 @@ end function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) opt = result.src::Compiler.OptimizationState ir = opt.result.ir::Compiler.IRCode - override = GlobalRef(@__MODULE__(), :with_new_compiler) + override = with_new_compiler for inst in ir.stmts stmt = inst[:stmt] isexpr(stmt, :call) || continue From 451766a295cc48179745999fb38430747332c67d Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 10 Apr 2025 21:36:00 -0400 Subject: [PATCH 068/662] [CompilerDevTools] Properly handle builtins in with_new_compiler (#58074) --- .../extras/CompilerDevTools/src/CompilerDevTools.jl | 10 ++++++---- Compiler/extras/CompilerDevTools/test/runtests.jl | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index 269de99ebe8c7..e08d08779e097 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -54,10 +54,8 @@ function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::C isexpr(stmt, :call) || continue f = stmt.args[1] f === override && continue - if isa(f, GlobalRef) - T = widenconst(argextype(f, ir)) - T <: Core.Builtin && continue - end + T = widenconst(argextype(f, ir)) + T <: Core.Builtin && continue insert!(stmt.args, 1, override) insert!(stmt.args, 3, interp.owner) end @@ -67,6 +65,10 @@ end with_new_compiler(f, args...; owner::SplitCacheOwner = SplitCacheOwner()) = with_new_compiler(f, owner, args...) function with_new_compiler(f, owner::SplitCacheOwner, args...) + # We try to avoid introducing `with_new_compiler` in the first place, + # but if we can't see the type, it's still possible to end up with a + # builtin here - simply forward to the ordinary builtin call. + isa(f, Core.Builtin) && return f(args...) mi = lookup_method_instance(f, args...) new_compiler_ci = Core.OptimizedGenerics.CompilerPlugins.typeinf( owner, mi, Compiler.SOURCE_MODE_ABI diff --git a/Compiler/extras/CompilerDevTools/test/runtests.jl b/Compiler/extras/CompilerDevTools/test/runtests.jl index e076de388f96b..89dc4696d9e1c 100644 --- a/Compiler/extras/CompilerDevTools/test/runtests.jl +++ b/Compiler/extras/CompilerDevTools/test/runtests.jl @@ -17,4 +17,8 @@ using CompilerDevTools: lookup_method_instance, SplitCacheInterp # required extra work to be cached under the same cache owner. mi = lookup_method_instance(do_work, 1, 2) @test haskey(cache, mi) + + # Should not error with a builtin whose type we do not know + f_unknown_builtin() = Base.compilerbarrier(:type, isa)(1, Int) + with_new_compiler(f_unknown_builtin, interp.owner) === true end; From 92fc06fd83ac76c6a495019db112a4e907526591 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 11 Apr 2025 02:21:31 -0400 Subject: [PATCH 069/662] Fix missing CodeInstance owner lookup in _jl_invoke (#58072) `_jl_invoke` has a manually inlined version of `jl_method_compiled` that was missing the `->owner` check. I don't think there's any strong reason to duplicate this code path, so unify them and make sure the `->owner` path is there. This fixes an issue where calling `CompilerDevTools`'s `with_new_compiler` would cause the creation of CodeInstances that would then be dispatched to outside the `with_new_compiler` context. --- src/gf.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/gf.c b/src/gf.c index bc66ebc858e1e..f3d0eeceafaf1 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2651,20 +2651,30 @@ JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, si JL_DLLEXPORT jl_value_t *(*const jl_rettype_inferred_addr)(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT = jl_rettype_inferred_native; -jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) +STATIC_INLINE jl_callptr_t jl_method_compiled_callptr(jl_method_instance_t *mi, size_t world, jl_code_instance_t **codeinst_out) JL_NOTSAFEPOINT { jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { if (codeinst->owner != jl_nothing) continue; if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { - if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL) - return codeinst; + jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); + if (!invoke) + continue; + *codeinst_out = codeinst; + return invoke; } } return NULL; } +jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) JL_NOTSAFEPOINT +{ + jl_code_instance_t *codeinst = NULL; + jl_method_compiled_callptr(mi, world, &codeinst); + return codeinst; +} + jl_mutex_t precomp_statement_out_lock; _Atomic(uint8_t) jl_force_trace_compile_timing_enabled = 0; @@ -3465,17 +3475,11 @@ STATIC_INLINE jl_value_t *verify_type(jl_value_t *v) JL_NOTSAFEPOINT STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc, size_t world) { - // manually inlined copy of jl_method_compiled - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mfunc->cache); - while (codeinst) { - if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { - jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); - if (invoke != NULL) { - jl_value_t *res = invoke(F, args, nargs, codeinst); - return verify_type(res); - } - } - codeinst = jl_atomic_load_relaxed(&codeinst->next); + jl_code_instance_t *codeinst = NULL; + jl_callptr_t invoke = jl_method_compiled_callptr(mfunc, world, &codeinst); + if (invoke) { + jl_value_t *res = invoke(F, args, nargs, codeinst); + return verify_type(res); } int64_t last_alloc = jl_options.malloc_log ? jl_gc_diff_total_bytes() : 0; int last_errno = errno; @@ -3489,7 +3493,7 @@ STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t errno = last_errno; if (jl_options.malloc_log) jl_gc_sync_total_bytes(last_alloc); // discard allocation count from compilation - jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); + invoke = jl_atomic_load_acquire(&codeinst->invoke); jl_value_t *res = invoke(F, args, nargs, codeinst); return verify_type(res); } From b422883e5753f74c95d126fb22fe48e499a88b95 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 11 Apr 2025 02:22:06 -0400 Subject: [PATCH 070/662] Move inlinability determination into cache transform (#57979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently the inlineability determination is in a bit of an odd spot - just after the optimizers while everything is still in IRCode. It seems more sensible to move this code into the cache transformation code, which is the first place that makes an actual decision based on inlineability. If an external AbstractInterpreter does not need to covert to CodeInfo for compilation purposes this also potentially saves that extra conversion. While we're at it, clean up some naming to deconflict it with other uses. --------- Co-authored-by: Cédric Belmant --- .../CompilerDevTools/src/CompilerDevTools.jl | 2 +- Compiler/src/optimize.jl | 98 +++++-------- Compiler/src/ssair/inlining.jl | 2 +- Compiler/src/typeinfer.jl | 131 +++++++++++++++--- Compiler/test/codegen.jl | 1 + Compiler/test/inline.jl | 4 + 6 files changed, 149 insertions(+), 89 deletions(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index e08d08779e097..ddf202f378fb5 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -47,7 +47,7 @@ end function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) opt = result.src::Compiler.OptimizationState - ir = opt.result.ir::Compiler.IRCode + ir = opt.optresult.ir::Compiler.IRCode override = with_new_compiler for inst in ir.stmts stmt = inst[:stmt] diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 3fdd0ee4a7030..4fa894ab0e37a 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -116,11 +116,14 @@ function inline_cost_clamp(x::Int) return convert(InlineCostType, x) end +const SRC_FLAG_DECLARED_INLINE = 0x1 +const SRC_FLAG_DECLARED_NOINLINE = 0x2 + is_declared_inline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 1 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE is_declared_noinline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 2 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE ##################### # OptimizationState # @@ -157,6 +160,7 @@ code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.wor mutable struct OptimizationResult ir::IRCode + inline_flag::UInt8 simplified::Bool # indicates whether the IR was processed with `cfg_simplify!` end @@ -168,7 +172,7 @@ end mutable struct OptimizationState{Interp<:AbstractInterpreter} linfo::MethodInstance src::CodeInfo - result::Union{Nothing, OptimizationResult} + optresult::Union{Nothing, OptimizationResult} stmt_info::Vector{CallInfo} mod::Module sptypes::Vector{VarState} @@ -236,13 +240,29 @@ include("ssair/EscapeAnalysis.jl") include("ssair/passes.jl") include("ssair/irinterp.jl") +function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector) + ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult)) +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType) + src = ir_to_codeinf!(opt, edges) + src.inlining_cost = inlining_cost + src +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector) + src = ir_to_codeinf!(opt) + src.edges = edges + src +end + function ir_to_codeinf!(opt::OptimizationState) - (; linfo, src, result) = opt - if result === nothing + (; linfo, src, optresult) = opt + if optresult === nothing return src end - src = ir_to_codeinf!(src, result.ir) - opt.result = nothing + src = ir_to_codeinf!(src, optresult.ir) + opt.optresult = nothing opt.src = src maybe_validate_code(linfo, src, "optimized") return src @@ -485,63 +505,12 @@ end abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = types(src)[s] """ - finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) + finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) -Post-process information derived by Julia-level optimizations for later use. -In particular, this function determines the inlineability of the optimized code. +Called at the end of optimization to store the resulting IR back into the OptimizationState. """ -function finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) - (; src, linfo) = opt - (; def, specTypes) = linfo - - force_noinline = is_declared_noinline(src) - - # compute inlining and other related optimizations - result = caller.result - @assert !(result isa LimitedAccuracy) - result = widenslotwrapper(result) - - opt.result = OptimizationResult(ir, false) - - # determine and cache inlineability - if !force_noinline - sig = unwrap_unionall(specTypes) - if !(isa(sig, DataType) && sig.name === Tuple.name) - force_noinline = true - end - if !is_declared_inline(src) && result === Bottom - force_noinline = true - end - end - if force_noinline - set_inlineable!(src, false) - elseif isa(def, Method) - if is_declared_inline(src) && isdispatchtuple(specTypes) - # obey @inline declaration if a dispatch barrier would not help - set_inlineable!(src, true) - else - # compute the cost (size) of inlining this code - params = OptimizationParams(interp) - cost_threshold = default = params.inline_cost_threshold - if ⊑(optimizer_lattice(interp), result, Tuple) && !isconcretetype(widenconst(result)) - cost_threshold += params.inline_tupleret_bonus - end - # if the method is declared as `@inline`, increase the cost threshold 20x - if is_declared_inline(src) - cost_threshold += 19*default - end - # a few functions get special treatment - if def.module === _topmod(def.module) - name = def.name - if name === :iterate || name === :unsafe_convert || name === :cconvert - cost_threshold += 4*default - end - end - src.inlining_cost = inline_cost(ir, params, cost_threshold) - end - end +function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) + opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false) return nothing end @@ -1015,7 +984,8 @@ end function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) @timeit "optimizer" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) - return finish(interp, opt, ir, caller) + finishopt!(interp, opt, ir) + return nothing end macro pass(name, expr) @@ -1459,7 +1429,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod return thiscost end -function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Int) +function inline_cost_model(ir::IRCode, params::OptimizationParams, cost_threshold::Int) bodycost = 0 for i = 1:length(ir.stmts) stmt = ir[SSAValue(i)][:stmt] diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 7b4ab34200764..59eb08d8faed9 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -976,7 +976,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, ir::IRCode, preserve_local return ir, spec_info, DebugInfo(ir.debuginfo, length(ir.stmts)) end function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, preserve_local_sources::Bool) - result = opt.result + result = opt.optresult if result !== nothing !result.simplified && simplify_ir!(result) return retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 5bdd57938e1fb..310e51e8da136 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -104,7 +104,10 @@ end function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) - if isdefined(result, :ci) + if caller.cache_mode === CACHE_MODE_LOCAL + @assert !isdefined(result, :ci) + result.src = transform_result_for_local_cache(interp, result) + elseif isdefined(result, :ci) edges = result_edges(interp, caller) ci = result.ci # if we aren't cached, we don't need this edge @@ -115,11 +118,16 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation store_backedges(ci, edges) end inferred_result = nothing - uncompressed = inferred_result + uncompressed = result.src const_flag = is_result_constabi_eligible(result) + debuginfo = get_debuginfo(result.src) discard_src = caller.cache_mode === CACHE_MODE_NULL || const_flag if !discard_src inferred_result = transform_result_for_cache(interp, result, edges) + if inferred_result !== nothing + uncompressed = inferred_result + debuginfo = get_debuginfo(inferred_result) + end # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? if inferred_result isa CodeInfo result.src = inferred_result @@ -128,8 +136,6 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation resize!(inferred_result.slottypes::Vector{Any}, nslots) resize!(inferred_result.slotnames, nslots) end - di = inferred_result.debuginfo - uncompressed = inferred_result inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing @@ -137,18 +143,21 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation inferred_result = nothing end end - if !@isdefined di - di = DebugInfo(result.linfo) + if debuginfo === nothing + debuginfo = DebugInfo(result.linfo) end time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), - result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, di, edges) + result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) engine_reject(interp, ci) codegen = codegen_cache(interp) - if !discard_src && codegen !== nothing && uncompressed isa CodeInfo + if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) + if isa(uncompressed, OptimizationState) + uncompressed = ir_to_codeinf!(uncompressed, edges) + end # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work codegen[ci] = uncompressed if bootstrapping_compiler && inferred_result == nothing @@ -299,36 +308,113 @@ function adjust_cycle_frame!(sv::InferenceState, cycle_valid_worlds::WorldRange, return nothing end +function get_debuginfo(src) + isa(src, CodeInfo) && return src.debuginfo + isa(src, OptimizationState) && return src.src.debuginfo + return nothing +end + function is_result_constabi_eligible(result::InferenceResult) result_type = result.result return isa(result_type, Const) && is_foldable_nothrow(result.ipo_effects) && is_inlineable_constant(result_type.val) end -function transform_result_for_cache(::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult) + src = result.src + isa(src, OptimizationState) || return MAX_INLINE_COST + compute_inlining_cost(interp, result, src.optresult) +end + +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult, optresult#=::OptimizationResult=#) + return inline_cost_model(interp, result, optresult.inline_flag, optresult.ir) +end + +function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, + inline_flag::UInt8, ir::IRCode) + + inline_flag === SRC_FLAG_DECLARED_NOINLINE && return MAX_INLINE_COST + + mi = result.linfo + (; def, specTypes) = mi + if !isa(def, Method) + return MAX_INLINE_COST + end + + declared_inline = inline_flag === SRC_FLAG_DECLARED_INLINE + + rt = result.result + @assert !(rt isa LimitedAccuracy) + rt = widenslotwrapper(rt) + + sig = unwrap_unionall(specTypes) + if !(isa(sig, DataType) && sig.name === Tuple.name) + return MAX_INLINE_COST + end + if !declared_inline && rt === Bottom + return MAX_INLINE_COST + end + + if declared_inline && isdispatchtuple(specTypes) + # obey @inline declaration if a dispatch barrier would not help + return MIN_INLINE_COST + else + # compute the cost (size) of inlining this code + params = OptimizationParams(interp) + cost_threshold = default = params.inline_cost_threshold + if ⊑(optimizer_lattice(interp), rt, Tuple) && !isconcretetype(widenconst(rt)) + cost_threshold += params.inline_tupleret_bonus + end + # if the method is declared as `@inline`, increase the cost threshold 20x + if declared_inline + cost_threshold += 19*default + end + # a few functions get special treatment + if def.module === _topmod(def.module) + name = def.name + if name === :iterate || name === :unsafe_convert || name === :cconvert + cost_threshold += 4*default + end + end + return inline_cost_model(ir, params, cost_threshold) + end +end + +function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) src = result.src if isa(src, OptimizationState) - src = ir_to_codeinf!(src) + # Compute and store any information required to determine the inlineability of the callee. + opt = src + opt.src.inlining_cost = compute_inlining_cost(interp, result) + end + return src +end + +function transform_result_for_cache(interp::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) + inlining_cost = nothing + src = result.src + if isa(src, OptimizationState) + opt = src + inlining_cost = compute_inlining_cost(interp, result, opt.optresult) + discard_optimized_result(interp, opt, inlining_cost) && return nothing + src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) src.edges = edges + src.inlining_cost = inlining_cost !== nothing ? inlining_cost : compute_inlining_cost(interp, result) end return src end +function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) + may_discard_trees(interp) || return false + return inlining_cost == MAX_INLINE_COST +end + function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo) def = mi.def isa(def, Method) || return ci # don't compress toplevel code - can_discard_trees = may_discard_trees(interp) - cache_the_tree = !can_discard_trees || is_inlineable(ci) - if cache_the_tree - if may_compress(interp) - return ccall(:jl_compress_ir, String, (Any, Any), def, ci) - else - return ci - end - else - return nothing - end + may_compress(interp) && return ccall(:jl_compress_ir, String, (Any, Any), def, ci) + return ci end function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) @@ -1101,8 +1187,7 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - src = ir_to_codeinf!(opt) - src.edges = Core.svec(opt.inlining.edges...) + src = ir_to_codeinf!(opt, frame, Core.svec(opt.inlining.edges...)) end result.src = frame.src = src end diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index fd8bbae70a346..45db9e73d5a3f 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -4,6 +4,7 @@ using Random using InteractiveUtils +using InteractiveUtils: code_llvm, code_native using Libdl using Test diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index 3160d4ca74e6a..bb0a2602a0c23 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +module inline_tests + using Test using Base.Meta using Core: ReturnNode @@ -2311,3 +2313,5 @@ g_noinline_invoke(x) = f_noinline_invoke(x) let src = code_typed1(g_noinline_invoke, (Union{Symbol,Nothing},)) @test !any(@nospecialize(x)->isa(x,GlobalRef), src.code) end + +end # module inline_tests From 88f219e058162cd09a2f4958c29e1b67b742a858 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 11 Apr 2025 02:22:42 -0400 Subject: [PATCH 071/662] fix syntax for `true` and `false` as symbols (#58071) `isidentifier` already handles these, so this makes no observable difference but I noticed that `keyword_syms` was unexpectedly a `Set{Any}`. --- base/show.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/show.jl b/base/show.jl index 34a08393c9853..879916d794f96 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1782,8 +1782,8 @@ end const keyword_syms = Set([ :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif, - :end, :export, :false, :finally, :for, :function, :global, :if, :import, - :let, :local, :macro, :module, :public, :quote, :return, :struct, :true, + :end, :export, :var"false", :finally, :for, :function, :global, :if, :import, + :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true", :try, :using, :while ]) function is_valid_identifier(sym) From bd193e4a8b11af93be79591072e26d0242e569e1 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 11 Apr 2025 22:01:38 -0400 Subject: [PATCH 072/662] Drop sources for constabi results in local cache (#58082) Restores dropping these after #57979 --- Compiler/src/typeinfer.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 310e51e8da136..365c651182784 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -380,6 +380,9 @@ function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, end function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) + if is_result_constabi_eligible(result) + return nothing + end src = result.src if isa(src, OptimizationState) # Compute and store any information required to determine the inlineability of the callee. From a690758d458b46368d4b5720502215fe04864056 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Sat, 12 Apr 2025 12:23:56 +0530 Subject: [PATCH 073/662] Eagerly evaluate indices in `eachindex` check (#58054) This reduces TTFX in `eachindex` calls with multiple arguments, with minimal impact on performance. Much of the latency comes from the error path, and specializing it for the common case of 2 arguments helps a lot with reducing latency. In this, I've also unrolled the `join` in the error path, and we recursively generate a `LazyString`s instead. This helps in reducing TTFX for a longer list of arguments. ```julia julia> a = zeros(2,2); julia> @time eachindex(a, a); 0.046902 seconds (128.39 k allocations: 6.652 MiB, 99.93% compilation time) # nightly 0.015368 seconds (19.91 k allocations: 1.048 MiB, 99.79% compilation time) # this PR julia> @btime eachindex($a, $a, $a, $a, $a, $a, $a, $a); 6.945 ns (0 allocations: 0 bytes) # nightly 6.855 ns (0 allocations: 0 bytes) # this PR ``` This reduces TTFX for a longer list of arguments as well: ```julia julia> @time eachindex(a, a, a, a, a, a, a, a); 0.052552 seconds (196.87 k allocations: 10.068 MiB, 99.53% compilation time) # nightly 0.043401 seconds (69.13 k allocations: 3.454 MiB, 99.34% compilation time) # this PR ``` For Cartesian indexing, ```julia julia> a = zeros(2,2); julia> v = view(a, 1:2, 1:2); julia> @time eachindex(a, v); 0.051333 seconds (171.34 k allocations: 8.921 MiB, 99.94% compilation time) # nightly 0.016340 seconds (26.95 k allocations: 1.405 MiB, 99.79% compilation time) # this PR julia> @btime eachindex($a, $v, $a, $v, $a, $v, $a, $v); 9.339 ns (0 allocations: 0 bytes) # nightly 9.357 ns (0 allocations: 0 bytes) # this PR ``` --- base/abstractarray.jl | 22 ++++++++++------------ base/multidimensional.jl | 4 +++- base/reinterpretarray.jl | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 18fa1a79534a1..8389896ba034a 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -321,11 +321,13 @@ eachindex(itrs...) = keys(itrs...) eachindex(A::AbstractVector) = (@inline(); axes1(A)) -@noinline function throw_eachindex_mismatch_indices(::IndexLinear, inds...) - throw(DimensionMismatch("all inputs to eachindex must have the same indices, got $(join(inds, ", ", " and "))")) -end -@noinline function throw_eachindex_mismatch_indices(::IndexCartesian, inds...) - throw(DimensionMismatch("all inputs to eachindex must have the same axes, got $(join(inds, ", ", " and "))")) +# we unroll the join for easier inference +_join_comma_and(indsA, indsB) = LazyString(indsA, " and ", indsB) +_join_comma_and(indsA, indsB, indsC...) = LazyString(indsA, ", ", _join_comma_and(indsB, indsC...)) +@noinline function throw_eachindex_mismatch_indices(indices_str, indsA, indsBs...) + throw(DimensionMismatch( + LazyString("all inputs to eachindex must have the same ", indices_str, ", got ", + _join_comma_and(indsA, indsBs...)))) end """ @@ -390,15 +392,11 @@ eachindex(::IndexLinear, A::AbstractVector) = (@inline; axes1(A)) function eachindex(::IndexLinear, A::AbstractArray, B::AbstractArray...) @inline indsA = eachindex(IndexLinear(), A) - _all_match_first(X->eachindex(IndexLinear(), X), indsA, B...) || - throw_eachindex_mismatch_indices(IndexLinear(), eachindex(A), eachindex.(B)...) + indsBs = map(X -> eachindex(IndexLinear(), X), B) + all(==(indsA), indsBs) || + throw_eachindex_mismatch_indices("indices", indsA, indsBs...) indsA end -function _all_match_first(f::F, inds, A, B...) where F<:Function - @inline - (inds == f(A)) & _all_match_first(f, inds, B...) -end -_all_match_first(f::F, inds) where F<:Function = true # keys with an IndexStyle keys(s::IndexStyle, A::AbstractArray, B::AbstractArray...) = eachindex(s, A, B...) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 5e3fdefbde9ae..1b2dd748bda97 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -416,7 +416,9 @@ module IteratorsMD @inline function eachindex(::IndexCartesian, A::AbstractArray, B::AbstractArray...) axsA = axes(A) - Base._all_match_first(axes, axsA, B...) || Base.throw_eachindex_mismatch_indices(IndexCartesian(), axes(A), axes.(B)...) + axsBs = map(axes, B) + all(==(axsA), axsBs) || + Base.throw_eachindex_mismatch_indices("axes", axsA, axsBs...) CartesianIndices(axsA) end diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 4feb14539f9dc..d7f6a15afa0a7 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -271,7 +271,8 @@ SCartesianIndices2{K}(indices2::AbstractUnitRange{Int}) where {K} = (@assert K:: eachindex(::IndexSCartesian2{K}, A::ReshapedReinterpretArray) where {K} = SCartesianIndices2{K}(eachindex(IndexLinear(), parent(A))) @inline function eachindex(style::IndexSCartesian2{K}, A::AbstractArray, B::AbstractArray...) where {K} iter = eachindex(style, A) - _all_match_first(C->eachindex(style, C), iter, B...) || throw_eachindex_mismatch_indices(IndexSCartesian2{K}(), axes(A), axes.(B)...) + itersBs = map(C->eachindex(style, C), B) + all(==(iter), itersBs) || throw_eachindex_mismatch_indices("axes", axes(A), map(axes, B)...) return iter end From 1aeb1b06493fcd209d3d7860b102d68eb6235c4f Mon Sep 17 00:00:00 2001 From: LiHong Date: Sun, 13 Apr 2025 00:39:50 +0800 Subject: [PATCH 074/662] Fix broken `debug` building (#58088) In commit f5278d80, a `OncePerProcess` variable `private_shlibdir` was added to improve the library path resolving logic. However it only consindered release building and left debug mode broken. This brings it back. Also during the research of getting the building mode at runtime, I found two small similar functions, and it may be better to combine them. --- base/libdl.jl | 5 +++-- base/linking.jl | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base/libdl.jl b/base/libdl.jl index 0b65ae8b381b8..73fd9d9c26871 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -5,7 +5,7 @@ module Libdl Interface to libdl. Provides dynamic linking support. """ Libdl -import Base.DL_LOAD_PATH +import Base: DL_LOAD_PATH, isdebugbuild export DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e, @@ -341,7 +341,8 @@ Base.print(io::IO, llp::LazyLibraryPath) = print(io, string(llp)) # Helper to get `$(private_shlibdir)` at runtime struct PrivateShlibdirGetter; end const private_shlibdir = Base.OncePerProcess{String}() do - dirname(dlpath("libjulia-internal")) + libname = ifelse(isdebugbuild(), "libjulia-internal-debug", "libjulia-internal") + dirname(dlpath(libname)) end Base.string(::PrivateShlibdirGetter) = private_shlibdir() diff --git a/base/linking.jl b/base/linking.jl index f3dbe6abba3ec..a60e27c3051a8 100644 --- a/base/linking.jl +++ b/base/linking.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license module Linking +import Base: isdebugbuild import Base.Libc: Libdl # from LLD_jll @@ -123,7 +124,6 @@ else "-shared" end -is_debug() = ccall(:jl_is_debugbuild, Cint, ()) == 1 libdir() = abspath(Sys.BINDIR, Base.LIBDIR) private_libdir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) if Sys.iswindows() @@ -137,7 +137,7 @@ verbose_linking() = something(Base.get_bool_env("JULIA_VERBOSE_LINKING", false), function link_image_cmd(path, out) PRIVATE_LIBDIR = "-L$(private_libdir())" SHLIBDIR = "-L$(shlibdir())" - LIBS = is_debug() ? ("-ljulia-debug", "-ljulia-internal-debug") : + LIBS = isdebugbuild() ? ("-ljulia-debug", "-ljulia-internal-debug") : ("-ljulia", "-ljulia-internal") @static if Sys.iswindows() LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") From 885d4f4a9f940d6ee0073b1ddc569fb3313a5554 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 12 Apr 2025 15:51:14 -0400 Subject: [PATCH 075/662] Always set `result.src` to the result of transform_result_for_cache (#58083) Inlining can look at `result.src` for things that are put in the global cache via the `VolatileInferenceResult` mechanism. As a result, we need to appropriately update the reference here or inlining will see un-transformed (and thus potentially garbage) objects. The dataflow here is quite confusing and should undergo a larger refactor, but this fixes the immediate issue. --- Compiler/src/typeinfer.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 365c651182784..3eb159cfa1bc0 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -127,10 +127,11 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation if inferred_result !== nothing uncompressed = inferred_result debuginfo = get_debuginfo(inferred_result) + # Inlining may fast-path the global cache via `VolatileInferenceResult`, so store it back here + result.src = inferred_result end # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? if inferred_result isa CodeInfo - result.src = inferred_result if may_compress(interp) nslots = length(inferred_result.slotflags) resize!(inferred_result.slottypes::Vector{Any}, nslots) From 8681bb80627da285b450905fe81652dd9b5243ab Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 13 Apr 2025 00:27:03 +0200 Subject: [PATCH 076/662] verify that `optimize_until` is a valid pass (#58033) I've noticed a few places in tests where `optimize_until` is called with a non-existent pass name. This is an attempt to catch those places and hopefully prevent further possible regressions when someone renames a pass and forgets to update the tests that matches on that name. --------- Co-authored-by: Shuhei Kadowaki --- Compiler/src/optimize.jl | 17 ++++++++++++----- Compiler/test/irpasses.jl | 2 +- Compiler/test/ssair.jl | 3 +++ test/show.jl | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 4fa894ab0e37a..ae247cf699582 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -988,14 +988,16 @@ function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::I return nothing end -macro pass(name, expr) +const ALL_PASS_NAMES = String[] +macro pass(name::String, expr) optimize_until = esc(:optimize_until) stage = esc(:__stage__) - macrocall = :(@timeit $(esc(name)) $(esc(expr))) + macrocall = :(@timeit $name $(esc(expr))) macrocall.args[2] = __source__ # `@timeit` may want to use it + push!(ALL_PASS_NAMES, name) quote $macrocall - matchpass($optimize_until, ($stage += 1), $(esc(name))) && $(esc(:(@goto __done__))) + matchpass($optimize_until, ($stage += 1), $name) && $(esc(:(@goto __done__))) end end @@ -1006,8 +1008,13 @@ matchpass(::Nothing, _, _) = false function run_passes_ipo_safe( ci::CodeInfo, sv::OptimizationState, - optimize_until = nothing, # run all passes by default -) + optimize_until::Union{Nothing, Int, String} = nothing) # run all passes by default + if optimize_until isa String && !contains_is(ALL_PASS_NAMES, optimize_until) + error("invalid `optimize_until` argument, no such optimization pass") + elseif optimize_until isa Int && (optimize_until < 1 || optimize_until > length(ALL_PASS_NAMES)) + error("invalid `optimize_until` argument, no such optimization pass") + end + __stage__ = 0 # used by @pass # NOTE: The pass name MUST be unique for `optimize_until::String` to work @pass "convert" ir = convert_to_ircode(ci, sv) diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index ed697ab738e11..5854f38901d98 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -1459,7 +1459,7 @@ function f_with_early_try_catch_exit() result end -let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="compact"))) +let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="slot2reg"))) for i = 1:length(ir.stmts) expr = ir.stmts[i][:stmt] if isa(expr, PhiCNode) diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 4d94e5b7dc4ca..9c92ed2f62679 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -821,3 +821,6 @@ let cl = Int32[32, 1, 1, 1000, 240, 230] cl2 = ccall(:jl_uncompress_codelocs, Any, (Any, Int), str, 2) @test cl == cl2 end + +@test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = "nonexisting pass name") +@test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = typemax(Int)) diff --git a/test/show.jl b/test/show.jl index 44996a7b630b2..17f0e2bfbf5e3 100644 --- a/test/show.jl +++ b/test/show.jl @@ -2563,7 +2563,7 @@ end mktemp() do f, io redirect_stdout(io) do let io = IOBuffer() - for i = 1:10 + for i = 1:length(Base.Compiler.ALL_PASS_NAMES) # make sure we don't error on printing IRs at any optimization level ir = only(Base.code_ircode(sin, (Float64,); optimize_until=i))[1] @test try; show(io, ir); true; catch; false; end From 1570bed51327161b6d2628e280a6f7728d27db09 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 12 Apr 2025 23:28:53 -0400 Subject: [PATCH 077/662] Align `:method` Expr return value between interpreter and codegen (#58076) Interpreter/inference think the 3-argument `:method` Expr returns `nothing`. Codegen thinks it returns the new method. I think the latter makes more sense, because it lets us write explicit syntax-level dependency links between method definitions and constants (used e.g. for external abstract interpreters), which is something that Revise may need in the future. Adjust the interpreter/inference to properly return the method. --- Compiler/src/abstractinterpretation.jl | 2 +- src/interpreter.c | 7 ++++--- src/julia-syntax.scm | 29 ++++++++++++++++---------- test/core.jl | 5 +++-- test/syntax.jl | 8 +++++++ 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 1c6bfd93935ae..05a857babaa55 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3409,7 +3409,7 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, ssta elseif ehead === :cfunction return abstract_eval_cfunction(interp, e, sstate, sv) elseif ehead === :method - rt = (length(e.args) == 1) ? Any : Nothing + rt = (length(e.args) == 1) ? Any : Method return RTEffects(rt, Any, EFFECTS_UNKNOWN) elseif ehead === :copyast return abstract_eval_copyast(interp, e, sstate, sv) diff --git a/src/interpreter.c b/src/interpreter.c index 7ab284df78dff..2f7f9f8947576 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -106,9 +106,9 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) } atypes = eval_value(args[1], s); meth = eval_value(args[2], s); - jl_method_def((jl_svec_t*)atypes, mt, (jl_code_info_t*)meth, s->module); + jl_method_t *ret = jl_method_def((jl_svec_t*)atypes, mt, (jl_code_info_t*)meth, s->module); JL_GC_POP(); - return jl_nothing; + return (jl_value_t *)ret; } // expression evaluator @@ -626,7 +626,8 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, } else if (toplevel) { if (head == jl_method_sym && jl_expr_nargs(stmt) > 1) { - eval_methoddef((jl_expr_t*)stmt, s); + jl_value_t *res = eval_methoddef((jl_expr_t*)stmt, s); + s->locals[jl_source_nslots(s->src) + s->ip] = res; } else if (head == jl_toplevel_sym) { jl_value_t *res = jl_toplevel_eval(s->module, stmt); diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 0b6f56ac838a6..ae5a4148da88d 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -439,9 +439,11 @@ ,body)))) (if (or (symbol? name) (globalref? name)) `(block ,@generator (method ,name) (latestworld-if-toplevel) ,mdef (unnecessary ,name)) ;; return the function - (if (not (null? generator)) - `(block ,@generator ,mdef) - mdef)))))) + (if (overlay? name) + (if (not (null? generator)) + `(block ,@generator ,mdef) + mdef) + `(block ,@generator ,mdef (null)))))))) ;; wrap expr in nested scopes assigning names to vals (define (scopenest names vals expr) @@ -4127,6 +4129,7 @@ f(x) = yt(x) '() (map-cl-convert (butlast (cdr sig)) fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals))) + (r (make-ssavalue)) (sig (and sig (if (eq? (car sig) 'block) (last sig) sig)))) @@ -4150,7 +4153,7 @@ f(x) = yt(x) ((null? cvs) `(block ,@sp-inits - (method ,(cadr e) ,(cl-convert + (= ,r (method ,(cadr e) ,(cl-convert ;; anonymous functions with keyword args generate global ;; functions that refer to the type of a local function (rename-sig-types sig namemap) @@ -4162,17 +4165,19 @@ f(x) = yt(x) `(lambda ,(cadr lam2) (,(clear-capture-bits (car vis)) ,@(cdr vis)) - ,body))) - (latestworld))) + ,body)))) + (latestworld) + ,r)) (else (let* ((exprs (lift-toplevel (convert-lambda lam2 '|#anon| #t '() #f parsed-method-stack))) (top-stmts (cdr exprs)) (newlam (compact-and-renumber (linearize (car exprs)) 'none 0))) `(toplevel-butfirst (block ,@sp-inits - (method ,(cadr e) ,(cl-convert sig fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals) - ,(julia-bq-macro newlam)) - (latestworld)) + (= ,r (method ,(cadr e) ,(cl-convert sig fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals) + ,(julia-bq-macro newlam))) + (latestworld) + ,r) ,@top-stmts)))) ;; local case - lift to a new type at top level @@ -4999,8 +5004,10 @@ f(x) = yt(x) (let ((l (make-ssavalue))) (emit `(= ,l ,(compile lam break-labels #t #f))) l)))) - (emit `(method ,(or (cadr e) '(false)) ,sig ,lam)) - (if value (compile '(null) break-labels value tail))) + (let ((val (make-ssavalue))) + (emit `(= ,val (method ,(or (cadr e) '(false)) ,sig ,lam))) + (if tail (emit-return tail val)) + val)) (cond (tail (emit-return tail e)) (value e) (else (emit e))))) diff --git a/test/core.jl b/test/core.jl index 4f27c4896da9d..76df797f7484d 100644 --- a/test/core.jl +++ b/test/core.jl @@ -8308,11 +8308,12 @@ end module OverlayModule using Base.Experimental: @MethodTable, @overlay +using Test @MethodTable mt # long function def -@overlay mt function sin(x::Float64) - 1 +let m = @overlay mt function sin(x::Float64); 1; end + @test isa(m, Method) end # short function def @overlay mt cos(x::Float64) = 2 diff --git a/test/syntax.jl b/test/syntax.jl index ec6c2ffb00635..7a7ca63f797da 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4315,3 +4315,11 @@ module DoubleImport import Random end @test DoubleImport.Random === Test.Random + +# Expr(:method) returns the method +let ex = @Meta.lower function return_my_method(); 1; end + code = ex.args[1].code + idx = findfirst(ex->Meta.isexpr(ex, :method) && length(ex.args) > 1, code) + code[end] = Core.ReturnNode(Core.SSAValue(idx)) + @test isa(Core.eval(@__MODULE__, ex), Method) +end From 92cceb09f8870565d88d62259b3e3880ad06a865 Mon Sep 17 00:00:00 2001 From: Monica Kumaran Date: Sat, 12 Apr 2025 23:07:35 -0700 Subject: [PATCH 078/662] DimensionMismatch: Mention mismatched elements in addition to shape (#58091) In response to https://github.com/JuliaLang/julia/issues/58047 Co-authored-by: Monica Kumaran --- base/reshapedarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 9146dd5497078..e831a75dfe3da 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -225,7 +225,7 @@ function _reshape(parent::AbstractArray, dims::Dims) end @noinline function _throw_dmrs(n, str, dims) - throw(DimensionMismatch("parent has $n elements, which is incompatible with $str $dims")) + throw(DimensionMismatch("parent has $n elements, which is incompatible with $str $dims ($(prod(dims)) elements)")) end # Reshaping a ReshapedArray From b265dbac3ef8d1f8f7d779044f458a3c5e2b4689 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 13 Apr 2025 08:08:55 +0200 Subject: [PATCH 079/662] fix zero-dimensional `reverse!` (#58086) Also changed the check to compare against the tuple argument, instead of against the method static parameter for the tuple length. Should be better when the length isn't known at compile time. Fixes #58085 --- base/arraymath.jl | 2 +- test/arrayops.jl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/base/arraymath.jl b/base/arraymath.jl index 62dc3772e4938..53a7d132a2c0c 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -72,12 +72,12 @@ _reverse!(A::AbstractArray{<:Any,N}, ::Colon) where {N} = _reverse!(A, ntuple(id _reverse!(A, dim::Integer) = _reverse!(A, (Int(dim),)) _reverse!(A, dims::NTuple{M,Integer}) where {M} = _reverse!(A, Int.(dims)) function _reverse!(A::AbstractArray{<:Any,N}, dims::NTuple{M,Int}) where {N,M} + dims === () && return A # nothing to reverse dimrev = ntuple(k -> k in dims, Val{N}()) # boolean tuple indicating reversed dims if N < M || M != sum(dimrev) throw(ArgumentError("invalid dimensions $dims in reverse!")) end - M == 0 && return A # nothing to reverse # swapping loop only needs to traverse ≈half of the array halfsz = ntuple(k -> k == dims[1] ? size(A,k) ÷ 2 : size(A,k), Val{N}()) diff --git a/test/arrayops.jl b/test/arrayops.jl index b2da3eac6386b..d5fba79c47017 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1761,6 +1761,12 @@ end end end +@testset "reverse zero dims" begin + a = fill(3) + @test a == reverse(a) + @test a === reverse!(a) +end + @testset "isdiag, istril, istriu" begin # Scalar @test isdiag(3) From 772745beb7d71a68960733b866448d3630b8266f Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Sun, 13 Apr 2025 20:47:50 +0530 Subject: [PATCH 080/662] Forward `to_shape` for `AbstractOneTo` to `Integer` method (#57990) This allows custom `Integer` types to add methods to `to_shape` and have the behavior of `OneTo` be altered accordingly. --- base/abstractarray.jl | 4 +++- test/abstractarray.jl | 2 ++ test/testhelpers/InfiniteArrays.jl | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 8389896ba034a..bc41591ec0cae 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -838,7 +838,9 @@ to_shape(dims::DimsOrInds) = map(to_shape, dims)::DimsOrInds # each dimension to_shape(i::Int) = i to_shape(i::Integer) = Int(i) -to_shape(r::AbstractOneTo) = Int(last(r)) +to_shape(r::AbstractOneTo) = _to_shape(last(r)) +_to_shape(x::Integer) = to_shape(x) +_to_shape(x) = Int(x) to_shape(r::AbstractUnitRange) = r """ diff --git a/test/abstractarray.jl b/test/abstractarray.jl index fc3a893b2250f..e4dcb1959ec18 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2246,5 +2246,7 @@ end b = similar(A, SizedArrays.SOneTo(1), 2, Base.OneTo(2)) @test b isa Array{Int, 3} @test size(b) == (1, 2, 2) + + @test_throws "no method matching $Int(::$Infinity)" similar(ones(2), OneToInf()) end end diff --git a/test/testhelpers/InfiniteArrays.jl b/test/testhelpers/InfiniteArrays.jl index cec3c94aaa296..a5f77356900e9 100644 --- a/test/testhelpers/InfiniteArrays.jl +++ b/test/testhelpers/InfiniteArrays.jl @@ -37,7 +37,7 @@ Define an `AbstractInfUnitRange` that behaves like `1:∞`, with the added distinction that the limits are guaranteed (by the type system) to be 1 and ∞. """ -struct OneToInf{T<:Integer} <: AbstractUnitRange{T} end +struct OneToInf{T<:Integer} <: Base.AbstractOneTo{T} end OneToInf() = OneToInf{Int}() From f3474b5591b6dda800a88d1819d87b0c34edbfb1 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Mon, 14 Apr 2025 13:38:35 -0400 Subject: [PATCH 081/662] remove unnecessary edge from `exp_impl` to `pow` (#58062) This was just intended to be a constant and we have binary Float64 constants --- base/special/exp.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/special/exp.jl b/base/special/exp.jl index 38d7509807aed..f0db1f70a354b 100644 --- a/base/special/exp.jl +++ b/base/special/exp.jl @@ -224,7 +224,7 @@ end twopk = (k + UInt64(53)) << 52 return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 end - #k == 1024 && return (small_part * 2.0) * 2.0^1023 + #k == 1024 && return (small_part * 2.0) * 0x1p1023 end twopk = Int64(k) << 52 return reinterpret(T, twopk + reinterpret(Int64, small_part)) @@ -252,7 +252,7 @@ end twopk = (k + UInt64(53)) << 52 return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 end - k == 1024 && return (small_part * 2.0) * 2.0^1023 + k == 1024 && return (small_part * 2.0) * 0x1p1023 end twopk = Int64(k) << 52 return reinterpret(T, twopk + reinterpret(Int64, small_part)) From 5c6d5de87fe101b39463baecd0af3225b4d6e5e2 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Mon, 14 Apr 2025 19:41:34 +0100 Subject: [PATCH 082/662] Adapt some more code for LLVM 20 (#57793) I missed some deprecations in the previous pr and if we're using jitlink another header file needs to be included. --- src/codegen.cpp | 4 ++++ src/jitlayers.cpp | 3 ++- src/llvm-julia-licm.cpp | 4 ++++ src/llvm-late-gc-lowering.cpp | 24 ++++++++++++++++++++++-- src/llvm-propagate-addrspaces.cpp | 4 ++++ src/llvm-ptls.cpp | 8 ++++++++ test/llvmpasses/gcroots.ll | 2 +- 7 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index af86b568c2b0f..c89fc423bd948 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5823,7 +5823,11 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) unsigned nb = jl_datatype_size(phiType); dest = emit_static_alloca(ctx, nb, align); phi = cast(dest->clone()); +#if JL_LLVM_VERSION >= 200000 + phi->insertBefore(dest->getIterator()); +#else phi->insertBefore(dest); +#endif ctx.builder.CreateMemCpy(phi, align, dest, align, nb, false); ctx.builder.CreateLifetimeEnd(dest); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index bf49b7010b97b..081f3f59acedd 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -15,7 +15,8 @@ #include #include #if JL_LLVM_VERSION >= 200000 -#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h" +#include +#include #endif #if JL_LLVM_VERSION >= 180000 #include diff --git a/src/llvm-julia-licm.cpp b/src/llvm-julia-licm.cpp index 68fe41216bfd4..2fb17f26eb694 100644 --- a/src/llvm-julia-licm.cpp +++ b/src/llvm-julia-licm.cpp @@ -59,7 +59,11 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest, MemorySSAUpdater &MSSAU, ScalarEvolution *SE, MemorySSA::InsertionPlace Place = MemorySSA::BeforeTerminator) { +#if JL_LLVM_VERSION >= 200000 + I.moveBefore(Dest.getIterator()); +#else I.moveBefore(&Dest); +#endif if (MSSAU.getMemorySSA()) if (MemoryUseOrDef *OldMemAcc = cast_or_null( MSSAU.getMemorySSA()->getMemoryAccess(&I))) diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index a41e947c0b6b3..378466ad36ef1 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -2372,7 +2372,11 @@ void LateLowerGCFrame::PlaceGCFrameStore(State &S, unsigned R, unsigned MinColor // free to rewrite them if convenient. We need to change // it back here for the store. assert(Val->getType() == T_prjlvalue); +#if JL_LLVM_VERSION >= 200000 new StoreInst(Val, slotAddress, InsertBefore->getIterator()); +#else + new StoreInst(Val, slotAddress, InsertBefore); +#endif } void LateLowerGCFrame::PlaceGCFrameReset(State &S, unsigned R, unsigned MinColorRoot, @@ -2382,10 +2386,18 @@ void LateLowerGCFrame::PlaceGCFrameReset(State &S, unsigned R, unsigned MinColor auto slotAddress = CallInst::Create( getOrDeclare(jl_intrinsics::getGCFrameSlot), {GCFrame, ConstantInt::get(Type::getInt32Ty(InsertBefore->getContext()), Colors[R] + MinColorRoot)}, +#if JL_LLVM_VERSION >= 200000 + "gc_slot_addr_" + StringRef(std::to_string(Colors[R] + MinColorRoot)), InsertBefore->getIterator()); +#else "gc_slot_addr_" + StringRef(std::to_string(Colors[R] + MinColorRoot)), InsertBefore); +#endif // Reset the slot to NULL. Value *Val = ConstantPointerNull::get(T_prjlvalue); +#if JL_LLVM_VERSION >= 200000 + new StoreInst(Val, slotAddress, InsertBefore->getIterator()); +#else new StoreInst(Val, slotAddress, InsertBefore); +#endif } void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, @@ -2406,7 +2418,7 @@ void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, for (int Idx : *LastLive) { if (Colors[Idx] >= PreAssignedColors && !HasBitSet(NowLive, Idx)) { PlaceGCFrameReset(S, Idx, MinColorRoot, Colors, GCFrame, - S.ReverseSafepointNumbering[*rit]); + S.ReverseSafepointNumbering[*rit]); } } // store values which are alive in this safepoint but @@ -2438,7 +2450,7 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss getOrDeclare(jl_intrinsics::newGCFrame), {ConstantInt::get(T_int32, 0)}, "gcframe"); - gcframe->insertBefore(&*F->getEntryBlock().begin()); + gcframe->insertBefore(F->getEntryBlock().begin()); auto pushGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::pushGCFrame), @@ -2536,7 +2548,11 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss assert(Elem->getType() == T_prjlvalue); //auto Idxs = ArrayRef(Tracked[i]); //Value *Elem = ExtractScalar(Base, true, Idxs, SI); +#if JL_LLVM_VERSION >= 200000 Value *shadowStore = new StoreInst(Elem, slotAddress, SI->getIterator()); +#else + Value *shadowStore = new StoreInst(Elem, slotAddress, SI); +#endif (void)shadowStore; // TODO: shadowStore->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); AllocaSlot++; @@ -2554,7 +2570,11 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss auto popGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::popGCFrame), {gcframe}); +#if JL_LLVM_VERSION >= 200000 + popGcframe->insertBefore(BB.getTerminator()->getIterator()); +#else popGcframe->insertBefore(BB.getTerminator()); +#endif } } } diff --git a/src/llvm-propagate-addrspaces.cpp b/src/llvm-propagate-addrspaces.cpp index 55c9e731e1525..5be8a30e3405e 100644 --- a/src/llvm-propagate-addrspaces.cpp +++ b/src/llvm-propagate-addrspaces.cpp @@ -289,7 +289,11 @@ bool propagateJuliaAddrspaces(Function &F) { PropagateJuliaAddrspacesVisitor visitor; visitor.visit(F); for (auto it : visitor.ToInsert) +#if JL_LLVM_VERSION >= 200000 + it.first->insertBefore(it.second->getIterator()); +#else it.first->insertBefore(it.second); +#endif for (Instruction *I : visitor.ToDelete) I->eraseFromParent(); visitor.ToInsert.clear(); diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 807d3cb95422d..ca1868b85d4e0 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -179,7 +179,11 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter, adoptFunc->copyMetadata(pgcstack_getter, 0); } adopt->setCalledFunction(adoptFunc); +#if JL_LLVM_VERSION >= 200000 + adopt->insertBefore(slowTerm->getIterator()); +#else adopt->insertBefore(slowTerm); +#endif phi->addIncoming(adopt, slowTerm->getParent()); // emit fast branch code builder.SetInsertPoint(fastTerm->getParent()); @@ -237,7 +241,11 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter, builder.SetInsertPoint(pgcstack); auto phi = builder.CreatePHI(T_pppjlvalue, 2, "pgcstack"); pgcstack->replaceAllUsesWith(phi); +#if JL_LLVM_VERSION >= 200000 + pgcstack->moveBefore(slowTerm->getIterator()); +#else pgcstack->moveBefore(slowTerm); +#endif // refresh the basic block in the builder builder.SetInsertPoint(pgcstack); auto getter = builder.CreateLoad(T_pgcstack_getter, pgcstack_func_slot); diff --git a/test/llvmpasses/gcroots.ll b/test/llvmpasses/gcroots.ll index 9f9282cd3c870..d8c1438e4ff63 100644 --- a/test/llvmpasses/gcroots.ll +++ b/test/llvmpasses/gcroots.ll @@ -764,7 +764,7 @@ define i8 @gather_arrayptrs_alltrue() { ; OPAQUE: %gcframe = alloca ptr addrspace(10), i32 3 -; OPAQUE: %arrayptrs = call <2 x ptr addrspace(13)> @llvm.masked.gather.v2p13.v2p11(<2 x ptr addrspace(11)> %arrayptrptrs, i32 16, <2 x i1> , <2 x ptr addrspace(13)> zeroinitializer) +; OPAQUE: %arrayptrs = call <2 x ptr addrspace(13)> @llvm.masked.gather.v2p13.v2p11(<2 x ptr addrspace(11)> %arrayptrptrs, i32 16, <2 x i1> {{(|splat \(i1 true\))}}, <2 x ptr addrspace(13)> zeroinitializer) ; OPAQUE: [[GEP0:%.*]] = getelementptr inbounds ptr addrspace(10), ptr %gcframe, i32 2 ; OPAQUE: store ptr addrspace(10) %obj1, ptr [[GEP0]] ; From cf875c1ff1f8a3679f84e51789654e50630c361f Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 14 Apr 2025 21:47:16 -0400 Subject: [PATCH 083/662] gf.c: Fix backedge de-duplication bug (#58106) This code was checking for the old edge type (`MethodInstance`) instead of the new one (`CodeInstance`), causing duplicate non-invoke edges to accumulate in our `backedges`. --- Compiler/src/typeinfer.jl | 2 ++ Compiler/test/invalidation.jl | 37 +++++++++++++++++++++++++++++++++++ src/gf.c | 2 +- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 3eb159cfa1bc0..1afe4c297e5ba 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -706,6 +706,8 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) # `invoke` edge elseif isa(callee, CodeInstance) callee = get_ci_mi(callee) + else + callee = callee::MethodInstance end ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) i += 2 diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index 29058ffa5cfc6..c0925694d332e 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -162,6 +162,43 @@ begin @test "42" == String(take!(GLOBAL_BUFFER)) end +begin + deduped_callee(x::Int) = @noinline rand(Int) + deduped_caller1(x::Int) = @noinline deduped_callee(x) + deduped_caller2(x::Int) = @noinline deduped_callee(x) + + # run inference on both `deduped_callerx` and `deduped_callee` + let (src, rt) = code_typed((Int,); interp=InvalidationTester()) do x + @inline deduped_caller1(x) + @inline deduped_caller2(x) + end |> only + @test rt === Int + @test any(isinvoke(:deduped_callee), src.code) + end + + # Verify that adding the backedge again does not actually add a new backedge + let mi1 = Base.method_instance(deduped_caller1, (Int,)), + mi2 = Base.method_instance(deduped_caller2, (Int,)), + ci1 = mi1.cache + ci2 = mi2.cache + + callee_mi = Base.method_instance(deduped_callee, (Int,)) + + # Inference should have added the callers to the callee's backedges + @test ci1 in callee_mi.backedges + @test ci2 in callee_mi.backedges + + N = length(callee_mi.backedges) + Core.Compiler.store_backedges(ci1, Core.svec(callee_mi)) + Core.Compiler.store_backedges(ci2, Core.svec(callee_mi)) + N′ = length(callee_mi.backedges) + + # The number of backedges should not be affected by an additional store, + # since de-duplication should have noticed the edge is already tracked + @test N == N′ + end +end + # we can avoid adding backedge even if the callee's return type is not the top # when the return value is not used within the caller begin take!(GLOBAL_BUFFER) diff --git a/src/gf.c b/src/gf.c index f3d0eeceafaf1..6583262798806 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2004,7 +2004,7 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, if (ciedge != (jl_value_t*)caller) continue; jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(backedges, i - 1) : NULL; - if (invokeTypes && jl_is_method_instance(invokeTypes)) + if (invokeTypes && jl_is_code_instance(invokeTypes)) invokeTypes = NULL; if ((invokesig == NULL && invokeTypes == NULL) || (invokesig && invokeTypes && jl_types_equal(invokesig, invokeTypes))) { From d9764d68aed9d7bfe0715afa5ba0229c905fba98 Mon Sep 17 00:00:00 2001 From: Alex Wadell Date: Mon, 14 Apr 2025 22:12:26 -0400 Subject: [PATCH 084/662] Fix `--project=@script` when outside script directory (#56351) --- base/initdefs.jl | 24 +++++++------------ base/options.jl | 1 + src/jloptions.c | 7 ++++-- src/jloptions.h | 1 + test/cmdlineargs.jl | 10 ++++++++ test/project/ScriptProject/Project.toml | 2 ++ .../ScriptProject/SubProject/Project.toml | 2 ++ test/project/ScriptProject/bin/script.jl | 1 + 8 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 test/project/ScriptProject/Project.toml create mode 100644 test/project/ScriptProject/SubProject/Project.toml create mode 100644 test/project/ScriptProject/bin/script.jl diff --git a/base/initdefs.jl b/base/initdefs.jl index ef1b40796f1ad..cc829f1823dd4 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -284,22 +284,14 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing} env == "@temp" && return mktempdir() env == "@stdlib" && return Sys.STDLIB if startswith(env, "@script") - if @isdefined(PROGRAM_FILE) - dir = dirname(PROGRAM_FILE) - else - cmds = unsafe_load_commands(JLOptions().commands) - if any(cmd::Pair{Char, String}->cmd_suppresses_program(first(cmd)), cmds) - # Usage error. The user did not pass a script. - return nothing - end - dir = dirname(ARGS[1]) - end - if env == "@script" # complete match, not startswith, so search upwards - return current_project(dir) - else - # starts with, so assume relative path is after - return abspath(replace(env, "@script" => dir)) - end + program_file = JLOptions().program_file + program_file = program_file != C_NULL ? unsafe_string(program_file) : nothing + isnothing(program_file) && return nothing # User did not pass a script + + # Expand trailing relative path + dir = dirname(program_file) + dir = env != "@script" ? (dir * env[length("@script")+1:end]) : dir + return current_project(dir) end env = replace(env, '#' => VERSION.major, count=1) env = replace(env, '#' => VERSION.minor, count=1) diff --git a/base/options.jl b/base/options.jl index 3281ec0de98d2..818fd33d6f7fc 100644 --- a/base/options.jl +++ b/base/options.jl @@ -17,6 +17,7 @@ struct JLOptions nprocs::Int32 machine_file::Ptr{UInt8} project::Ptr{UInt8} + program_file::Ptr{UInt8} isinteractive::Int8 color::Int8 historyfile::Int8 diff --git a/src/jloptions.c b/src/jloptions.c index 78f6669108229..cd7bc4fdd4de2 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -104,6 +104,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // nprocs NULL, // machine_file NULL, // project + NULL, // program_file 0, // isinteractive 0, // color JL_OPTIONS_HISTORYFILE_ON, // history file @@ -169,11 +170,12 @@ static const char opts[] = " --help-hidden Print uncommon options not shown by `-h`\n\n" // startup options - " --project[={|@temp|@.}] Set as the active project/environment.\n" + " --project[={|@temp|@.|@script[]}] Set as the active project/environment.\n" " Or, create a temporary environment with `@temp`\n" " The default @. option will search through parent\n" " directories until a Project.toml or JuliaProject.toml\n" - " file is found.\n" + " file is found. @script is similar, but searches up from\n" + " the programfile or a path relative to programfile.\n" " -J, --sysimage Start up with the given system image file\n" " -H, --home Set location of `julia` executable\n" " --startup-file={yes*|no} Load `JULIA_DEPOT_PATH/config/startup.jl`; \n" @@ -1012,6 +1014,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) "This is a bug, please report it.", c); } } + jl_options.program_file = optind < argc ? strdup(argv[optind]) : ""; parsing_args_done: if (!jl_options.use_experimental_features) { if (jl_options.trim != JL_TRIM_NO) diff --git a/src/jloptions.h b/src/jloptions.h index a8cc4a9a9e33d..06e00e9309dba 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -21,6 +21,7 @@ typedef struct { int32_t nprocs; const char *machine_file; const char *project; + const char *program_file; int8_t isinteractive; int8_t color; int8_t historyfile; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 83052e116a748..7934d9c60d54d 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -257,6 +257,16 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test expanded == readchomp(addenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "@foo", "HOME" => homedir())) end + # --project=@script handling + let expanded = abspath(joinpath(@__DIR__, "project", "ScriptProject")) + script = joinpath(expanded, "bin", "script.jl") + # Check running julia with --project=@script both within and outside the script directory + @testset "--@script from $name" for (name, dir) in [("project", expanded), ("outside", pwd())] + @test joinpath(expanded, "Project.toml") == readchomp(Cmd(`$exename --project=@script $script`; dir)) + @test joinpath(expanded, "SubProject", "Project.toml") == readchomp(Cmd(`$exename --project=@script/../SubProject $script`; dir)) + end + end + # handling of `@temp` in --project and JULIA_PROJECT @test tempdir() == readchomp(`$exename --project=@temp -e 'println(Base.active_project())'`)[1:lastindex(tempdir())] @test tempdir() == readchomp(addenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "@temp", "HOME" => homedir()))[1:lastindex(tempdir())] diff --git a/test/project/ScriptProject/Project.toml b/test/project/ScriptProject/Project.toml new file mode 100644 index 0000000000000..3301f2b79da83 --- /dev/null +++ b/test/project/ScriptProject/Project.toml @@ -0,0 +1,2 @@ +name = "ScriptProject" +uuid = "6646321a-c4de-46ad-9761-435e5bb1f223" diff --git a/test/project/ScriptProject/SubProject/Project.toml b/test/project/ScriptProject/SubProject/Project.toml new file mode 100644 index 0000000000000..e6c472c7a33f6 --- /dev/null +++ b/test/project/ScriptProject/SubProject/Project.toml @@ -0,0 +1,2 @@ +name = "SubProject" +uuid = "50d58d6a-5ae2-46f7-9677-83c51ca667d5" diff --git a/test/project/ScriptProject/bin/script.jl b/test/project/ScriptProject/bin/script.jl new file mode 100644 index 0000000000000..e38351c9ab9a8 --- /dev/null +++ b/test/project/ScriptProject/bin/script.jl @@ -0,0 +1 @@ +println(Base.active_project()) From 2e8d0532761b90b2cbc6c234d0fd890c1d91b42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Tue, 15 Apr 2025 03:36:17 -0600 Subject: [PATCH 085/662] Fix some typos in comments and tests (#58118) --- Compiler/test/irpasses.jl | 2 +- base/deprecated.jl | 2 +- base/invalidation.jl | 2 +- base/strings/string.jl | 4 ++-- src/task.c | 2 +- test/core.jl | 2 +- test/file.jl | 4 ++-- test/threads_exec.jl | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 5854f38901d98..526c322ec6573 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -1181,7 +1181,7 @@ end @test Compiler.is_effect_free(Base.infer_effects(getfield, (Complex{Int}, Symbol))) -# We consider a potential deprecatio warning an effect, so for completely unkown getglobal, +# We consider a potential deprecatio warning an effect, so for completely unknown getglobal, # we taint the effect_free bit. @test !Compiler.is_effect_free(Base.infer_effects(getglobal, (Module, Symbol))) diff --git a/base/deprecated.jl b/base/deprecated.jl index eeb7c0e60638e..c5701adf1a420 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -87,7 +87,7 @@ else end ``` -To find out the correct verrsion to use as the first argument, you may use +To find out the correct version to use as the first argument, you may use `Base.__next_removal_version`, which is set to the next version number in which the list of changes will be cleared. diff --git a/base/invalidation.jl b/base/invalidation.jl index e974bcd226de8..bb312931a567b 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -210,7 +210,7 @@ function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method b = convert(Core.Binding, gr) if binding_was_invalidated(b) # TODO: We could turn this into an addition if condition. For now, use it as a reasonably cheap - # additional consistency chekc + # additional consistency check @assert !image_backedges_only push!(methods_with_invalidated_source, method) end diff --git a/base/strings/string.jl b/base/strings/string.jl index c4e3edcdf0a6a..c2aa95690e99d 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -226,7 +226,7 @@ end end)(s, i, n, l) end -## checking UTF-8 & ACSII validity ## +## checking UTF-8 & ASCII validity ## #= The UTF-8 Validation is performed by a shift based DFA. ┌───────────────────────────────────────────────────────────────────┐ @@ -374,7 +374,7 @@ end ## -# Classifcations of string +# Classifications of string # 0: neither valid ASCII nor UTF-8 # 1: valid ASCII # 2: valid UTF-8 diff --git a/src/task.c b/src/task.c index 068689d534a03..ae36c98066d01 100644 --- a/src/task.c +++ b/src/task.c @@ -993,7 +993,7 @@ the xoshiro256 state. There are two problems with that fix, however: need four variations of the internal RNG stream for the four xoshiro256 registers. That means we'd have to apply the PCG finalizer, add it to our dot product accumulator field in the child task, then apply the - MurmurHash3 finalizer to that dot product and use the result to purturb + MurmurHash3 finalizer to that dot product and use the result to perturb the main RNG state. We avoid both problems by recognizing that the mixing function can be much less diff --git a/test/core.jl b/test/core.jl index 76df797f7484d..ea4c13debd7b7 100644 --- a/test/core.jl +++ b/test/core.jl @@ -7340,7 +7340,7 @@ Array{Int}(undef, bignum, bignum, 0, bignum, bignum) # but also test that it does throw if the axes multiply to a multiple of typemax(UInt) @test_throws ArgumentError Array{Int}(undef, bignum, bignum) @test_throws ArgumentError Array{Int}(undef, 1, bignum, bignum) -# also test that we always throw erros for negative dims even if other dims are 0 or the product is positive +# also test that we always throw errors for negative dims even if other dims are 0 or the product is positive @test_throws ArgumentError Array{Int}(undef, 0, -4, -4) @test_throws ArgumentError Array{Int}(undef, -4, 1, 0) @test_throws ArgumentError Array{Int}(undef, -4, -4, 1) diff --git a/test/file.jl b/test/file.jl index 6425155c82965..9134073511cb2 100644 --- a/test/file.jl +++ b/test/file.jl @@ -471,7 +471,7 @@ if Sys.iswindows() else @test filesize(dir) > 0 end -# We need both: one to check passed time, one to comapare file's mtime() +# We need both: one to check passed time, one to compare file's mtime() nowtime = time_ns() / 1e9 nowwall = time() # Allow 10s skew in addition to the time it took us to actually execute this code @@ -1383,7 +1383,7 @@ if !Sys.iswindows() stat_d_mv = stat(d_mv) # make sure d does not exist anymore @test !ispath(d) - # comare s, with d_mv + # compare s, with d_mv @test isfile(s) == isfile(d_mv) @test islink(s) == islink(d_mv) islink(s) && @test readlink(s) == readlink(d_mv) diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 629f474f53a38..0f54a4c6d9ee5 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -898,7 +898,7 @@ function _atthreads_greedy_dynamic_schedule() end @test _atthreads_greedy_dynamic_schedule() == threadpoolsize(:default) * threadpoolsize(:default) -function _atthreads_dymamic_greedy_schedule() +function _atthreads_dynamic_greedy_schedule() inc = Threads.Atomic{Int}(0) Threads.@threads :dynamic for _ = 1:threadpoolsize(:default) Threads.@threads :greedy for _ = 1:threadpoolsize(:default) @@ -907,7 +907,7 @@ function _atthreads_dymamic_greedy_schedule() end return inc[] end -@test _atthreads_dymamic_greedy_schedule() == threadpoolsize(:default) * threadpoolsize(:default) +@test _atthreads_dynamic_greedy_schedule() == threadpoolsize(:default) * threadpoolsize(:default) function _atthreads_static_greedy_schedule() ids = zeros(Int, threadpoolsize(:default)) From cca5ac75406b7e728a5ff3802b76fab16975ac2a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 15 Apr 2025 13:10:00 -0400 Subject: [PATCH 086/662] prevent allocation of Memory (layout and object) when not concrete (#58064) Fix #54969 --- src/datatype.c | 8 +++++++- src/julia.h | 2 +- test/core.jl | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index f2f633adc3623..1fd7cf4b502f9 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -500,7 +500,13 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) jl_value_t *kind = jl_tparam0(st); jl_value_t *eltype = jl_tparam1(st); jl_value_t *addrspace = jl_tparam2(st); - if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) { + if (!st->isconcretetype) { + // Since parent dt has an opaque layout, we may end up here being asked to copy that layout to subtypes, + // but we don't actually want to do that unless this object is constructable (or at least has a layout). + // The real layout is stored only on the wrapper. + return; + } + if (!jl_is_type(eltype)) { // this is expected to have a layout, but since it is not constructable, we don't care too much what it is static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), {0}}; st->layout = &opaque_ptr_layout; diff --git a/src/julia.h b/src/julia.h index e5e3428466ba1..21af97deaf984 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1491,7 +1491,7 @@ JL_DLLEXPORT jl_value_t *jl_unwrap_unionall(jl_value_t *v JL_PROPAGATES_ROOT) JL #define jl_inlinedatatype_layout(t) (((jl_datatype_t*)t)->layout) STATIC_INLINE const jl_datatype_layout_t *jl_datatype_layout(jl_datatype_t *t) JL_NOTSAFEPOINT { - if (jl_is_layout_opaque(t->layout)) // e.g. GenericMemory + if (t->layout == NULL || jl_is_layout_opaque(t->layout)) // e.g. GenericMemory t = (jl_datatype_t*)jl_unwrap_unionall(t->name->wrapper); return t->layout; } diff --git a/test/core.jl b/test/core.jl index ea4c13debd7b7..8f4696346aef4 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4951,6 +4951,9 @@ let ft = Base.datatype_fieldtypes @test !isdefined(ft(B12238.body.body)[1], :instance) # has free type vars end +# issue #54969 +@test !isdefined(Memory.body, :instance) + # `where` syntax in constructor definitions (A12238{T} where T<:Real)(x) = 0 @test A12238{<:Real}(0) == 0 From 5fedf470f04f0ac04dbf346b822ca1edaf189f42 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 15 Apr 2025 13:10:18 -0400 Subject: [PATCH 087/662] much faster code-coverage for packages (#57988) We already have an excellent framework for selective code reuse, so use that tool instead of a sledgehammer for inserting selective coverage and malloc instrumentation. As a small bonus, this should also be significantly more accurate by not being vulnerable to precompilation inserting incorrect (uninstrumented) contents into the caches. --- Compiler/src/inferencestate.jl | 24 +++++++------ Compiler/src/utilities.jl | 13 ++++++- base/loading.jl | 47 ++---------------------- base/staticdata.jl | 65 +++++++++++++++++++++++++++++----- 4 files changed, 85 insertions(+), 64 deletions(-) diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index 659b867128177..dc9430169ab82 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -575,21 +575,23 @@ function (::ComputeTryCatch{Handler})(code::Vector{Any}, bbs::Union{Vector{Basic end # check if coverage mode is enabled -function should_insert_coverage(mod::Module, debuginfo::DebugInfo) - coverage_enabled(mod) && return true - JLOptions().code_coverage == 3 || return false +should_insert_coverage(mod::Module, debuginfo::DebugInfo) = should_instrument(mod, debuginfo, true) + +function should_instrument(mod::Module, debuginfo::DebugInfo, only_if_affects_optimizer::Bool=false) + instrumentation_enabled(mod, only_if_affects_optimizer) && return true + JLOptions().code_coverage == 3 || JLOptions().malloc_log == 3 || return false # path-specific coverage mode: if any line falls in a tracked file enable coverage for all - return _should_insert_coverage(debuginfo) + return _should_instrument(debuginfo) end -_should_insert_coverage(mod::Symbol) = is_file_tracked(mod) -_should_insert_coverage(mod::Method) = _should_insert_coverage(mod.file) -_should_insert_coverage(mod::MethodInstance) = _should_insert_coverage(mod.def) -_should_insert_coverage(mod::Module) = false -function _should_insert_coverage(info::DebugInfo) +_should_instrument(loc::Symbol) = is_file_tracked(loc) +_should_instrument(loc::Method) = _should_instrument(loc.file) +_should_instrument(loc::MethodInstance) = _should_instrument(loc.def) +_should_instrument(loc::Module) = false +function _should_instrument(info::DebugInfo) linetable = info.linetable - linetable === nothing || (_should_insert_coverage(linetable) && return true) - _should_insert_coverage(info.def) && return true + linetable === nothing || (_should_instrument(linetable) && return true) + _should_instrument(info.def) && return true return false end diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index 17a9073d709af..e52943d2c960e 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -329,7 +329,7 @@ end inlining_enabled() = (JLOptions().can_inline == 1) -function coverage_enabled(m::Module) +function instrumentation_enabled(m::Module, only_if_affects_optimizer::Bool) generating_output() && return false # don't alter caches cov = JLOptions().code_coverage if cov == 1 # user @@ -340,6 +340,17 @@ function coverage_enabled(m::Module) elseif cov == 2 # all return true end + if !only_if_affects_optimizer + log = JLOptions().malloc_log + if log == 1 # user + m = moduleroot(m) + m === Core && return false + isdefined(Main, :Base) && m === Main.Base && return false + return true + elseif log == 2 # all + return true + end + end return false end diff --git a/base/loading.jl b/base/loading.jl index c7397df8f4515..b26a063247169 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1243,21 +1243,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0) # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}, ignore_native::Union{Nothing,Bool}=nothing; register::Bool=true) - if isnothing(ignore_native) - if JLOptions().code_coverage == 0 && JLOptions().malloc_log == 0 - ignore_native = false - else - io = open(path, "r") - try - iszero(isvalid_cache_header(io)) && return ArgumentError("Incompatible header in cache file $path.") - _, (includes, _, _), _, _, _, _, _, _ = parse_cache_header(io, path) - ignore_native = pkg_tracked(includes) - finally - close(io) - end - end - end +function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}; register::Bool=true) assert_havelock(require_lock) timing_imports = TIMING_IMPORTS[] > 0 try @@ -1276,6 +1262,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No depmods[i] = dep end + ignore_native = false unlock(require_lock) # temporarily _unlock_ during these operations sv = try if ocachepath !== nothing @@ -1949,44 +1936,16 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) return ErrorException("Required dependency $modkey failed to load from a cache file.") end -# returns whether the package is tracked in coverage or malloc tracking based on -# JLOptions and includes -function pkg_tracked(includes) - if JLOptions().code_coverage == 0 && JLOptions().malloc_log == 0 - return false - elseif JLOptions().code_coverage == 1 || JLOptions().malloc_log == 1 # user - # Just say true. Pkgimages aren't in Base - return true - elseif JLOptions().code_coverage == 2 || JLOptions().malloc_log == 2 # all - return true - elseif JLOptions().code_coverage == 3 || JLOptions().malloc_log == 3 # tracked path - if JLOptions().tracked_path == C_NULL - return false - else - tracked_path = unsafe_string(JLOptions().tracked_path) - if isempty(tracked_path) - return false - else - return any(includes) do inc - startswith(inc.filename, tracked_path) - end - end - end - end -end - # loads a precompile cache file, ignoring stale_cachefile tests # load all dependent modules first function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames io = open(path, "r") - ignore_native = false try iszero(isvalid_cache_header(io)) && return ArgumentError("Incompatible header in cache file $path.") _, (includes, _, _), depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io, path) - ignore_native = pkg_tracked(includes) pkgimage = !isempty(clone_targets) if pkgimage @@ -2013,7 +1972,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union depmods[i] = dep end # then load the file - loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native; register = true) + loaded = _include_from_serialized(pkg, path, ocachepath, depmods; register = true) return loaded end diff --git a/base/staticdata.jl b/base/staticdata.jl index d51e7892bbe56..741937369b73b 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -2,8 +2,8 @@ module StaticData -using Core: CodeInstance, MethodInstance -using Base: get_world_counter +using .Core: CodeInstance, MethodInstance +using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1 const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) @@ -73,6 +73,51 @@ end get_require_world() = unsafe_load(cglobal(:jl_require_world, UInt)) +function gen_staged_sig(def::Method, mi::MethodInstance) + isdefined(def, :generator) || return nothing + isdispatchtuple(mi.specTypes) || return nothing + gen = Core.Typeof(def.generator) + return Tuple{gen, UInt, Method, Vararg} + ## more precise method lookup, but more costly and likely not actually better? + #tts = (mi.specTypes::DataType).parameters + #sps = Any[Core.Typeof(mi.sparam_vals[i]) for i in 1:length(mi.sparam_vals)] + #if def.isva + # return Tuple{gen, UInt, Method, sps..., tts[1:def.nargs - 1]..., Tuple{tts[def.nargs - 1:end]...}} + #else + # return Tuple{gen, UInt, Method, sps..., tts...} + #end +end + +function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def::Method, validation_world::UInt) + if JLOptions().code_coverage != 0 || JLOptions().malloc_log != 0 + # test if the code needs to run with instrumentation, in which case we cannot use existing generated code + if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower (and less accurate?) + Compiler.should_instrument(def.module, def.debuginfo) : + Compiler.should_instrument(def.module, codeinst.debuginfo) + return true + end + gensig = gen_staged_sig(def, mi) + if gensig !== nothing + # if this is defined by a generator, try to consider forcing re-running the generators too, to add coverage for them + minworld = Ref{UInt}(1) + maxworld = Ref{UInt}(typemax(UInt)) + has_ambig = Ref{Int32}(0) + result = _methods_by_ftype(gensig, nothing, -1, validation_world, #=ambig=#false, minworld, maxworld, has_ambig) + if result !== nothing + for k = 1:length(result) + match = result[k]::Core.MethodMatch + genmethod = match.method + # no, I refuse to refuse to recurse into your cursed generated function generators and will only test one level deep here + if isdefined(genmethod, :debuginfo) && Compiler.should_instrument(genmethod.module, genmethod.debuginfo) + return true + end + end + end + end + end + return false +end + # Test all edges relevant to a method: # - Visit the entire call graph, starting from edges[idx] to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable @@ -84,6 +129,12 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi return 0, world, max_valid2 end end + mi = get_ci_mi(codeinst) + def = mi.def::Method + if needs_instrumentation(codeinst, mi, def, validation_world) + return 0, world, UInt(0) + end + # Implicitly referenced bindings in the current module do not get explicit edges. # If they were invalidated, they'll be in `mwis`. If they weren't, they imply a minworld # of `get_require_world`. In principle, this is only required for methods that do reference @@ -92,8 +143,6 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi # but no implicit edges) is rare and there would be little benefit to lower the minworld for it # in any case, so we just always use `get_require_world` here. local minworld::UInt, maxworld::UInt = get_require_world(), validation_world - def = get_ci_mi(codeinst).def - @assert def isa Method if haskey(visiting, codeinst) return visiting[codeinst], minworld, maxworld end @@ -225,7 +274,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end @atomic :monotonic child.max_world = maxworld if maxworld == validation_world && validation_world == get_world_counter() - Base.Compiler.store_backedges(child, child.edges) + Compiler.store_backedges(child, child.edges) end @assert visiting[child] == length(stack) + 1 delete!(visiting, child) @@ -243,7 +292,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n minworld = Ref{UInt}(1) maxworld = Ref{UInt}(typemax(UInt)) has_ambig = Ref{Int32}(0) - result = Base._methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) + result = _methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) if result === nothing maxworld[] = 0 else @@ -306,11 +355,11 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn else minworld = 1 maxworld = typemax(UInt) - mt = Base.get_methodtable(expected) + mt = get_methodtable(expected) if mt === nothing maxworld = 0 else - matched, valid_worlds = Base.Compiler._findsup(invokesig, mt, world) + matched, valid_worlds = Compiler._findsup(invokesig, mt, world) minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world if matched === nothing maxworld = 0 From ac29e3a6c7b116d5e25e9c3420caf7bc6269f638 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 11 Apr 2025 12:20:27 -0400 Subject: [PATCH 088/662] inference: ensure source is saved for IRInterp when assumed to be available --- Compiler/src/typeinfer.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 1afe4c297e5ba..43b743982ea89 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -399,7 +399,7 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen if isa(src, OptimizationState) opt = src inlining_cost = compute_inlining_cost(interp, result, opt.optresult) - discard_optimized_result(interp, opt, inlining_cost) && return nothing + discard_optimized_result(interp, opt, inlining_cost, result.ipo_effects) && return nothing src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) @@ -409,7 +409,7 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen return src end -function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) +function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#, effects::Effects) may_discard_trees(interp) || return false return inlining_cost == MAX_INLINE_COST end From 7193a6a9bb685d21da16bcb88f8440b3a3dd0392 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 11 Apr 2025 12:21:34 -0400 Subject: [PATCH 089/662] staticdata: prune backedges table during serialization After the bindings change, there is quite a lot of garbage in here now at runtime (it does not de-duplicate entries added for bindings), so attempt to do a bit of that during serialization. --- src/staticdata.c | 118 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index c96faf5a71a54..465861861d905 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -863,40 +863,60 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } - if (s->incremental && jl_is_method_instance(v)) { + if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; - jl_value_t *def = mi->def.value; - if (needs_uniquing(v, s->query_cache)) { - // we only need 3 specific fields of this (the rest are not used) - jl_queue_for_serialization(s, mi->def.value); - jl_queue_for_serialization(s, mi->specTypes); - jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); - goto done_fields; - } - else if (jl_is_method(def) && jl_object_in_image(def)) { - // we only need 3 specific fields of this (the rest are restored afterward, if valid) - // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, - // so must not be present here - record_field_change((jl_value_t**)&mi->backedges, NULL); - record_field_change((jl_value_t**)&mi->cache, NULL); + if (s->incremental) { + jl_value_t *def = mi->def.value; + if (needs_uniquing(v, s->query_cache)) { + // we only need 3 specific fields of this (the rest are not used) + jl_queue_for_serialization(s, mi->def.value); + jl_queue_for_serialization(s, mi->specTypes); + jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); + goto done_fields; + } + else if (jl_is_method(def) && jl_object_in_image(def)) { + // we only need 3 specific fields of this (the rest are restored afterward, if valid) + // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, + // so must not be present here + record_field_change((jl_value_t**)&mi->backedges, NULL); + record_field_change((jl_value_t**)&mi->cache, NULL); + } + else { + assert(!needs_recaching(v, s->query_cache)); + } + // n.b. opaque closures cannot be inspected and relied upon like a + // normal method since they can get improperly introduced by generated + // functions, so if they appeared at all, we will probably serialize + // them wrong and segfault. The jl_code_for_staged function should + // prevent this from happening, so we do not need to detect that user + // error now. } - else { - assert(!needs_recaching(v, s->query_cache)); + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + size_t i = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge((jl_array_t*)backedges, i, &invokeTypes, &caller); + if (invokeTypes) + jl_queue_for_serialization(s, invokeTypes); + } } - // n.b. opaque closures cannot be inspected and relied upon like a - // normal method since they can get improperly introduced by generated - // functions, so if they appeared at all, we will probably serialize - // them wrong and segfault. The jl_code_for_staged function should - // prevent this from happening, so we do not need to detect that user - // error now. - } - if (s->incremental && jl_is_binding(v)) { - if (needs_uniquing(v, s->query_cache)) { - jl_binding_t *b = (jl_binding_t*)v; + } + if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + if (s->incremental && needs_uniquing(v, s->query_cache)) { jl_queue_for_serialization(s, b->globalref->mod); jl_queue_for_serialization(s, b->globalref->name); goto done_fields; } + // don't recurse into backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + } } if (s->incremental && jl_is_globalref(v)) { jl_globalref_t *gr = (jl_globalref_t*)v; @@ -2572,6 +2592,35 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } +static void jl_prune_mi_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge(backedges, i, &invokeTypes, &caller); + if (ptrhash_get(&serialization_order, caller) != HT_NOTFOUND) + ins = set_next_edge(backedges, ins, invokeTypes, caller); + } + jl_array_del_end(backedges, n - ins); +} + +static void jl_prune_binding_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + for (i = 0; i < n; i++) { + jl_value_t *b = jl_array_ptr_ref(backedges, i); + if (ptrhash_get(&serialization_order, b) != HT_NOTFOUND) + jl_array_ptr_set(backedges, ins, b); + } + jl_array_del_end(backedges, n - ins); +} + + uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED @@ -3145,12 +3194,11 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_queue_for_serialization(&s, global_roots_keyset); jl_serialize_reachable(&s); } - // step 1.5: prune (garbage collect) some special weak references from - // built-in type caches too + // step 1.5: prune (garbage collect) some special weak references known caches for (i = 0; i < serialization_queue.len; i++) { jl_value_t *v = (jl_value_t*)serialization_queue.items[i]; if (jl_options.trim) { - if (jl_is_method(v)){ + if (jl_is_method(v)) { jl_method_t *m = (jl_method_t*)v; jl_value_t *specializations_ = jl_atomic_load_relaxed(&m->specializations); if (!jl_is_svec(specializations_)) @@ -3178,6 +3226,16 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); } + else if (jl_is_method_instance(v)) { + jl_method_instance_t *mi = (jl_method_instance_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + jl_prune_mi_backedges((jl_array_t*)backedges); + } + else if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + jl_prune_binding_backedges((jl_array_t*)backedges); + } } } From fb5fffd63f0545d20120bde2a8c54802f6caff02 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 11 Apr 2025 12:42:40 -0400 Subject: [PATCH 090/662] staticdata: stop compressing and storing large IR Compressing IR adds Methods roots, which tend to become quite expensive to store after some time. We also used to store the inferred code, but since we never actually look at that again now (since now the compiler avoids using --output-ji or --sysimage-native-code=no or --pkgimages=no) it is now quite a bit of wasted space also. For some reason, this also generates 20% more native code in the system image, which isn't quite clear if that is good or not. --- src/aotcompile.cpp | 14 -------------- src/gf.c | 20 -------------------- src/julia_internal.h | 1 - src/staticdata.c | 24 +++++++++++++----------- 4 files changed, 13 insertions(+), 46 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index d687f44808409..5cdecda9d8582 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -675,20 +675,6 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[0] = (jl_value_t*)codeinfos; void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, external_linkage); - // examine everything just emitted and save it to the caches - if (!external_linkage) { - for (size_t i = 0, l = jl_array_nrows(codeinfos); i < l; i++) { - jl_value_t *item = jl_array_ptr_ref(codeinfos, i); - if (jl_is_code_instance(item)) { - // now add it to our compilation results - jl_code_instance_t *codeinst = (jl_code_instance_t*)item; - jl_code_info_t *src = (jl_code_info_t*)jl_array_ptr_ref(codeinfos, ++i); - assert(jl_is_code_info(src)); - jl_add_codeinst_to_cache(codeinst, src); - } - } - } - // move everything inside, now that we've merged everything // (before adding the exported headers) ((jl_native_code_desc_t*)data)->M.withModuleDo([&](Module &M) { diff --git a/src/gf.c b/src/gf.c index 6583262798806..1a51f7cd817ec 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2836,30 +2836,10 @@ void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_c jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_PROPAGATES_ROOT); -JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src) -{ - assert(jl_is_code_info(src)); - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - if (jl_generating_output() && jl_is_method(mi->def.method) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { - jl_value_t *compressed = jl_compress_ir(mi->def.method, src); - // These should already be compatible (and should be an assert), but make sure of it anyways - if (jl_is_svec(src->edges)) { - jl_atomic_store_release(&codeinst->edges, (jl_svec_t*)src->edges); - jl_gc_wb(codeinst, src->edges); - } - jl_atomic_store_release(&codeinst->debuginfo, src->debuginfo); - jl_gc_wb(codeinst, src->debuginfo); - jl_atomic_store_release(&codeinst->inferred, compressed); - jl_gc_wb(codeinst, compressed); - } -} - - JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src) { assert(jl_is_code_info(src)); jl_emit_codeinst_to_jit(codeinst, src); - jl_add_codeinst_to_cache(codeinst, src); } jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) diff --git a/src/julia_internal.h b/src/julia_internal.h index 9966d36027473..fd8720b7f1701 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -685,7 +685,6 @@ JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROP JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile); JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src); -JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( diff --git a/src/staticdata.c b/src/staticdata.c index 465861861d905..4c1489770aa7e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -934,18 +934,20 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } } - if (s->incremental && jl_is_code_instance(v)) { + if (jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; jl_method_instance_t *mi = jl_get_ci_mi(ci); - // make sure we don't serialize other reachable cache entries of foreign methods - // Should this now be: - // if (ci !in ci->defs->cache) - // record_field_change((jl_value_t**)&ci->next, NULL); - // Why are we checking that the method/module this originates from is in_image? - // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)mi->def.value)) { - // TODO: if (ci in ci->defs->cache) - record_field_change((jl_value_t**)&ci->next, NULL); + if (s->incremental) { + // make sure we don't serialize other reachable cache entries of foreign methods + // Should this now be: + // if (ci !in ci->defs->cache) + // record_field_change((jl_value_t**)&ci->next, NULL); + // Why are we checking that the method/module this originates from is in_image? + // and then disconnect this CI? + if (jl_object_in_image((jl_value_t*)mi->def.value)) { + // TODO: if (ci in ci->defs->cache) + record_field_change((jl_value_t**)&ci->next, NULL); + } } jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred); if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) @@ -973,7 +975,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (inferred == jl_nothing) { record_field_change((jl_value_t**)&ci->inferred, jl_nothing); } - else if (jl_is_string(inferred)) { + else if (s->incremental && jl_is_string(inferred)) { // New roots for external methods if (jl_object_in_image((jl_value_t*)def)) { void **pfound = ptrhash_bp(&s->method_roots_index, def); From b29c80894ed0bd80c0d1cc98d11405f320289623 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerberg <39104088+nhz2@users.noreply.github.com> Date: Tue, 15 Apr 2025 15:27:50 -0400 Subject: [PATCH 091/662] Fix `IOBuffer` `skip` regression (#57963) Fixes #57962 This reverts the changes to `skip` added in #57570 The code before #57570 was written in a very specific way to avoid overflow errors, so I'm not sure why this was changed. --- base/iobuffer.jl | 4 ++-- test/iobuffer.jl | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 49bc25d10780d..c115def5d9ade 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -486,8 +486,8 @@ function skip(io::GenericIOBuffer, n::Int) else # Don't use seek in order to allow a non-seekable IO to still skip bytes. # Handle overflow. - maxptr = io.size + 1 - io.ptr = n > maxptr || io.ptr - n > maxptr ? maxptr : io.ptr + n + n_max = io.size + 1 - io.ptr + io.ptr += min(n, n_max) io end end diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 7ed5c1f5b3ed6..2b84fe79307ae 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -497,6 +497,13 @@ end end end +@testset "issue #57962" begin + io = IOBuffer(repeat("x", 400)) + skip(io, 10) + skip(io, 400) + @test isempty(read(io)) +end + @testset "pr #11554" begin io = IOBuffer(SubString("***αhelloworldω***", 4, 16)) io2 = IOBuffer(Vector{UInt8}(b"goodnightmoon"), read=true, write=true) From 48660a617318c0907ed0b2a260c4c46672d975ed Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 15 Apr 2025 16:00:59 -0400 Subject: [PATCH 092/662] [DOC] Update installation docs: /downloads/ => /install/ (#58127) I've created an [Install](https://julialang.org/install/) page separate from the [Downloads](https://julialang.org/downloads/) page on the website. This updates various references to point to the correct pages. --- README.md | 29 +++++++++++++++++------------ doc/man/julia.1 | 2 +- doc/src/devdocs/build/windows.md | 2 +- doc/src/index.md | 2 +- doc/src/manual/faq.md | 3 +-- doc/src/manual/getting-started.md | 2 +- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 021322336d286..0ed6ed9117434 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ and installing Julia, below. ## Resources - **Homepage:** -- **Binaries:** +- **Install:** - **Source code:** - **Documentation:** - **Packages:** @@ -63,17 +63,22 @@ helpful to start contributing to the Julia codebase. ## Binary Installation -If you would rather not compile the latest Julia from source, -platform-specific tarballs with pre-compiled binaries are also -[available for download](https://julialang.org/downloads/). The -downloads page also provides details on the -[different tiers of support](https://julialang.org/downloads/#supported_platforms) -for OS and platform combinations. - -If everything works correctly, you will see a Julia banner and an -interactive prompt into which you can enter expressions for -evaluation. You can read about [getting -started](https://docs.julialang.org/en/v1/manual/getting-started/) in the manual. +The recommended way of installing Julia is to use `juliaup` which will install +the latest stable `julia` for you and help keep it up to date. It can also let +you install and run different Julia versions simultaneously. Instructions for +this can be find [here](https://julialang.org/install/). If you want to manually +download specific Julia binaries, you can find those on the [downloads +page](https://julialang.org/downloads/). The downloads page also provides +details on the [different tiers of +support](https://julialang.org/downloads/#supported_platforms) for OS and +platform combinations. + +If everything works correctly, you will get a `julia` program and when you run +it in a terminal or command prompt, you will see a Julia banner and an +interactive prompt into which you can enter expressions for evaluation. You can +read about [getting +started](https://docs.julialang.org/en/v1/manual/getting-started/) in the +manual. **Note**: Although some OS package managers provide Julia, such installations are neither maintained nor endorsed by the Julia diff --git a/doc/man/julia.1 b/doc/man/julia.1 index 2da11ae1b3f18..9646464e1e63d 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -321,7 +321,7 @@ Website: https://julialang.org/ .br Documentation: https://docs.julialang.org/ .br -Downloads: https://julialang.org/downloads/ +Install: https://julialang.org/install/ .SH LICENSING Julia is an open-source project. It is made available under the MIT license. diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index 77042b352047c..c19c7ea1e9949 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -32,7 +32,7 @@ or edit `%USERPROFILE%\.gitconfig` and add/edit the lines: ## Binary distribution For the binary distribution installation notes on Windows please see the instructions at -[https://julialang.org/downloads/platform/#windows](https://julialang.org/downloads/platform/#windows). +[https://julialang.org/downloads/platform/#windows](https://julialang.org/downloads/platform/#windows). Note, however, that on all platforms [using `juliaup`](https://julialang.org/install/) is recommended over manually installing binaries. ## Source distribution diff --git a/doc/src/index.md b/doc/src/index.md index 8c88af424e8e3..8342ff448625d 100644 --- a/doc/src/index.md +++ b/doc/src/index.md @@ -37,7 +37,7 @@ Markdown.parse(""" Below is a non-exhaustive list of links that will be useful as you learn and use the Julia programming language. - [Julia Homepage](https://julialang.org) -- [Download Julia](https://julialang.org/downloads/) +- [Install Julia](https://julialang.org/install/) - [Discussion forum](https://discourse.julialang.org) - [Julia YouTube](https://www.youtube.com/user/JuliaLanguage) - [Find Julia Packages](https://julialang.org/packages/) diff --git a/doc/src/manual/faq.md b/doc/src/manual/faq.md index be4f331cd6233..188b8b7f79f3a 100644 --- a/doc/src/manual/faq.md +++ b/doc/src/manual/faq.md @@ -1090,8 +1090,7 @@ You may wish to test against the nightly version to ensure that such regressions Finally, you may also consider building Julia from source for yourself. This option is mainly for those individuals who are comfortable at the command line, or interested in learning. If this describes you, you may also be interested in reading our [guidelines for contributing](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md). -Links to each of these download types can be found on the download page at [https://julialang.org/downloads/](https://julialang.org/downloads/). -Note that not all versions of Julia are available for all platforms. +The [`juliaup` install manager](https://julialang.org/install/) has pre-defined channels named `release` and `lts` for the latest stable release and the current LTS release, as well as version-specific channels. ### How can I transfer the list of installed packages after updating my version of Julia? diff --git a/doc/src/manual/getting-started.md b/doc/src/manual/getting-started.md index 2c69aabbda192..b0299a4563f98 100644 --- a/doc/src/manual/getting-started.md +++ b/doc/src/manual/getting-started.md @@ -1,7 +1,7 @@ # [Getting Started](@id man-getting-started) Julia installation is straightforward, whether using precompiled binaries or compiling from source. -Download and install Julia by following the instructions at [https://julialang.org/downloads/](https://julialang.org/downloads/). +Download and install Julia by following the instructions at [https://julialang.org/install/](https://julialang.org/install/). If you are coming to Julia from one of the following languages, then you should start by reading the section on noteworthy differences from [MATLAB](@ref Noteworthy-differences-from-MATLAB), [R](@ref Noteworthy-differences-from-R), [Python](@ref Noteworthy-differences-from-Python), [C/C++](@ref Noteworthy-differences-from-C/C) or [Common Lisp](@ref Noteworthy-differences-from-Common-Lisp). This will help you avoid some common pitfalls since Julia differs from those languages in many subtle ways. From 06d2a19815318a68ff402e16da4378a84289ea30 Mon Sep 17 00:00:00 2001 From: Liam Healy Date: Tue, 15 Apr 2025 19:35:26 -0400 Subject: [PATCH 093/662] Document where to put startup.jl if JULIA_DEPOT_PATH is set (#58126) --- doc/src/manual/command-line-interface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 14dd60d89b384..5a4c710faa22f 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -148,7 +148,8 @@ atreplinit() do repl # ... end ``` - +If [`JULIA_DEPOT_PATH`](@ref JULIA_DEPOT_PATH) is set, the startup file should be located there +`$JULIA_DEPOT_PATH/config/startup.jl`. ## [Command-line switches for Julia](@id command-line-interface) From 83524acfee6d1a99d175f0923005a229f9e885e5 Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:56:07 +0200 Subject: [PATCH 094/662] Fix newly introduced typo in README.md (#58135) Sorry for this mini-PR, but just saw the typo and having it on the front page might make an unfortunate impression. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ed6ed9117434..9f1b8c8c3002f 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ helpful to start contributing to the Julia codebase. The recommended way of installing Julia is to use `juliaup` which will install the latest stable `julia` for you and help keep it up to date. It can also let you install and run different Julia versions simultaneously. Instructions for -this can be find [here](https://julialang.org/install/). If you want to manually +this can be found [here](https://julialang.org/install/). If you want to manually download specific Julia binaries, you can find those on the [downloads page](https://julialang.org/downloads/). The downloads page also provides details on the [different tiers of From caa97c91ec998ebfc09c2589a85998c0a0c7bb97 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 16 Apr 2025 08:11:10 -0500 Subject: [PATCH 095/662] Ensure completion of invalidation log (#58137) --- src/gf.c | 5 +++-- test/worlds.jl | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/gf.c b/src/gf.c index 6583262798806..c8de6d03449cd 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2445,13 +2445,14 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) // found that this specialization dispatch got replaced by m // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); // but ignore invoke-type edges - invalidated = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); + int invalidatedmi = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); - if (_jl_debug_method_invalidation && invalidated) { + if (_jl_debug_method_invalidation && invalidatedmi) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); loctag = jl_cstr_to_string("jl_method_table_insert"); jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } + invalidated |= invalidatedmi; } } } diff --git a/test/worlds.jl b/test/worlds.jl index 4520ab257d078..dd98170721b1a 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -420,6 +420,27 @@ ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) "jl_method_table_insert" ] +# logging issue #58080 +f58080(::Integer) = 1 +callsf58080rts(x) = f58080(Base.inferencebarrier(x)::Signed) +invokesf58080s(x) = invoke(f58080, Tuple{Signed}, x) +# compilation +invokesf58080s(1) # invoked callee +callsf58080rts(1) # runtime-dispatched callee +# invalidation +logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1); +f58080(::Int) = 2 +f58080(::Signed) = 4 +ccall(:jl_debug_method_invalidation, Any, (Cint,), 0); +@test logmeths[1].def.name === :callsf58080rts +m58080i = which(f58080, (Int,)) +m58080s = which(f58080, (Signed,)) +idxi = findfirst(==(m58080i), logmeths) +@test logmeths[idxi+1] == "jl_method_table_insert" +@test logmeths[idxi+2].def.name === :invokesf58080s +@test logmeths[end-1] == m58080s +@test logmeths[end] == "jl_method_table_insert" + # issue #50091 -- missing invoke edge affecting nospecialized dispatch module ExceptionUnwrapping @nospecialize From 1faa69853e3a9e9456b27a5f6a954b26400569ca Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Apr 2025 13:26:08 -0400 Subject: [PATCH 096/662] fix static show in some edge cases (#58128) Fix #51870 In a different PR, I was making precompile printing slightly more accurate, and it caused nearly everything to run into this bug and crashed the build, so I needed to fix this now since an upcoming PR will rely heavily on this being corrected. --- src/rtutils.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rtutils.c b/src/rtutils.c index 3283388cb1d9b..5966497ec331c 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -675,8 +675,7 @@ static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT if (globname && dv->name->module) { jl_binding_t *b = jl_get_module_binding(dv->name->module, globname, 0); jl_value_t *bv = jl_get_binding_value_if_latest_resolved_and_const_debug_only(b); - // The `||` makes this function work for both function instances and function types. - if (bv && (bv == v || jl_typeof(bv) == v)) + if (bv && ((jl_value_t*)dv == v ? jl_typeof(bv) == v : bv == v)) return 1; } return 0; From 37a754164ba93b5f6c147760a37396415dd815cc Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 17 Apr 2025 02:11:01 -0300 Subject: [PATCH 097/662] Make trimmed binaries initialize on first call (#58141) --- contrib/juliac.jl | 19 ++----------------- src/jlapi.c | 25 ++++++++++++------------- src/julia_internal.h | 3 +++ src/threading.c | 15 +++++++-------- 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/contrib/juliac.jl b/contrib/juliac.jl index 7087462afc7a1..b110f1d233690 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac.jl @@ -88,8 +88,6 @@ allflags = Base.shell_split(allflags) rpath = get_rpath(; relative = relative_rpath) rpath = Base.shell_split(rpath) tmpdir = mktempdir(cleanup=false) -initsrc_path = joinpath(tmpdir, "init.c") -init_path = joinpath(tmpdir, "init.a") img_path = joinpath(tmpdir, "img.a") bc_path = joinpath(tmpdir, "img-bc.a") @@ -122,19 +120,6 @@ function compile_products(enable_trim::Bool) exit(1) end - # Compile the initialization code - open(initsrc_path, "w") do io - print(io, """ - #include - __attribute__((constructor)) void static_init(void) { - if (jl_is_initialized()) - return; - julia_init(JL_IMAGE_IN_MEMORY); - jl_exception_clear(); - } - """) - end - run(`cc $(cflags) -g -c -o $init_path $initsrc_path`) end function link_products() @@ -150,11 +135,11 @@ function link_products() julia_libs = Base.shell_split(Base.isdebugbuild() ? "-ljulia-debug -ljulia-internal-debug" : "-ljulia -ljulia-internal") try if output_type == "--output-lib" - cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)` + cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` elseif output_type == "--output-sysimage" cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` else - cmd2 = `cc $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)` + cmd2 = `cc $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` end verbose && println("Running: $cmd2") run(cmd2) diff --git a/src/jlapi.c b/src/jlapi.c index 53585555b8a23..6ef0dd17befed 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -88,7 +88,17 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, if (jl_is_initialized()) return; libsupport_init(); - jl_options.julia_bindir = julia_bindir; + if (julia_bindir) { + jl_options.julia_bindir = julia_bindir; + } else { +#ifdef _OS_WINDOWS_ + jl_options.julia_bindir = strdup(jl_get_libdir()); +#else + int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); + if (written < 0) + abort(); // unexpected: memory allocation failed +#endif + } if (image_path != NULL) jl_options.image_file = image_path; else @@ -105,18 +115,7 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, */ JL_DLLEXPORT void jl_init(void) { - char *libbindir = NULL; -#ifdef _OS_WINDOWS_ - libbindir = strdup(jl_get_libdir()); -#else - (void)asprintf(&libbindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); -#endif - if (!libbindir) { - printf("jl_init unable to find libjulia!\n"); - abort(); - } - jl_init_with_image(libbindir, jl_get_default_sysimg_path()); - free(libbindir); + jl_init_with_image(NULL, jl_get_default_sysimg_path()); } static void _jl_exception_clear(jl_task_t *ct) JL_NOTSAFEPOINT diff --git a/src/julia_internal.h b/src/julia_internal.h index fd8720b7f1701..864097d8d22fe 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -3,6 +3,7 @@ #ifndef JL_INTERNAL_H #define JL_INTERNAL_H +#include "dtypes.h" #include "options.h" #include "julia_assert.h" #include "julia_locks.h" @@ -192,6 +193,8 @@ void JL_UV_LOCK(void); extern _Atomic(unsigned) _threadedregion; extern _Atomic(uint16_t) io_loop_tid; +JL_DLLEXPORT void jl_enter_threaded_region(void); +JL_DLLEXPORT void jl_exit_threaded_region(void); int jl_running_under_rr(int recheck) JL_NOTSAFEPOINT; //-------------------------------------------------- diff --git a/src/threading.c b/src/threading.c index 6b51efda9abc1..f01db388f4cc3 100644 --- a/src/threading.c +++ b/src/threading.c @@ -1,11 +1,9 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license - #include #include #include #include #include - #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" @@ -457,15 +455,16 @@ JL_DLLEXPORT jl_gcframe_t **jl_autoinit_and_adopt_thread(void) { if (!jl_is_initialized()) { void *retaddr = __builtin_extract_return_addr(__builtin_return_address(0)); - void *sysimg_handle = jl_find_dynamic_library_by_addr(retaddr, /* throw_err */ 0); - - if (sysimg_handle == NULL) { + void *handle = jl_find_dynamic_library_by_addr(retaddr, 0); + if (handle == NULL) { fprintf(stderr, "error: runtime auto-initialization failed due to bad sysimage lookup\n" " (this should not happen, please file a bug report)\n"); - abort(); + exit(1); } - - assert(0 && "TODO: implement auto-init"); + const char *image_path = jl_pathname_for_handle(handle); + jl_enter_threaded_region(); // This should maybe be behind a lock, but it's harmless if done twice + jl_init_with_image(NULL, image_path); + return &jl_get_current_task()->gcstack; } return jl_adopt_thread(); From 75c44d042a069fec4492be76bcb8d5625b02daee Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 17 Apr 2025 08:31:04 +0200 Subject: [PATCH 098/662] transition `@timeit` in `Core.Compiler` to use tracy instead (#49675) Co-authored-by: Kristoffer Carlsson Co-authored-by: gbaraldi Co-authored-by: Cody Tapscott --- Compiler/src/Compiler.jl | 1 + Compiler/src/inferencestate.jl | 2 +- Compiler/src/optimize.jl | 28 +++++------ Compiler/src/profiling.jl | 33 +++++++++++++ Compiler/src/profiling/ittapi.jl | 18 +++++++ Compiler/src/profiling/tracy.jl | 63 ++++++++++++++++++++++++ Compiler/src/ssair/inlining.jl | 4 +- Compiler/src/ssair/slot2ssa.jl | 8 +-- Compiler/src/utilities.jl | 2 +- Compiler/test/inline.jl | 8 +-- Compiler/test/irpasses.jl | 4 +- Compiler/test/ssair.jl | 14 +++--- src/timing.c | 24 +++++++++ src/timing.h | 4 ++ stdlib/InteractiveUtils/test/runtests.jl | 2 +- 15 files changed, 179 insertions(+), 36 deletions(-) create mode 100644 Compiler/src/profiling.jl create mode 100644 Compiler/src/profiling/ittapi.jl create mode 100644 Compiler/src/profiling/tracy.jl diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 298ddd729ddd0..a046c71beff8f 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -120,6 +120,7 @@ function is_return_type(Core.@nospecialize(f)) return false end +include("profiling.jl") include("sort.jl") # We don't include some.jl, but this definition is still useful. diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index dc9430169ab82..8a8e5354e3d59 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -188,7 +188,7 @@ mutable struct LazyGenericDomtree{IsPostDom} end function get!(x::LazyGenericDomtree{IsPostDom}) where {IsPostDom} isdefined(x, :domtree) && return x.domtree - return @timeit "domtree 2" x.domtree = IsPostDom ? + return @zone "CC: DOMTREE_2" x.domtree = IsPostDom ? construct_postdomtree(x.ir) : construct_domtree(x.ir) end diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index ae247cf699582..fdf97c447559d 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -982,7 +982,7 @@ end # run the optimization work function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) - @timeit "optimizer" ir = run_passes_ipo_safe(opt.src, opt) + @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) finishopt!(interp, opt, ir) return nothing @@ -992,7 +992,7 @@ const ALL_PASS_NAMES = String[] macro pass(name::String, expr) optimize_until = esc(:optimize_until) stage = esc(:__stage__) - macrocall = :(@timeit $name $(esc(expr))) + macrocall = :(@zone $name $(esc(expr))) macrocall.args[2] = __source__ # `@timeit` may want to use it push!(ALL_PASS_NAMES, name) quote @@ -1017,20 +1017,20 @@ function run_passes_ipo_safe( __stage__ = 0 # used by @pass # NOTE: The pass name MUST be unique for `optimize_until::String` to work - @pass "convert" ir = convert_to_ircode(ci, sv) - @pass "slot2reg" ir = slot2reg(ir, ci, sv) + @pass "CC: CONVERT" ir = convert_to_ircode(ci, sv) + @pass "CC: SLOT2REG" ir = slot2reg(ir, ci, sv) # TODO: Domsorting can produce an updated domtree - no need to recompute here - @pass "compact 1" ir = compact!(ir) - @pass "inlining" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) - # @timeit "verify 2" verify_ir(ir) - @pass "compact 2" ir = compact!(ir) - @pass "SROA" ir = sroa_pass!(ir, sv.inlining) - @pass "ADCE" (ir, made_changes) = adce_pass!(ir, sv.inlining) + @pass "CC: COMPACT_1" ir = compact!(ir) + @pass "CC: INLINING" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) + # @zone "CC: VERIFY 2" verify_ir(ir) + @pass "CC: COMPACT_2" ir = compact!(ir) + @pass "CC: SROA" ir = sroa_pass!(ir, sv.inlining) + @pass "CC: ADCE" (ir, made_changes) = adce_pass!(ir, sv.inlining) if made_changes - @pass "compact 3" ir = compact!(ir, true) + @pass "CC: COMPACT_3" ir = compact!(ir, true) end if is_asserts() - @timeit "verify 3" begin + @zone "CC: VERIFY_3" begin verify_ir(ir, true, false, optimizer_lattice(sv.inlining.interp), sv.linfo) verify_linetable(ir.debuginfo, length(ir.stmts)) end @@ -1291,10 +1291,10 @@ end function slot2reg(ir::IRCode, ci::CodeInfo, sv::OptimizationState) # need `ci` for the slot metadata, IR for the code svdef = sv.linfo.def - @timeit "domtree 1" domtree = construct_domtree(ir) + @zone "CC: DOMTREE_1" domtree = construct_domtree(ir) defuse_insts = scan_slot_def_use(Int(ci.nargs), ci, ir.stmts.stmt) 𝕃ₒ = optimizer_lattice(sv.inlining.interp) - @timeit "construct_ssa" ir = construct_ssa!(ci, ir, sv, domtree, defuse_insts, 𝕃ₒ) # consumes `ir` + @zone "CC: CONSTRUCT_SSA" ir = construct_ssa!(ci, ir, sv, domtree, defuse_insts, 𝕃ₒ) # consumes `ir` # NOTE now we have converted `ir` to the SSA form and eliminated slots # let's resize `argtypes` now and remove unnecessary types for the eliminated slots resize!(ir.argtypes, ci.nargs) diff --git a/Compiler/src/profiling.jl b/Compiler/src/profiling.jl new file mode 100644 index 0000000000000..c29644cc902fc --- /dev/null +++ b/Compiler/src/profiling.jl @@ -0,0 +1,33 @@ +const WITH_ITTAPI = ccall(:jl_ittapi_enabled, Cint, ()) != 0 +const WITH_TRACY = ccall(:jl_tracy_enabled, Cint, ()) != 0 + +include("profiling/tracy.jl") +include("profiling/ittapi.jl") + +if WITH_TRACY || WITH_ITTAPI + macro zone(name, ex::Expr) + srcloc = WITH_TRACY && Tracy.tracy_zone_create(name, ex, __source__) + tracy_begin_expr = WITH_TRACY ? :(ctx_tracy = Tracy.tracy_zone_begin($srcloc, true)) : :() + tracy_end_expr = WITH_TRACY ? :(Tracy.tracy_zone_end(ctx_tracy)) : :() + + event = WITH_ITTAPI && ITTAPI.ittapi_zone_create(name, ex, __source__) + ittapi_begin_expr = WITH_ITTAPI ? :(ctx_ittapi = ITTAPI.ittapi_zone_begin($event, true)) : :() + ittapi_end_expr = WITH_ITTAPI ? :(ITTAPI.ittapi_zone_end(ctx_ittapi)) : :() + + return quote + $tracy_begin_expr + $ittapi_begin_expr + $(Expr(:tryfinally, + :($(esc(ex))), + quote + $tracy_end_expr + $ittapi_end_expr + end + )) + end + end +else + macro zone(name::String, ex::Expr) + esc(ex) + end +end diff --git a/Compiler/src/profiling/ittapi.jl b/Compiler/src/profiling/ittapi.jl new file mode 100644 index 0000000000000..84140ef3a6002 --- /dev/null +++ b/Compiler/src/profiling/ittapi.jl @@ -0,0 +1,18 @@ +# Stubs +module ITTAPI + +import ..String, ..Expr, ..LineNumberNode, ..nothing + +function ittapi_zone_create(name::String, ex::Expr, linfo::LineNumberNode) + return nothing +end + +function ittapi_zone_begin(loc, active) + return nothing +end + +function ittapi_zone_end(ctx) + return nothing +end + +end diff --git a/Compiler/src/profiling/tracy.jl b/Compiler/src/profiling/tracy.jl new file mode 100644 index 0000000000000..5b1b7195b0f50 --- /dev/null +++ b/Compiler/src/profiling/tracy.jl @@ -0,0 +1,63 @@ +module Tracy + +import ..@noinline, ..@atomic, ..Cint, ..Vector, ..push!, ..unsafe_convert, ..esc, + ..pointer_from_objref, ..String, ..Ptr, ..UInt8, ..Cvoid, + ..Expr, ..LineNumberNode, ..Symbol, ..UInt32, ..C_NULL, ..(===) + +_strpointer(s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) + +mutable struct TracySrcLoc + @atomic zone_name::Ptr{UInt8} + @atomic function_name::Ptr{UInt8} + @atomic file::Ptr{UInt8} + const line::UInt32 + const color::UInt32 + # Roots + const zone_name_str::String + const function_name_sym::Symbol + const file_sym::Symbol +end +TracySrcLoc(zone_name::String, function_name::Symbol, file::Symbol, line::UInt32, color::UInt32) = + TracySrcLoc(C_NULL, C_NULL, C_NULL, line, color, zone_name, function_name, file) + +@noinline function reinit!(srcloc::TracySrcLoc) + @atomic :monotonic srcloc.file = unsafe_convert(Ptr{UInt8}, srcloc.file_sym) + @atomic :monotonic srcloc.function_name = unsafe_convert(Ptr{UInt8}, srcloc.function_name_sym) + @atomic :release srcloc.zone_name = _strpointer(srcloc.zone_name_str) +end + +struct TracyZoneCtx + id::UInt32 + active::Cint +end + +const srclocs = Vector{TracySrcLoc}() + +function tracy_zone_create(name::String, ex::Expr, linfo::LineNumberNode) + # Intern strings + for loc in srclocs + if loc.zone_name_str === name + name = loc.zone_name_str + break + end + end + loc = TracySrcLoc(name, Symbol("unknown"), linfo.file, UInt32(linfo.line), UInt32(0)) + # Also roots `loc` in `srclocs` + push!(srclocs, loc) + return loc +end + +function tracy_zone_begin(loc, active) + if (@atomic :acquire loc.zone_name) === Ptr{UInt8}(0) + reinit!(loc) + end + # `loc` is rooted in the global `srclocs` + ptr = Ptr{TracySrcLoc}(pointer_from_objref(loc)) + return ccall((:___tracy_emit_zone_begin, "libTracyClient"), TracyZoneCtx, (Ptr{TracySrcLoc}, Cint), ptr, active) +end + +function tracy_zone_end(ctx) + ccall((:___tracy_emit_zone_end, "libTracyClient"), Cvoid, (TracyZoneCtx,), ctx) +end + +end diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 59eb08d8faed9..c7e052ed17218 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -73,10 +73,10 @@ add_inlining_edge!(et::InliningEdgeTracker, edge::MethodInstance) = add_inlining function ssa_inlining_pass!(ir::IRCode, state::InliningState, propagate_inbounds::Bool) # Go through the function, performing simple inlining (e.g. replacing call by constants # and analyzing legality of inlining). - @timeit "analysis" todo = assemble_inline_todo!(ir, state) + @zone "CC: ANALYSIS" todo = assemble_inline_todo!(ir, state) isempty(todo) && return ir # Do the actual inlining for every call we identified - @timeit "execution" ir = batch_inline!(ir, todo, propagate_inbounds, state.interp) + @zone "CC: EXECUTION" ir = batch_inline!(ir, todo, propagate_inbounds, state.interp) return ir end diff --git a/Compiler/src/ssair/slot2ssa.jl b/Compiler/src/ssair/slot2ssa.jl index e0f3e207789a3..d9ea2539f0ffc 100644 --- a/Compiler/src/ssair/slot2ssa.jl +++ b/Compiler/src/ssair/slot2ssa.jl @@ -574,7 +574,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, for (; leave_block) in catch_entry_blocks new_phic_nodes[leave_block] = NewPhiCNode2[] end - @timeit "idf" for (idx, slot) in Iterators.enumerate(defuses) + @zone "CC: IDF" for (idx, slot) in Iterators.enumerate(defuses) # No uses => no need for phi nodes isempty(slot.uses) && continue # TODO: Restore this optimization @@ -600,7 +600,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, continue end - @timeit "liveness" (live = compute_live_ins(cfg, slot)) + @zone "CC: LIVENESS" (live = compute_live_ins(cfg, slot)) for li in live.live_in_bbs push!(live_slots[li], idx) cidx = findfirst(x::TryCatchRegion->x.leave_block==li, catch_entry_blocks) @@ -671,7 +671,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, worklist = Tuple{Int, Int, Vector{Pair{Any, Any}}}[(1, 0, initial_incoming_vals)] visited = BitSet() new_nodes = ir.new_nodes - @timeit "SSA Rename" while !isempty(worklist) + @zone "CC: SSA_RENAME" while !isempty(worklist) (item, pred, incoming_vals) = pop!(worklist) if sv.bb_vartables[item] === nothing continue @@ -891,6 +891,6 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, local node = new_nodes.stmts[i] node[:stmt] = new_to_regular(renumber_ssa!(node[:stmt], ssavalmap), nstmts) end - @timeit "domsort" ir = domsort_ssa!(ir, domtree) + @zone "CC: DOMSORT" ir = domsort_ssa!(ir, domtree) return ir end diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index e52943d2c960e..a5f271b0c3ef9 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -4,7 +4,7 @@ # generic # ########### -if !@isdefined(var"@timeit") +if !@isdefined(var"@zone") # This is designed to allow inserting timers when loading a second copy # of inference for performing performance experiments. macro timeit(args...) diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index bb0a2602a0c23..0a88907965f5a 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -1852,7 +1852,7 @@ multi_inlining1(a::Int, b::Int) = @noinline func_mul_int(a, b) let i::Int, continue_::Bool interp = Compiler.NativeInterpreter() # check if callsite `@noinline` annotation works - ir, = only(Base.code_ircode(multi_inlining1, (Int,Int); optimize_until="inlining", interp)) + ir, = only(Base.code_ircode(multi_inlining1, (Int,Int); optimize_until="CC: INLINING", interp)) i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) @test i !== nothing # now delete the callsite flag, and see the second inlining pass can inline the call @@ -1876,7 +1876,7 @@ multi_inlining2(a::Int, b::Int) = call_func_mul_int(a, b) let i::Int, continue_::Bool interp = Compiler.NativeInterpreter() # check if callsite `@noinline` annotation works - ir, = only(Base.code_ircode(multi_inlining2, (Int,Int); optimize_until="inlining", interp)) + ir, = only(Base.code_ircode(multi_inlining2, (Int,Int); optimize_until="CC: INLINING", interp)) i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) @test i !== nothing # now delete the callsite flag, and see the second inlining pass can inline the call @@ -2220,7 +2220,7 @@ struct Issue52644 end issue52644(::DataType) = :DataType issue52644(::UnionAll) = :UnionAll -let ir = Base.code_ircode((Issue52644,); optimize_until="inlining") do t +let ir = Base.code_ircode((Issue52644,); optimize_until="CC: INLINING") do t issue52644(t.tuple) end |> only |> first ir.argtypes[1] = Tuple{} @@ -2229,7 +2229,7 @@ let ir = Base.code_ircode((Issue52644,); optimize_until="inlining") do t @test irfunc(Issue52644(Tuple{<:Integer})) === :UnionAll end issue52644_single(x::DataType) = :DataType -let ir = Base.code_ircode((Issue52644,); optimize_until="inlining") do t +let ir = Base.code_ircode((Issue52644,); optimize_until="CC: INLINING") do t issue52644_single(t.tuple) end |> only |> first ir.argtypes[1] = Tuple{} diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 526c322ec6573..d777a2c387b3d 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -921,7 +921,7 @@ let # Test that CFG simplify doesn't try to merge every block in a loop into end # `cfg_simplify!` shouldn't error in a presence of `try/catch` block -let ir = Base.code_ircode(; optimize_until="slot2reg") do +let ir = Base.code_ircode(; optimize_until="CC: SLOT2REG") do v = try catch end @@ -1459,7 +1459,7 @@ function f_with_early_try_catch_exit() result end -let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="slot2reg"))) +let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="CC: SLOT2REG"))) for i = 1:length(ir.stmts) expr = ir.stmts[i][:stmt] if isa(expr, PhiCNode) diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 9c92ed2f62679..0ae7c99fc3a62 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -419,7 +419,7 @@ end @test first(only(Base.code_ircode(+, (Float64, Float64)))) isa Compiler.IRCode @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = 3))) isa Compiler.IRCode - @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = "SROA"))) isa + @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = "CC: SROA"))) isa Compiler.IRCode function demo(f) @@ -429,7 +429,7 @@ end end @test first(only(Base.code_ircode(demo))) isa Compiler.IRCode @test first(only(Base.code_ircode(demo; optimize_until = 3))) isa Compiler.IRCode - @test first(only(Base.code_ircode(demo; optimize_until = "SROA"))) isa Compiler.IRCode + @test first(only(Base.code_ircode(demo; optimize_until = "CC: SROA"))) isa Compiler.IRCode end # slots after SSA conversion @@ -442,12 +442,12 @@ end let # #self#, a, b, c, d unopt = code_typed1(f_with_slots, (Int,Int); optimize=false) @test length(unopt.slotnames) == length(unopt.slotflags) == length(unopt.slottypes) == 5 - ir_withslots = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="convert"))) + ir_withslots = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="CC: CONVERT"))) @test length(ir_withslots.argtypes) == 5 # #self#, a, b opt = code_typed1(f_with_slots, (Int,Int); optimize=true) @test length(opt.slotnames) == length(opt.slotflags) == length(opt.slottypes) == 3 - ir_ssa = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="slot2reg"))) + ir_ssa = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="CC: SLOT2REG"))) @test length(ir_ssa.argtypes) == 3 end @@ -598,7 +598,7 @@ import Core: SSAValue import .Compiler: NewInstruction, insert_node! # insert_node! for pending node -let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b +let ir = Base.code_ircode((Int,Int); optimize_until="CC: INLINING") do a, b a^b end |> only |> first ir = Compiler.compact!(ir) @@ -660,7 +660,7 @@ let code = Any[ end # insert_node! with new instruction with flag computed -let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b +let ir = Base.code_ircode((Int,Int); optimize_until="CC: INLINING") do a, b a^b end |> only |> first ir = Compiler.compact!(ir) @@ -725,7 +725,7 @@ end @test any(j -> isa(unopt.code[j], Core.Const) && unopt.ssavaluetypes[j] == Union{}, 1:length(unopt.code)) # Any GotoIfNot destinations after IRCode conversion should not be statically unreachable - ircode = first(only(Base.code_ircode(f_with_maybe_nonbool_cond, (Int, Bool); optimize_until="convert"))) + ircode = first(only(Base.code_ircode(f_with_maybe_nonbool_cond, (Int, Bool); optimize_until="CC: CONVERT"))) for i = 1:length(ircode.stmts) expr = ircode.stmts[i][:stmt] if isa(expr, GotoIfNot) diff --git a/src/timing.c b/src/timing.c index 265e50ad3dd74..ce9f98882710b 100644 --- a/src/timing.c +++ b/src/timing.c @@ -16,6 +16,30 @@ jl_module_t *jl_module_root(jl_module_t *m); extern "C" { #endif +JL_DLLEXPORT int jl_tracy_enabled(void) { +#ifdef USE_TRACY + return 1; +#else + return 0; +#endif +} + +JL_DLLEXPORT int jl_ittapi_enabled(void) { +#ifdef USE_ITTAPI + return 1; +#else + return 0; +#endif +} + +JL_DLLEXPORT int jl_nvtx_enabled(void) { +#ifdef USE_NVTX + return 1; +#else + return 0; +#endif +} + #ifdef ENABLE_TIMINGS #ifndef HAVE_TIMING_SUPPORT diff --git a/src/timing.h b/src/timing.h index 61118cc3b41ab..ca6aebce445b8 100644 --- a/src/timing.h +++ b/src/timing.h @@ -55,6 +55,10 @@ JL_DLLEXPORT jl_timing_event_t *_jl_timing_event_create(const char *subsystem, c JL_DLLEXPORT void _jl_timing_block_init(char *buf, size_t size, jl_timing_event_t *event); JL_DLLEXPORT void _jl_timing_block_start(jl_timing_block_t *cur_block); JL_DLLEXPORT void _jl_timing_block_end(jl_timing_block_t *cur_block); +JL_DLLEXPORT int jl_tracy_enabled(void); +JL_DLLEXPORT int jl_ittapi_enabled(void); +JL_DLLEXPORT int jl_nvtx_enabled(void); + #ifdef __cplusplus } diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 084ac973b913d..48ec3910b0e16 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -826,7 +826,7 @@ end @test Base.infer_return_type(sin, (Int,)) == InteractiveUtils.@infer_return_type sin(42) @test Base.infer_exception_type(sin, (Int,)) == InteractiveUtils.@infer_exception_type sin(42) @test first(InteractiveUtils.@code_ircode sin(42)) isa Core.Compiler.IRCode -@test first(InteractiveUtils.@code_ircode optimize_until="inlining" sin(42)) isa Core.Compiler.IRCode +@test first(InteractiveUtils.@code_ircode optimize_until="CC: INLINING" sin(42)) isa Core.Compiler.IRCode @testset "Docstrings" begin @test isempty(Docs.undocumented_names(InteractiveUtils)) From d0703e7dae69e198f701d8558a2cb73ee54a787e Mon Sep 17 00:00:00 2001 From: Zentrik Date: Thu, 17 Apr 2025 10:51:33 +0100 Subject: [PATCH 099/662] Revert "remove some more serialized junk from the sysimg" (#58150) Revert #58078 as it broke CI extensively. --- Compiler/src/typeinfer.jl | 4 +- src/aotcompile.cpp | 14 ++++ src/gf.c | 20 ++++++ src/julia_internal.h | 1 + src/staticdata.c | 142 +++++++++++--------------------------- 5 files changed, 78 insertions(+), 103 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 43b743982ea89..1afe4c297e5ba 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -399,7 +399,7 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen if isa(src, OptimizationState) opt = src inlining_cost = compute_inlining_cost(interp, result, opt.optresult) - discard_optimized_result(interp, opt, inlining_cost, result.ipo_effects) && return nothing + discard_optimized_result(interp, opt, inlining_cost) && return nothing src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) @@ -409,7 +409,7 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen return src end -function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#, effects::Effects) +function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) may_discard_trees(interp) || return false return inlining_cost == MAX_INLINE_COST end diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5cdecda9d8582..d687f44808409 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -675,6 +675,20 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[0] = (jl_value_t*)codeinfos; void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, external_linkage); + // examine everything just emitted and save it to the caches + if (!external_linkage) { + for (size_t i = 0, l = jl_array_nrows(codeinfos); i < l; i++) { + jl_value_t *item = jl_array_ptr_ref(codeinfos, i); + if (jl_is_code_instance(item)) { + // now add it to our compilation results + jl_code_instance_t *codeinst = (jl_code_instance_t*)item; + jl_code_info_t *src = (jl_code_info_t*)jl_array_ptr_ref(codeinfos, ++i); + assert(jl_is_code_info(src)); + jl_add_codeinst_to_cache(codeinst, src); + } + } + } + // move everything inside, now that we've merged everything // (before adding the exported headers) ((jl_native_code_desc_t*)data)->M.withModuleDo([&](Module &M) { diff --git a/src/gf.c b/src/gf.c index 5b21dfa7e1143..c8de6d03449cd 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2837,10 +2837,30 @@ void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_c jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_PROPAGATES_ROOT); +JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src) +{ + assert(jl_is_code_info(src)); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + if (jl_generating_output() && jl_is_method(mi->def.method) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { + jl_value_t *compressed = jl_compress_ir(mi->def.method, src); + // These should already be compatible (and should be an assert), but make sure of it anyways + if (jl_is_svec(src->edges)) { + jl_atomic_store_release(&codeinst->edges, (jl_svec_t*)src->edges); + jl_gc_wb(codeinst, src->edges); + } + jl_atomic_store_release(&codeinst->debuginfo, src->debuginfo); + jl_gc_wb(codeinst, src->debuginfo); + jl_atomic_store_release(&codeinst->inferred, compressed); + jl_gc_wb(codeinst, compressed); + } +} + + JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src) { assert(jl_is_code_info(src)); jl_emit_codeinst_to_jit(codeinst, src); + jl_add_codeinst_to_cache(codeinst, src); } jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) diff --git a/src/julia_internal.h b/src/julia_internal.h index 864097d8d22fe..26380768c0b60 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -688,6 +688,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROP JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile); JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src); +JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( diff --git a/src/staticdata.c b/src/staticdata.c index 4c1489770aa7e..c96faf5a71a54 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -863,60 +863,40 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } - if (jl_is_method_instance(v)) { + if (s->incremental && jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; - if (s->incremental) { - jl_value_t *def = mi->def.value; - if (needs_uniquing(v, s->query_cache)) { - // we only need 3 specific fields of this (the rest are not used) - jl_queue_for_serialization(s, mi->def.value); - jl_queue_for_serialization(s, mi->specTypes); - jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); - goto done_fields; - } - else if (jl_is_method(def) && jl_object_in_image(def)) { - // we only need 3 specific fields of this (the rest are restored afterward, if valid) - // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, - // so must not be present here - record_field_change((jl_value_t**)&mi->backedges, NULL); - record_field_change((jl_value_t**)&mi->cache, NULL); - } - else { - assert(!needs_recaching(v, s->query_cache)); - } - // n.b. opaque closures cannot be inspected and relied upon like a - // normal method since they can get improperly introduced by generated - // functions, so if they appeared at all, we will probably serialize - // them wrong and segfault. The jl_code_for_staged function should - // prevent this from happening, so we do not need to detect that user - // error now. + jl_value_t *def = mi->def.value; + if (needs_uniquing(v, s->query_cache)) { + // we only need 3 specific fields of this (the rest are not used) + jl_queue_for_serialization(s, mi->def.value); + jl_queue_for_serialization(s, mi->specTypes); + jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); + goto done_fields; } - // don't recurse into all backedges memory (yet) - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); - if (backedges) { - jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); - size_t i = 0, n = jl_array_nrows(backedges); - while (i < n) { - jl_value_t *invokeTypes; - jl_code_instance_t *caller; - i = get_next_edge((jl_array_t*)backedges, i, &invokeTypes, &caller); - if (invokeTypes) - jl_queue_for_serialization(s, invokeTypes); - } + else if (jl_is_method(def) && jl_object_in_image(def)) { + // we only need 3 specific fields of this (the rest are restored afterward, if valid) + // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, + // so must not be present here + record_field_change((jl_value_t**)&mi->backedges, NULL); + record_field_change((jl_value_t**)&mi->cache, NULL); } - } - if (jl_is_binding(v)) { - jl_binding_t *b = (jl_binding_t*)v; - if (s->incremental && needs_uniquing(v, s->query_cache)) { + else { + assert(!needs_recaching(v, s->query_cache)); + } + // n.b. opaque closures cannot be inspected and relied upon like a + // normal method since they can get improperly introduced by generated + // functions, so if they appeared at all, we will probably serialize + // them wrong and segfault. The jl_code_for_staged function should + // prevent this from happening, so we do not need to detect that user + // error now. + } + if (s->incremental && jl_is_binding(v)) { + if (needs_uniquing(v, s->query_cache)) { + jl_binding_t *b = (jl_binding_t*)v; jl_queue_for_serialization(s, b->globalref->mod); jl_queue_for_serialization(s, b->globalref->name); goto done_fields; } - // don't recurse into backedges memory (yet) - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); - if (backedges) { - jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); - } } if (s->incremental && jl_is_globalref(v)) { jl_globalref_t *gr = (jl_globalref_t*)v; @@ -934,20 +914,18 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } } - if (jl_is_code_instance(v)) { + if (s->incremental && jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; jl_method_instance_t *mi = jl_get_ci_mi(ci); - if (s->incremental) { - // make sure we don't serialize other reachable cache entries of foreign methods - // Should this now be: - // if (ci !in ci->defs->cache) - // record_field_change((jl_value_t**)&ci->next, NULL); - // Why are we checking that the method/module this originates from is in_image? - // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)mi->def.value)) { - // TODO: if (ci in ci->defs->cache) - record_field_change((jl_value_t**)&ci->next, NULL); - } + // make sure we don't serialize other reachable cache entries of foreign methods + // Should this now be: + // if (ci !in ci->defs->cache) + // record_field_change((jl_value_t**)&ci->next, NULL); + // Why are we checking that the method/module this originates from is in_image? + // and then disconnect this CI? + if (jl_object_in_image((jl_value_t*)mi->def.value)) { + // TODO: if (ci in ci->defs->cache) + record_field_change((jl_value_t**)&ci->next, NULL); } jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred); if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) @@ -975,7 +953,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (inferred == jl_nothing) { record_field_change((jl_value_t**)&ci->inferred, jl_nothing); } - else if (s->incremental && jl_is_string(inferred)) { + else if (jl_is_string(inferred)) { // New roots for external methods if (jl_object_in_image((jl_value_t*)def)) { void **pfound = ptrhash_bp(&s->method_roots_index, def); @@ -2594,35 +2572,6 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } -static void jl_prune_mi_backedges(jl_array_t *backedges) -{ - if (backedges == NULL) - return; - size_t i = 0, ins = 0, n = jl_array_nrows(backedges); - while (i < n) { - jl_value_t *invokeTypes; - jl_code_instance_t *caller; - i = get_next_edge(backedges, i, &invokeTypes, &caller); - if (ptrhash_get(&serialization_order, caller) != HT_NOTFOUND) - ins = set_next_edge(backedges, ins, invokeTypes, caller); - } - jl_array_del_end(backedges, n - ins); -} - -static void jl_prune_binding_backedges(jl_array_t *backedges) -{ - if (backedges == NULL) - return; - size_t i = 0, ins = 0, n = jl_array_nrows(backedges); - for (i = 0; i < n; i++) { - jl_value_t *b = jl_array_ptr_ref(backedges, i); - if (ptrhash_get(&serialization_order, b) != HT_NOTFOUND) - jl_array_ptr_set(backedges, ins, b); - } - jl_array_del_end(backedges, n - ins); -} - - uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED @@ -3196,11 +3145,12 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_queue_for_serialization(&s, global_roots_keyset); jl_serialize_reachable(&s); } - // step 1.5: prune (garbage collect) some special weak references known caches + // step 1.5: prune (garbage collect) some special weak references from + // built-in type caches too for (i = 0; i < serialization_queue.len; i++) { jl_value_t *v = (jl_value_t*)serialization_queue.items[i]; if (jl_options.trim) { - if (jl_is_method(v)) { + if (jl_is_method(v)){ jl_method_t *m = (jl_method_t*)v; jl_value_t *specializations_ = jl_atomic_load_relaxed(&m->specializations); if (!jl_is_svec(specializations_)) @@ -3228,16 +3178,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); } - else if (jl_is_method_instance(v)) { - jl_method_instance_t *mi = (jl_method_instance_t*)v; - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); - jl_prune_mi_backedges((jl_array_t*)backedges); - } - else if (jl_is_binding(v)) { - jl_binding_t *b = (jl_binding_t*)v; - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); - jl_prune_binding_backedges((jl_array_t*)backedges); - } } } From c3e7b1b19741f9fad7aab7e0c97c34c288e1f1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 17 Apr 2025 23:43:13 +0200 Subject: [PATCH 100/662] InteractiveUtils: support type annotations as substitutes for values (#57909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend code introspection macros (`@which`, `@code_typed` and friends) to recognize type annotations of the form `f(1, ::Float64)` as types to be forwarded as is to the relevant function. --------- Co-authored-by: Cédric Belmant Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> --- NEWS.md | 2 + base/errorshow.jl | 5 +- doc/src/base/reflection.md | 2 +- stdlib/InteractiveUtils/src/macros.jl | 204 ++++++++++++++++++----- stdlib/InteractiveUtils/test/runtests.jl | 68 +++++++- 5 files changed, 225 insertions(+), 56 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9c6e35a6c8127..3f6ab863b05e8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -47,6 +47,8 @@ Standard library changes #### InteractiveUtils +* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}` ([#57909]). + External dependencies --------------------- diff --git a/base/errorshow.jl b/base/errorshow.jl index c53c605cbfb86..23d6b771e954c 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1070,9 +1070,8 @@ Experimental.register_error_hint(nonsetable_type_hint_handler, MethodError) # Display a hint in case the user tries to use the + operator on strings # (probably attempting concatenation) -function string_concatenation_hint_handler(io, ex, arg_types, kwargs) - @nospecialize - if (ex.f === +) && !isempty(arg_types) && all(i -> i <: AbstractString, arg_types) +function string_concatenation_hint_handler(@nospecialize(io::IO), ex::MethodError, arg_types::Vector{Any}, kwargs::Vector{Any}) + if (ex.f === +) && !isempty(arg_types) && all(@nospecialize(a) -> unwrapva(a) <: AbstractString, arg_types) print(io, "\nString concatenation is performed with ") printstyled(io, "*", color=:cyan) print(io, " (See also: https://docs.julialang.org/en/v1/manual/strings/#man-concatenation).") diff --git a/doc/src/base/reflection.md b/doc/src/base/reflection.md index d88c3c8b0d0cf..3abfac3012d51 100644 --- a/doc/src/base/reflection.md +++ b/doc/src/base/reflection.md @@ -87,7 +87,7 @@ be passed instead!). For example: ```jldoctest; setup = :(using InteractiveUtils) julia> InteractiveUtils.macroexpand(@__MODULE__, :(@edit println("")) ) -:(InteractiveUtils.edit(println, (Base.typesof)(""))) +:(InteractiveUtils.edit(println, InteractiveUtils.Tuple{(InteractiveUtils.Core).Typeof("")})) ``` The functions `Base.Meta.show_sexpr` and [`dump`](@ref) are used to display S-expr style views diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 68afc40976275..9c59dea132664 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -2,14 +2,34 @@ # macro wrappers for various reflection functions -using Base: typesof, insert!, replace_ref_begin_end!, - infer_return_type, infer_exception_type, infer_effects, code_ircode +using Base: insert!, replace_ref_begin_end!, + infer_return_type, infer_exception_type, infer_effects, code_ircode, isexpr # defined in Base so it's possible to time all imports, including InteractiveUtils and its deps # via. `Base.@time_imports` etc. import Base: @time_imports, @trace_compile, @trace_dispatch -separate_kwargs(args...; kwargs...) = (args, values(kwargs)) +typesof_expr(args::Vector{Any}, where_params::Union{Nothing, Vector{Any}} = nothing) = rewrap_where(:(Tuple{$(get_typeof.(args)...)}), where_params) + +function extract_where_parameters(ex::Expr) + isexpr(ex, :where) || return ex, nothing + ex.args[1], ex.args[2:end] +end + +function rewrap_where(ex::Expr, where_params::Union{Nothing, Vector{Any}}) + isnothing(where_params) && return ex + Expr(:where, ex, esc.(where_params)...) +end + +function get_typeof(@nospecialize ex) + isexpr(ex, :(::), 1) && return esc(ex.args[1]) + if isexpr(ex, :..., 1) + splatted = ex.args[1] + isexpr(splatted, :(::), 1) && return Expr(:curly, :Vararg, esc(splatted.args[1])) + return :(Any[Core.Typeof(x) for x in $(esc(splatted))]...) + end + return :(Core.Typeof($(esc(ex)))) +end """ Transform a dot expression into one where each argument has been replaced by a @@ -20,7 +40,7 @@ function recursive_dotcalls!(ex, args, i=1) if !(ex isa Expr) || ((ex.head !== :. || !(ex.args[2] isa Expr)) && (ex.head !== :call || string(ex.args[1])[1] != '.')) newarg = Symbol('x', i) - if Meta.isexpr(ex, :...) + if isexpr(ex, :...) push!(args, only(ex.args)) return Expr(:..., newarg), i+1 else @@ -37,20 +57,121 @@ function recursive_dotcalls!(ex, args, i=1) return ex, i end +function extract_farg(@nospecialize arg) + !isexpr(arg, :(::), 1) && return esc(arg) + fT = esc(arg.args[1]) + :($construct_callable($fT)) +end + +function construct_callable(@nospecialize(func::Type)) + # Support function singleton types such as `(::typeof(f))(args...)` + Base.issingletontype(func) && isdefined(func, :instance) && return func.instance + # Don't support type annotations otherwise, we don't want to give wrong answers + # for callables such as `(::Returns{Int})(args...)` where using `Returns{Int}` + # would give us code for the constructor, not for the callable object. + throw(ArgumentError("If a function type is explicitly provided, it must be a singleton whose only instance is the callable object")) +end + +function separate_kwargs(exs::Vector{Any}) + args = [] + kwargs = [] + for ex in exs + if isexpr(ex, :kw) + push!(kwargs, ex) + elseif isexpr(ex, :parameters) + for kw in ex.args + push!(kwargs, kw) + end + else + push!(args, ex) + end + end + args, kwargs +end + +function are_kwargs_valid(kwargs::Vector{Any}) + for kwarg in kwargs + isexpr(kwarg, :..., 1) && continue + isexpr(kwarg, :kw, 2) && isa(kwarg.args[1], Symbol) && continue + isa(kwarg, Symbol) && continue + return false + end + return true +end + +# Generate an expression that merges `kwargs` onto a single `NamedTuple` +function generate_merged_namedtuple_type(kwargs::Vector{Any}) + nts = Any[] + ntargs = Pair{Symbol, Any}[] + for ex in kwargs + if isexpr(ex, :..., 1) + if !isempty(ntargs) + # Construct a `NamedTuple` containing the previous parameters. + push!(nts, generate_namedtuple_type(ntargs)) + empty!(ntargs) + end + push!(nts, Expr(:call, typeof_nt, esc(ex.args[1]))) + elseif isexpr(ex, :kw, 2) + push!(ntargs, ex.args[1]::Symbol => get_typeof(ex.args[2])) + else + ex::Symbol + push!(ntargs, ex => get_typeof(ex)) + end + end + !isempty(ntargs) && push!(nts, generate_namedtuple_type(ntargs)) + return :($merge_namedtuple_types($(nts...))) +end + +function generate_namedtuple_type(ntargs::Vector{Pair{Symbol, Any}}) + names = Expr(:tuple) + tt = Expr(:curly, :Tuple) + for (name, type) in ntargs + push!(names.args, QuoteNode(name)) + push!(tt.args, type) + end + return :(NamedTuple{$names, $tt}) +end + +typeof_nt(nt::NamedTuple) = typeof(nt) +typeof_nt(nt::Base.Pairs) = typeof(values(nt)) + +function merge_namedtuple_types(nt::Type{<:NamedTuple}, nts::Type{<:NamedTuple}...) + @nospecialize + isempty(nts) && return nt + names = Symbol[] + types = Any[] + for nt in (nt, nts...) + for (name, type) in zip(fieldnames(nt), fieldtypes(nt)) + i = findfirst(==(name), names) + if isnothing(i) + push!(names, name) + push!(types, type) + else + types[i] = type + end + end + end + NamedTuple{Tuple(names), Tuple{types...}} +end + function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) - if Meta.isexpr(ex0, :ref) + if isexpr(ex0, :ref) ex0 = replace_ref_begin_end!(ex0) end # assignments get bypassed: @edit a = f(x) <=> @edit f(x) if isa(ex0, Expr) && ex0.head == :(=) && isa(ex0.args[1], Symbol) && isempty(kws) return gen_call_with_extracted_types(__module__, fcn, ex0.args[2]) end + where_params = nothing + if isa(ex0, Expr) + ex0, where_params = extract_where_parameters(ex0) + end if isa(ex0, Expr) - if ex0.head === :do && Meta.isexpr(get(ex0.args, 1, nothing), :call) + if ex0.head === :do && isexpr(get(ex0.args, 1, nothing), :call) if length(ex0.args) != 2 return Expr(:call, :error, "ill-formed do call") end - i = findlast(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args[1].args) + i = findlast(@nospecialize(a)->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args[1].args) args = copy(ex0.args[1].args) insert!(args, (isnothing(i) ? 2 : 1+i::Int), ex0.args[2]) ex0 = Expr(:call, args...) @@ -66,8 +187,7 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) dotfuncdef = Expr(:local, Expr(:(=), Expr(:call, dotfuncname, xargs...), ex)) return quote $(esc(dotfuncdef)) - local args = $typesof($(map(esc, args)...)) - $(fcn)($(esc(dotfuncname)), args; $(kws...)) + $(fcn)($(esc(dotfuncname)), $(typesof_expr(args, where_params)); $(kws...)) end elseif !codemacro fully_qualified_symbol = true # of the form A.B.C.D @@ -80,7 +200,9 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) ex1 = ex1.args[1] end fully_qualified_symbol &= ex1 isa Symbol - if fully_qualified_symbol + if fully_qualified_symbol || isexpr(ex1, :(::), 1) + getproperty_ex = :($(fcn)(Base.getproperty, $(typesof_expr(ex0.args, where_params)))) + isexpr(ex0.args[1], :(::), 1) && return getproperty_ex return quote local arg1 = $(esc(ex0.args[1])) if isa(arg1, Module) @@ -90,8 +212,7 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) :(error("expression is not a function call")) end) else - local args = $typesof($(map(esc, ex0.args)...)) - $(fcn)(Base.getproperty, args) + $getproperty_ex end end else @@ -102,32 +223,35 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) end end end - if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) - return quote - local arg1 = $(esc(ex0.args[1])) - local args, kwargs = $separate_kwargs($(map(esc, ex0.args[2:end])...)) - $(fcn)(Core.kwcall, - Tuple{typeof(kwargs), Core.Typeof(arg1), map(Core.Typeof, args)...}; - $(kws...)) + if any(@nospecialize(a)->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args) + args, kwargs = separate_kwargs(ex0.args) + are_kwargs_valid(kwargs) || return quote + error("keyword argument format unrecognized; they must be of the form `x` or `x = `") + $(esc(ex0)) # trigger syntax errors if any end + nt = generate_merged_namedtuple_type(kwargs) + tt = rewrap_where(:(Tuple{$nt, $(get_typeof.(args)...)}), where_params) + return :($(fcn)(Core.kwcall, $tt; $(kws...))) elseif ex0.head === :call + argtypes = Any[get_typeof(arg) for arg in ex0.args[2:end]] if ex0.args[1] === :^ && length(ex0.args) >= 3 && isa(ex0.args[3], Int) - return Expr(:call, fcn, :(Base.literal_pow), - Expr(:call, typesof, esc(ex0.args[1]), esc(ex0.args[2]), - esc(Val(ex0.args[3])))) + farg = :(Base.literal_pow) + pushfirst!(argtypes, :(typeof(^))) + argtypes[3] = :(Val{$(ex0.args[3])}) + else + farg = extract_farg(ex0.args[1]) end - return Expr(:call, fcn, esc(ex0.args[1]), - Expr(:call, typesof, map(esc, ex0.args[2:end])...), - kws...) + tt = rewrap_where(:(Tuple{$(argtypes...)}), where_params) + return Expr(:call, fcn, farg, tt, kws...) elseif ex0.head === :(=) && length(ex0.args) == 2 lhs, rhs = ex0.args if isa(lhs, Expr) if lhs.head === :(.) return Expr(:call, fcn, Base.setproperty!, - Expr(:call, typesof, map(esc, lhs.args)..., esc(rhs)), kws...) + typesof_expr(Any[lhs.args..., rhs], where_params), kws...) elseif lhs.head === :ref return Expr(:call, fcn, Base.setindex!, - Expr(:call, typesof, esc(lhs.args[1]), esc(rhs), map(esc, lhs.args[2:end])...), kws...) + typesof_expr(Any[lhs.args[1], rhs, lhs.args[2:end]...], where_params), kws...) end end elseif ex0.head === :vcat || ex0.head === :typed_vcat @@ -138,23 +262,19 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) f, hf = Base.typed_vcat, Base.typed_hvcat args = ex0.args[2:end] end - if any(a->isa(a,Expr) && a.head === :row, args) + if any(@nospecialize(a)->isa(a,Expr) && a.head === :row, args) rows = Any[ (isa(x,Expr) && x.head === :row ? x.args : Any[x]) for x in args ] lens = map(length, rows) - return Expr(:call, fcn, hf, - Expr(:call, typesof, - (ex0.head === :vcat ? [] : Any[esc(ex0.args[1])])..., - Expr(:tuple, lens...), - map(esc, vcat(rows...))...), kws...) + args = Any[Expr(:tuple, lens...); vcat(rows...)] + ex0.head === :typed_vcat && pushfirst!(args, ex0.args[1]) + return Expr(:call, fcn, hf, typesof_expr(args, where_params), kws...) else - return Expr(:call, fcn, f, - Expr(:call, typesof, map(esc, ex0.args)...), kws...) + return Expr(:call, fcn, f, typesof_expr(ex0.args, where_params), kws...) end else for (head, f) in (:ref => Base.getindex, :hcat => Base.hcat, :(.) => Base.getproperty, :vect => Base.vect, Symbol("'") => Base.adjoint, :typed_hcat => Base.typed_hcat, :string => string) if ex0.head === head - return Expr(:call, fcn, f, - Expr(:call, typesof, map(esc, ex0.args)...), kws...) + return Expr(:call, fcn, f, typesof_expr(ex0.args, where_params), kws...) end end end @@ -170,16 +290,14 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) exret = Expr(:none) if ex.head === :call - if any(e->(isa(e, Expr) && e.head === :(...)), ex0.args) && + if any(@nospecialize(x) -> isexpr(x, :...), ex0.args) && (ex.args[1] === GlobalRef(Core,:_apply_iterate) || ex.args[1] === GlobalRef(Base,:_apply_iterate)) # check for splatting exret = Expr(:call, ex.args[2], fcn, - Expr(:tuple, esc(ex.args[3]), - Expr(:call, typesof, map(esc, ex.args[4:end])...))) + Expr(:tuple, extract_farg(ex.args[3]), typesof_expr(ex.args[4:end], where_params))) else - exret = Expr(:call, fcn, esc(ex.args[1]), - Expr(:call, typesof, map(esc, ex.args[2:end])...), kws...) + exret = Expr(:call, fcn, extract_farg(ex.args[1]), typesof_expr(ex.args[2:end], where_params), kws...) end end if ex.head === :thunk || exret.head === :none @@ -460,7 +578,7 @@ For `@activate Compiler`, the following options are available: """ macro activate(what) options = Symbol[] - if Meta.isexpr(what, :ref) + if isexpr(what, :ref) Component = what.args[1] for i = 2:length(what.args) arg = what.args[i] diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 48ec3910b0e16..d01e614abea84 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -328,23 +328,64 @@ try catch err13464 @test startswith(err13464.msg, "expression is not a function call") end + +# PR 57909 +@testset "Support for type annotations as arguments" begin + @test (@which (::Vector{Int})[::Int]).name === :getindex + @test (@which (::Vector{Int})[::Int] = ::Int).name === :setindex! + @test (@which (::Base.RefValue{Int}).x).name === :getproperty + @test (@which (::Base.RefValue{Int}).x = ::Int).name === :setproperty! + @test (@which (::Float64)^2).name === :literal_pow + @test (@which [::Int]).name === :vect + @test (@which [::Int 2]).name === :hcat + @test (@which [::Int; 2]).name === :vcat + @test (@which Int[::Int 2]).name === :typed_hcat + @test (@which Int[::Int; 2]).name === :typed_vcat + @test (@which [::Int 2;3 (::Int)]).name === :hvcat + @test (@which Int[::Int 2;3 (::Int)]).name === :typed_hvcat + @test (@which (::Vector{Float64})').name === :adjoint + @test (@which "$(::Symbol) is a symbol").sig === Tuple{typeof(string), Vararg{Union{Char, String, Symbol}}} + @test (@which +(::Any, ::Any, ::Any, ::Any...)).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} + @test (@which +(::Any, ::Any, ::Any, ::Vararg{Any})).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} + n = length(@code_typed +(::Float64, ::Vararg{Float64})) + @test n ≥ 2 + @test length(@code_typed +(::Float64, ::Float64...)) == n + @test (@which +(1, ::Float64)).sig === Tuple{typeof(+), Number, Number} + @test (@which +((1, 2)...)).name === :+ + @test (@which (::typeof(+))(::Int, ::Float64)).sig === Tuple{typeof(+), Number, Number} + @test (@code_typed .+(::Float64, ::Vector{Float64})) isa Pair + @test (@code_typed .+(::Float64, .*(::Vector{Float64}, ::Int))) isa Pair + @test (@which +(::T, ::T) where {T<:Number}).sig === Tuple{typeof(+), T, T} where {T<:Number} + @test (@which round(::Float64; digits=3)).name === :round + @test (@which round(1.2; digits = ::Int)).name === :round + @test (@code_typed round(::T; digits = ::T) where {T<:Float64})[2] === Union{} + @test (@code_typed round(::T; digits = ::T) where {T<:Int})[2] === Float64 + base = 10 + kwargs_1 = (; digits = 3) + kwargs_2 = (; sigdigits = 3) + @test (@which round(1.2; kwargs_1...)).name === :round + @test (@which round(1.2; digits = 1, kwargs_1...)).name === :round + @test (@code_typed round(1.2; digits = ::Float64, kwargs_1...))[2] === Float64 # picks `3::Int` from `kwargs_1` + @test (@code_typed round(1.2; kwargs_1..., digits = ::Float64))[2] === Union{} # picks `::Float64` from parameters + @test (@which round(1.2; digits = ::Float64, kwargs_1...)).name === :round + @test (@which round(1.2; sigdigits = ::Int, kwargs_1...)).name === :round + @test (@which round(1.2; kwargs_1..., kwargs_2..., base)).name === :round +end + module MacroTest -export @macrotest +var"@which" = parentmodule(@__MODULE__).var"@which" macro macrotest(x::Int, y::Symbol) end -macro macrotest(x::Int, y::Int) - nothing #This is here because of #15280 -end +macro macrotest(x::Int, y::Int) end end let - using .MacroTest a = 1 - m = getfield(@__MODULE__, Symbol("@macrotest")) - @test which(m, Tuple{LineNumberNode, Module, Int, Symbol}) == @which @macrotest 1 a - @test which(m, Tuple{LineNumberNode, Module, Int, Int}) == @which @macrotest 1 1 + m = MacroTest.var"@macrotest" + @test which(m, Tuple{LineNumberNode, Module, Int, Symbol}) == @eval MacroTest @which @macrotest 1 a + @test which(m, Tuple{LineNumberNode, Module, Int, Int}) == @eval MacroTest @which @macrotest 1 1 @test first(methods(m, Tuple{LineNumberNode, Module, Int, Int})) == @which MacroTest.@macrotest 1 1 - @test functionloc(@which @macrotest 1 1) == @functionloc @macrotest 1 1 + @test functionloc(@eval MacroTest @which @macrotest 1 1) == @functionloc MacroTest.@macrotest 1 1 end mutable struct A18434 @@ -574,6 +615,9 @@ end @test_throws err @code_lowered "" @test_throws err @code_lowered 1 @test_throws err @code_lowered 1.0 + + @test_throws "is too complex" @code_lowered a .= 1 + 2 + @test_throws "invalid keyword argument syntax" @eval @which round(1; digits(3)) end using InteractiveUtils: editor @@ -827,6 +871,12 @@ end @test Base.infer_exception_type(sin, (Int,)) == InteractiveUtils.@infer_exception_type sin(42) @test first(InteractiveUtils.@code_ircode sin(42)) isa Core.Compiler.IRCode @test first(InteractiveUtils.@code_ircode optimize_until="CC: INLINING" sin(42)) isa Core.Compiler.IRCode +# Test.@inferred also uses `gen_call_with_extracted_types` +@test Test.@inferred round(1.2) isa Float64 +@test Test.@inferred round(1.3; digits = 3) isa Float64 +# ensure proper inference of the macro output of `@inferred` +@test Base.infer_return_type(x -> Test.@inferred(round(x)), (Float64,)) === Float64 +@test Base.infer_return_type(x -> Test.@inferred(round(x; digits = 3)), (Float64,)) === Float64 @testset "Docstrings" begin @test isempty(Docs.undocumented_names(InteractiveUtils)) From 5c869cd363da605f51454be03c98fe4c1ec4ba5b Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:58:02 +0900 Subject: [PATCH 101/662] inference: minor refactoring on abstractinterpretation.jl (#58154) - add missing `@nospecialize` annotation - add more type annotations to functions - fixed capturing uninferrable variables --- Compiler/src/abstractinterpretation.jl | 39 +++++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 05a857babaa55..4f93f2d4df913 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2437,6 +2437,7 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s end @nospecs function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, M, s) + @nospecialize M s ⊑ = partialorder(typeinf_lattice(interp)) if isa(M, Const) && isa(s, Const) (M, s) = (M.val, s.val) @@ -2444,7 +2445,7 @@ end return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) end gr = GlobalRef(M, s) - (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition + (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition local rt kind = binding_kind(partition) if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED @@ -2574,13 +2575,14 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) gr = GlobalRef(M, s) - (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, binding, partition + v′ = RefValue{Any}(v) + (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition partition_T = nothing partition_rte = abstract_eval_partition_load(interp, binding, partition) if binding_kind(partition) == PARTITION_KIND_GLOBAL partition_T = partition_restriction(partition) end - partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]} + partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]} partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) Pair{RTEffects, Any}(partition_rte, partition_T) end @@ -3558,7 +3560,7 @@ function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::Global return partition end -function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, binding::Core.Binding, partition::Core.BindingPartition) +function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing}, binding::Core.Binding, partition::Core.BindingPartition) kind = binding_kind(partition) isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) @@ -3607,7 +3609,7 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing return RTEffects(rt, exct, effects) end -function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp, g::GlobalRef, wwr::WorldWithRange) +function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) local total_validity, rte, binding_partition binding = convert(Core.Binding, g) lookup_world = max_world(wwr.valid_worlds) @@ -3640,19 +3642,25 @@ function scan_specified_partitions(query::Function, walk_binding_partition::Func return Pair{WorldRange, typeof(rte)}(total_validity, rte) end -scan_leaf_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = +scan_leaf_partitions(query::Function, ::Nothing, g::GlobalRef, wwr::WorldWithRange) = + scan_specified_partitions(query, walk_binding_partition, nothing, g, wwr) +scan_leaf_partitions(query::Function, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) = scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) -scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = - scan_specified_partitions(query, - (b::Core.Binding, bpart::Core.BindingPartition, world::UInt)-> - Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(WorldRange(bpart.min_world, bpart.max_world), b=>bpart), - interp, g, wwr) +function scan_partitions(query::Function, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) + walk_binding_partition = function (b::Core.Binding, partition::Core.BindingPartition, world::UInt) + Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}( + WorldRange(partition.min_world, partition.max_world), b=>partition) + end + return scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) +end -abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) = +abstract_load_all_consistent_leaf_partitions(interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) +abstract_load_all_consistent_leaf_partitions(::Nothing, g::GlobalRef, wwr::WorldWithRange) = + scan_leaf_partitions(abstract_eval_partition_load, nothing, g, wwr) -function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) +function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) if saw_latestworld return RTEffects(Any, Any, generic_getglobal_effects) end @@ -3668,7 +3676,10 @@ function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState, if saw_latestworld return Pair{Any,Any}(newty, ErrorException) end - (valid_worlds, ret) = scan_partitions((interp, _, partition)->global_assignment_binding_rt_exct(interp, partition, newty), interp, g, sv.world) + newty′ = RefValue{Any}(newty) + (valid_worlds, ret) = scan_partitions(interp, g, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition + global_assignment_binding_rt_exct(interp, partition, newty′[]) + end update_valid_age!(sv, valid_worlds) return ret end From 73410981c85cf6e8005922439bef554cd06b3b0d Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Sat, 19 Apr 2025 02:18:31 -0400 Subject: [PATCH 102/662] Fix `jl_set_precompile_field_replace` for 0-size fields (#58155) --- src/staticdata.c | 4 ++-- test/precompile.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index c96faf5a71a54..18f7cbabfbf51 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1767,7 +1767,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED tot = offset; size_t fsz = jl_field_size(t, i); jl_value_t *replace = (jl_value_t*)ptrhash_get(&bits_replace, (void*)slot); - if (replace != HT_NOTFOUND) { + if (replace != HT_NOTFOUND && fsz > 0) { assert(t->name->mutabl && !jl_field_isptr(t, i)); jl_value_t *rty = jl_typeof(replace); size_t sz = jl_datatype_size(rty); @@ -2961,7 +2961,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, if (jl_field_isptr(st, field)) { record_field_change((jl_value_t**)fldaddr, newval); } - else { + else if (jl_field_size(st, field) > 0) { // replace the bits ptrhash_put(&bits_replace, (void*)fldaddr, newval); // and any pointers inside diff --git a/test/precompile.jl b/test/precompile.jl index 07384e66927a4..18e66e3172d2d 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -105,6 +105,11 @@ precompile_test_harness(false) do dir process_state_calls = 0 @assert process_state() === process_state() @assert process_state_calls === 0 + + const empty_state = Base.OncePerProcess{Nothing}() do + return nothing + end + @assert empty_state() === nothing end """) write(Foo2_file, From da994616fd20517901d252a2f1bb0f0e6a0ac4cb Mon Sep 17 00:00:00 2001 From: Nathan Zimmerberg <39104088+nhz2@users.noreply.github.com> Date: Sat, 19 Apr 2025 02:19:38 -0400 Subject: [PATCH 103/662] Fix use of `pointer` in `write(to::IO, from::GenericIOBuffer)` (#57949) --- base/iobuffer.jl | 4 ++-- test/iobuffer.jl | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index c115def5d9ade..a28fe5376755b 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -811,8 +811,8 @@ function write(to::IO, from::GenericIOBuffer) if to === from throw(ArgumentError("Writing all content fron an IOBuffer into itself in invalid")) else - available = bytesavailable(from) - written = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(available)) + from.readable || _throw_not_readable() + written = write(to, view(from.data, from.ptr:from.size)) from.ptr = from.size + 1 end return written diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 2b84fe79307ae..428e259f225fc 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -265,6 +265,12 @@ end write(to, from) @test String(take!(to)) == "abcd" @test eof(from) + + # Write to another IOBuffer when closed + to = IOBuffer() + from = IOBuffer(collect(b"abcdefghi")) + close(from) + @test_throws ArgumentError write(to, from) end @testset "Read/write empty IOBuffer" begin @@ -701,6 +707,11 @@ end data = 0x00:0xFF io = IOBuffer(data) @test read(io) == data + seekstart(io) + @test read(io, UInt16) === ltoh(0x0100) + out = IOBuffer() + write(out, io) + @test take!(out) == data[3:end] data = @view(collect(0x00:0x0f)[begin:2:end]) io = IOBuffer(data) From 0947114d9d443e14c751ac40c9b2d8c2245d045e Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Sat, 19 Apr 2025 09:05:09 -0400 Subject: [PATCH 104/662] Switch from segfault to `zip` behavior for mismatched indices in `map!` (#56673) --- base/abstractarray.jl | 19 +++++++++++++------ test/abstractarray.jl | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index bc41591ec0cae..1c1e7ebb5c3ee 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3417,12 +3417,19 @@ function ith_all(i, as) end function map_n!(f::F, dest::AbstractArray, As) where F - idxs1 = LinearIndices(As[1]) - @boundscheck LinearIndices(dest) == idxs1 && all(x -> LinearIndices(x) == idxs1, As) - for i = idxs1 - @inbounds I = ith_all(i, As) - val = f(I...) - @inbounds dest[i] = val + idxs = LinearIndices(dest) + if all(x -> LinearIndices(x) == idxs, As) + for i in idxs + @inbounds as = ith_all(i, As) + val = f(as...) + @inbounds dest[i] = val + end + else + for (i, Is...) in zip(eachindex(dest), map(eachindex, As)...) + as = ntuple(j->getindex(As[j], Is[j]), length(As)) + val = f(as...) + dest[i] = val + end end return dest end diff --git a/test/abstractarray.jl b/test/abstractarray.jl index e4dcb1959ec18..b882778e4b152 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -910,6 +910,26 @@ include("generic_map_tests.jl") generic_map_tests(map, map!) @test map!(-, [1]) == [-1] +@testset "#30624" begin + ### unstructured + @test map!(+, ones(3), ones(3), ones(3), [1]) == [3, 1, 1] + @test map!(+, ones(3), [1], ones(3), ones(3)) == [3, 1, 1] + @test map!(+, [1], [1], [], []) == [1] + @test map!(+, [[1]], [1], [], []) == [[1]] + + # TODO: decide if input axes & lengths should be validated + # @test_throws BoundsError map!(+, ones(1), ones(2)) + # @test_throws BoundsError map!(+, ones(1), ones(2, 2)) + + @test map!(+, ones(3), view(ones(2, 3), 1:2, 2:3), ones(3)) == [2, 2, 2] + @test map!(+, ones(3), ones(2, 2), ones(3)) == [2, 2, 2] + + ### structured (all mapped arguments are <:AbstractArray equal ndims > 1) + @test map!(+, ones(4), ones(2, 2), ones(2, 2)) == [2, 2, 2, 2] + @test map!(+, ones(4), ones(2, 2), ones(1, 2)) == [2, 2, 1, 1] + # @test_throws BoundsError map!(+, ones(3), ones(2, 2), ones(2, 2)) +end + test_UInt_indexing(TestAbstractArray) test_13315(TestAbstractArray) test_checksquare() From 828e3a1a7b56ab746b721ba110b4b7bd17cb815b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 19 Apr 2025 12:17:34 -0400 Subject: [PATCH 105/662] staticdata: fix many mistakes in staticdata stripping (#58166) While #58156 was supposed to be fairly trivial, it ended up uncovering many bugs in the unusual ways that trimming and binding partition had been added to the serialization format. This attempts to reorganize them to be handled more consistently and with fewer mistakes. --- contrib/generate_precompile.jl | 2 +- src/jltypes.c | 4 +- src/staticdata.c | 277 +++++++++++++++++++-------------- test/core.jl | 2 + test/trimming/Makefile | 18 +-- 5 files changed, 178 insertions(+), 125 deletions(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index fc229c394ca02..2292eb52fa2b3 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Prevent this from putting anything into the Main namespace -@eval Core.Module() begin +@eval Base module __precompile_script if Threads.maxthreadid() != 1 @warn "Running this file with multiple Julia threads may lead to a build error" Threads.maxthreadid() diff --git a/src/jltypes.c b/src/jltypes.c index ebef8e8df1747..98addd89e5ce5 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3266,9 +3266,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(5, jl_any_type, jl_ulong_type, jl_ulong_type, jl_any_type/*jl_binding_partition_type*/, jl_ulong_type), jl_emptysvec, 0, 1, 0); - const static uint32_t binding_partition_atomicfields[] = { 0b01111 }; // Set fields 1, 2, 3, 4 as atomic + const static uint32_t binding_partition_atomicfields[] = { 0b01110 }; // Set fields 2, 3, 4 as atomic jl_binding_partition_type->name->atomicfields = binding_partition_atomicfields; - const static uint32_t binding_partition_constfields[] = { 0x100001 }; // Set fields 1, 5 as constant + const static uint32_t binding_partition_constfields[] = { 0b10001 }; // Set fields 1, 5 as constant jl_binding_partition_type->name->constfields = binding_partition_constfields; jl_binding_type = diff --git a/src/staticdata.c b/src/staticdata.c index 18f7cbabfbf51..89708382eb148 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -576,7 +576,6 @@ typedef struct { static jl_value_t *jl_bigint_type = NULL; static int gmp_limb_size = 0; -static jl_sym_t *jl_docmeta_sym = NULL; #ifdef _P64 #define RELOC_TAG_OFFSET 61 @@ -786,39 +785,31 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ jl_queue_for_serialization(s, m->parent); if (!jl_options.strip_metadata) jl_queue_for_serialization(s, m->file); + jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset)); if (jl_options.trim) { jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); - } else { - jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); - } - jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset)); - if (jl_options.strip_metadata || jl_options.trim) { jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); for (size_t i = 0; i < jl_svec_len(table); i++) { jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); if ((void*)b == jl_nothing) break; - if (jl_options.strip_metadata) { - jl_sym_t *name = b->globalref->name; - if (name == jl_docmeta_sym && jl_get_binding_value(b)) - record_field_change((jl_value_t**)&b->value, jl_nothing); - } - if (jl_options.trim) { - jl_value_t *val = jl_get_binding_value(b); - // keep binding objects that are defined and ... - if (val && - // ... point to modules ... - (jl_is_module(val) || - // ... or point to __init__ methods ... - !strcmp(jl_symbol_name(b->globalref->name), "__init__") || - // ... or point to Base functions accessed by the runtime - (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || - !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { - jl_queue_for_serialization(s, b); - } + jl_value_t *val = jl_get_binding_value_in_world(b, jl_atomic_load_relaxed(&jl_world_counter)); + // keep binding objects that are defined in the latest world and ... + if (val && + // ... point to modules ... + (jl_is_module(val) || + // ... or point to __init__ methods ... + !strcmp(jl_symbol_name(b->globalref->name), "__init__") || + // ... or point to Base functions accessed by the runtime + (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || + !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { + jl_queue_for_serialization(s, b); } } } + else { + jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); + } for (size_t i = 0; i < module_usings_length(m); i++) { jl_queue_for_serialization(s, module_usings_getmod(m, i)); @@ -863,40 +854,51 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } - if (s->incremental && jl_is_method_instance(v)) { + if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; - jl_value_t *def = mi->def.value; - if (needs_uniquing(v, s->query_cache)) { - // we only need 3 specific fields of this (the rest are not used) - jl_queue_for_serialization(s, mi->def.value); - jl_queue_for_serialization(s, mi->specTypes); - jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); - goto done_fields; - } - else if (jl_is_method(def) && jl_object_in_image(def)) { - // we only need 3 specific fields of this (the rest are restored afterward, if valid) - // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, - // so must not be present here - record_field_change((jl_value_t**)&mi->backedges, NULL); - record_field_change((jl_value_t**)&mi->cache, NULL); + if (s->incremental) { + jl_value_t *def = mi->def.value; + if (needs_uniquing(v, s->query_cache)) { + // we only need 3 specific fields of this (the rest are not used) + jl_queue_for_serialization(s, mi->def.value); + jl_queue_for_serialization(s, mi->specTypes); + jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); + goto done_fields; + } + else if (jl_is_method(def) && jl_object_in_image(def)) { + // we only need 3 specific fields of this (the rest are restored afterward, if valid) + // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, + // so must not be present here + record_field_change((jl_value_t**)&mi->backedges, NULL); + record_field_change((jl_value_t**)&mi->cache, NULL); + } + else { + assert(!needs_recaching(v, s->query_cache)); + } + // n.b. opaque closures cannot be inspected and relied upon like a + // normal method since they can get improperly introduced by generated + // functions, so if they appeared at all, we will probably serialize + // them wrong and segfault. The jl_code_for_staged function should + // prevent this from happening, so we do not need to detect that user + // error now. } - else { - assert(!needs_recaching(v, s->query_cache)); + } + if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + if (jl_options.trim || jl_options.strip_ir) { + record_field_change((jl_value_t**)&mt->backedges, NULL); } - // n.b. opaque closures cannot be inspected and relied upon like a - // normal method since they can get improperly introduced by generated - // functions, so if they appeared at all, we will probably serialize - // them wrong and segfault. The jl_code_for_staged function should - // prevent this from happening, so we do not need to detect that user - // error now. - } - if (s->incremental && jl_is_binding(v)) { - if (needs_uniquing(v, s->query_cache)) { - jl_binding_t *b = (jl_binding_t*)v; + } + if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + if (s->incremental && needs_uniquing(v, s->query_cache)) { jl_queue_for_serialization(s, b->globalref->mod); jl_queue_for_serialization(s, b->globalref->name); goto done_fields; } + if (jl_options.trim || jl_options.strip_ir) { + record_field_change((jl_value_t**)&b->backedges, NULL); + } } if (s->incremental && jl_is_globalref(v)) { jl_globalref_t *gr = (jl_globalref_t*)v; @@ -1037,11 +1039,6 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ else if (jl_typetagis(v, jl_module_tag << 4)) { jl_queue_module_for_serialization(s, (jl_module_t*)v); } - else if (jl_is_binding_partition(v)) { - jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; - jl_queue_for_serialization_(s, bpart->restriction, 1, immediate); - jl_queue_for_serialization_(s, get_replaceable_field((jl_value_t**)&bpart->next, 0), 1, immediate); - } else if (layout->nfields > 0) { if (jl_options.trim) { if (jl_is_method(v)) { @@ -1059,15 +1056,23 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ record_field_change((jl_value_t **)&tn->mt, NULL); } } - else if (jl_is_binding(v)) { - record_field_change((jl_value_t**)&((jl_binding_t*)v)->backedges, NULL); - } + // TODO: prune any partitions and partition data that has been deleted in the current world + //else if (jl_is_binding(v)) { + // jl_binding_t *b = (jl_binding_t*)v; + //} + //else if (jl_is_binding_partition(v)) { + // jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; + //} } char *data = (char*)jl_data_ptr(v); size_t i, np = layout->npointers; + size_t fldidx = 1; for (i = 0; i < np; i++) { uint32_t ptr = jl_ptr_offset(t, i); - int mutabl = t->name->mutabl; + size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*); + while (offset >= (fldidx == layout->nfields ? jl_datatype_size(t) : jl_field_offset(t, fldidx))) + fldidx++; + int mutabl = !jl_field_isconst(t, fldidx - 1); jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], mutabl); jl_queue_for_serialization_(s, fld, 1, immediate); } @@ -1519,7 +1524,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (s->incremental) { if (needs_uniquing(v, s->query_cache)) { - if (jl_typetagis(v, jl_binding_type)) { + if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; if (b->globalref == NULL) jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity @@ -1729,32 +1734,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED ios_write(s->const_data, (char*)pdata, nb); write_pointer(f); } - else if (jl_is_binding_partition(v)) { - jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; - write_pointerfield(s, bpart->restriction); - size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (s->incremental) { - if (max_world == ~(size_t)0) { - // Still valid. Will be considered to be defined in jl_require_world - // after reload, which is the first world before new code runs. - // We use this as a quick check to determine whether a binding was - // invalidated. If a binding was first defined in or before - // jl_require_world, then we can assume that all precompile processes - // will have seen it consistently. - write_uint(f, jl_require_world); - write_uint(f, max_world); - } else { - // The world will not be reachable after loading - write_uint(f, 1); - write_uint(f, 0); - } - } else { - write_uint(f, jl_atomic_load_relaxed(&bpart->min_world)); - write_uint(f, max_world); - } - write_pointerfield(s, (jl_value_t*)jl_atomic_load_relaxed(&bpart->next)); - write_uint(f, bpart->kind); - } else { // Generic object::DataType serialization by field const char *data = (const char*)v; @@ -1794,9 +1773,12 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } size_t np = t->layout->npointers; + size_t fldidx = 1; for (i = 0; i < np; i++) { size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*); - int mutabl = t->name->mutabl; + while (offset >= (fldidx == nf ? jl_datatype_size(t) : jl_field_offset(t, fldidx))) + fldidx++; + int mutabl = !jl_field_isconst(t, fldidx - 1); jl_value_t *fld = get_replaceable_field((jl_value_t**)&data[offset], mutabl); size_t fld_pos = offset + reloc_offset; if (fld != NULL) { @@ -1816,15 +1798,33 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_typemap_entry_t *newentry = (jl_typemap_entry_t*)&s->s->buf[reloc_offset]; if (jl_atomic_load_relaxed(&newentry->max_world) == ~(size_t)0) { if (jl_atomic_load_relaxed(&newentry->min_world) > 1) { - jl_atomic_store_release(&newentry->min_world, ~(size_t)0); - jl_atomic_store_release(&newentry->max_world, WORLD_AGE_REVALIDATION_SENTINEL); + jl_atomic_store_relaxed(&newentry->min_world, ~(size_t)0); + jl_atomic_store_relaxed(&newentry->max_world, WORLD_AGE_REVALIDATION_SENTINEL); arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } else { // garbage newentry - delete it :( - jl_atomic_store_release(&newentry->min_world, 1); - jl_atomic_store_release(&newentry->max_world, 0); + jl_atomic_store_relaxed(&newentry->min_world, 1); + jl_atomic_store_relaxed(&newentry->max_world, 0); + } + } + else if (s->incremental && jl_is_binding_partition(v)) { + jl_binding_partition_t *newbpart = (jl_binding_partition_t*)&s->s->buf[reloc_offset]; + size_t max_world = jl_atomic_load_relaxed(&newbpart->max_world); + if (max_world == ~(size_t)0) { + // Still valid. Will be considered to be defined in jl_require_world + // after reload, which is the first world before new code runs. + // We use this as a quick check to determine whether a binding was + // invalidated. If a binding was first defined in or before + // jl_require_world, then we can assume that all precompile processes + // will have seen it consistently. + jl_atomic_store_relaxed(&newbpart->min_world, jl_require_world); + } + else { + // The world will not be reachable after loading + jl_atomic_store_relaxed(&newbpart->min_world, 1); + jl_atomic_store_relaxed(&newbpart->max_world, 0); } } else if (jl_is_method(v)) { @@ -2572,6 +2572,7 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } + uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED @@ -2587,15 +2588,12 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED if (ti == jl_nothing) continue; jl_binding_t *ref = ((jl_binding_t*)ti); - if (!((ptrhash_get(&serialization_order, ref) == HT_NOTFOUND) && - (ptrhash_get(&serialization_order, ref->globalref) == HT_NOTFOUND))) { - jl_svecset(bindings, i, jl_nothing); + if (ptrhash_get(&serialization_order, ref) != HT_NOTFOUND) arraylist_push(&bindings_list, ref); - } } - jl_genericmemory_t* bindingkeyset = jl_atomic_load_relaxed(&m->bindingkeyset); + jl_genericmemory_t *bindingkeyset = jl_atomic_load_relaxed(&m->bindingkeyset); _Atomic(jl_genericmemory_t*)bindingkeyset2; - jl_atomic_store_relaxed(&bindingkeyset2,(jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&bindingkeyset2, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_svec_t *bindings2 = jl_alloc_svec_uninit(bindings_list.len); for (i = 0; i < bindings_list.len; i++) { jl_binding_t *ref = (jl_binding_t*)bindings_list.items[i]; @@ -2676,7 +2674,7 @@ static void strip_specializations_(jl_method_instance_t *mi) record_field_change((jl_value_t**)&codeinst->debuginfo, (jl_value_t*)jl_nulldebuginfo); codeinst = jl_atomic_load_relaxed(&codeinst->next); } - if (jl_options.strip_ir) { + if (jl_options.trim || jl_options.strip_ir) { record_field_change((jl_value_t**)&mi->backedges, NULL); } } @@ -2749,8 +2747,6 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) static int strip_all_codeinfos_(jl_methtable_t *mt, void *_env) { - if (jl_options.strip_ir && mt->backedges) - record_field_change((jl_value_t**)&mt->backedges, NULL); return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), strip_all_codeinfos__, NULL); } @@ -2759,6 +2755,63 @@ static void jl_strip_all_codeinfos(void) jl_foreach_reachable_mtable(strip_all_codeinfos_, NULL); } +static int strip_module(jl_module_t *m, jl_sym_t *docmeta_sym) +{ + size_t world = jl_atomic_load_relaxed(&jl_world_counter); + jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); + for (size_t i = 0; i < jl_svec_len(table); i++) { + jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); + if ((void*)b == jl_nothing) + break; + jl_sym_t *name = b->globalref->name; + jl_value_t *v = jl_get_binding_value_in_world(b, world); + if (v) { + if (jl_is_module(v)) { + jl_module_t *child = (jl_module_t*)v; + if (child != m && child->parent == m && child->name == name) { + // this is the original/primary binding for the submodule + if (!strip_module(child, docmeta_sym)) + return 0; + } + } + } + if (name == docmeta_sym) { + if (jl_atomic_load_relaxed(&b->value)) + record_field_change((jl_value_t**)&b->value, jl_nothing); + // TODO: this is a pretty stupidly unsound way to do this, but it is way to late here to do this correctly (by calling delete_binding and getting an updated world age then dropping all partitions from older worlds) + jl_binding_partition_t *bp = jl_atomic_load_relaxed(&b->partitions); + while (bp) { + if (jl_bkind_is_defined_constant(jl_binding_kind(bp))) { + // XXX: bp->kind = PARTITION_KIND_UNDEF_CONST; + record_field_change((jl_value_t**)&bp->restriction, NULL); + } + bp = jl_atomic_load_relaxed(&bp->next); + } + } + } + return 1; +} + + +static void jl_strip_all_docmeta(jl_array_t *mod_array) +{ + jl_sym_t *docmeta_sym = NULL; + if (jl_base_module) { + jl_value_t *docs = jl_get_global(jl_base_module, jl_symbol("Docs")); + if (docs && jl_is_module(docs)) { + docmeta_sym = (jl_sym_t*)jl_get_global((jl_module_t*)docs, jl_symbol("META")); + } + } + if (!docmeta_sym) + return; + for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { + jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); + assert(jl_is_module(m)); + if (m->parent == m) // some toplevel modules (really just Base) aren't actually + strip_module(m, docmeta_sym); + } +} + // --- entry points --- jl_genericmemory_t *jl_global_roots_list; @@ -2934,8 +2987,10 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, htable_new(&field_replace, 0); htable_new(&bits_replace, 0); // strip metadata and IR when requested - if (jl_options.strip_metadata || jl_options.strip_ir) + if (jl_options.strip_metadata || jl_options.strip_ir) { jl_strip_all_codeinfos(); + jl_strip_all_docmeta(mod_array); + } // collect needed methods and replace method tables that are in the tags array htable_new(&new_methtables, 0); arraylist_t MIs; @@ -3081,12 +3136,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, gmp_limb_size = jl_unbox_long(jl_get_global((jl_module_t*)jl_get_global(jl_base_module, jl_symbol("GMP")), jl_symbol("BITS_PER_LIMB"))) / 8; } - if (jl_base_module) { - jl_value_t *docs = jl_get_global(jl_base_module, jl_symbol("Docs")); - if (docs && jl_is_module(docs)) { - jl_docmeta_sym = (jl_sym_t*)jl_get_global((jl_module_t*)docs, jl_symbol("META")); - } - } jl_genericmemory_t *global_roots_list = NULL; jl_genericmemory_t *global_roots_keyset = NULL; @@ -3145,16 +3194,18 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_queue_for_serialization(&s, global_roots_keyset); jl_serialize_reachable(&s); } - // step 1.5: prune (garbage collect) some special weak references from - // built-in type caches too + // step 1.5: prune (garbage collect) some special weak references known caches for (i = 0; i < serialization_queue.len; i++) { jl_value_t *v = (jl_value_t*)serialization_queue.items[i]; if (jl_options.trim) { - if (jl_is_method(v)){ + if (jl_is_method(v)) { jl_method_t *m = (jl_method_t*)v; jl_value_t *specializations_ = jl_atomic_load_relaxed(&m->specializations); - if (!jl_is_svec(specializations_)) + if (!jl_is_svec(specializations_)) { + if (ptrhash_get(&serialization_order, specializations_) == HT_NOTFOUND) + record_field_change((jl_value_t **)&m->specializations, (jl_value_t*)jl_emptysvec); continue; + } jl_svec_t *specializations = (jl_svec_t *)specializations_; size_t l = jl_svec_len(specializations), i; @@ -3397,8 +3448,8 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli jl_query_cache query_cache; init_query_cache(&query_cache); + mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) if (worklist) { - mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) // Generate _native_data` if (_native_data != NULL) { jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL, &query_cache); diff --git a/test/core.jl b/test/core.jl index 8f4696346aef4..f6cc5ae9f9f7a 100644 --- a/test/core.jl +++ b/test/core.jl @@ -26,6 +26,7 @@ for (T, c) in ( (Core.Memory, [:length, :ptr]), (Core.GenericMemoryRef, [:mem, :ptr_or_offset]), (Task, [:metrics_enabled]), + (Core.BindingPartition, [:restriction, :kind]), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if isconst(T, i))) == Set(c) end @@ -44,6 +45,7 @@ for (T, c) in ( (Core.Memory, []), (Core.GenericMemoryRef, []), (Task, [:_state, :running_time_ns, :finished_at, :first_enqueued_at, :last_started_running_at]), + (Core.BindingPartition, [:min_world, :max_world, :next]), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if Base.isfieldatomic(T, i))) == Set(c) end diff --git a/test/trimming/Makefile b/test/trimming/Makefile index 02e9fb5bdb257..c12fe8ee04c56 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -32,27 +32,27 @@ LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal release: hello$(EXE) basic_jll$(EXE) -hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe true +hello-o.a: $(SRCDIR)/hello.jl $(BUILDSCRIPT) + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true init.o: $(SRCDIR)/init.c $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) -basic_jll.o: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) +basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/basic_jll.jl --output-exe true + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true -hello$(EXE): hello.o init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) hello.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +hello$(EXE): hello-o.a init.o + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -basic_jll$(EXE): basic_jll.o init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) basic_jll.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +basic_jll$(EXE): basic_jll-o.a init.o + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) check: hello$(EXE) basic_jll$(EXE) $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming clean: - -rm -f hello$(EXE) basic_jll$(EXE) init.o hello.o basic_jll.o + -rm -f hello$(EXE) basic_jll$(EXE) init.o hello-o.a basic_jll-o.a .PHONY: release clean check From 3ea60d2274524369823efc7e39067570ba2cbb38 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 19 Apr 2025 21:37:34 -0400 Subject: [PATCH 106/662] staticdata: prune backedges table during serialization (#58156) After the bindings change, there is quite a lot of garbage in here now at runtime (it does not de-duplicate entries added for bindings), so attempt to do a bit of that during serialization. Re-landing part of #58078 --- src/staticdata.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/staticdata.c b/src/staticdata.c index 89708382eb148..6916bfe00ba5f 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -882,12 +882,38 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ // prevent this from happening, so we do not need to detect that user // error now. } + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + if (backedges) { + assert(!jl_options.trim && !jl_options.strip_ir); + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + size_t i = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge((jl_array_t*)backedges, i, &invokeTypes, &caller); + if (invokeTypes) + jl_queue_for_serialization(s, invokeTypes); + } + } } if (jl_is_mtable(v)) { jl_methtable_t *mt = (jl_methtable_t*)v; if (jl_options.trim || jl_options.strip_ir) { record_field_change((jl_value_t**)&mt->backedges, NULL); } + else { + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { + jl_value_t *t = jl_array_ptr_ref(backedges, i); + assert(!jl_is_code_instance(t)); + jl_queue_for_serialization(s, t); + } + } + } } if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; @@ -899,6 +925,18 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (jl_options.trim || jl_options.strip_ir) { record_field_change((jl_value_t**)&b->backedges, NULL); } + else { + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i++) { + jl_value_t *b = jl_array_ptr_ref(backedges, i); + if (!jl_is_code_instance(b) && !jl_is_method_instance(b) && !jl_is_method(b)) // otherwise usually a Binding? + jl_queue_for_serialization(s, b); + } + } + } } if (s->incremental && jl_is_globalref(v)) { jl_globalref_t *gr = (jl_globalref_t*)v; @@ -2572,6 +2610,52 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } +static void jl_prune_mi_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge(backedges, i, &invokeTypes, &caller); + if (ptrhash_get(&serialization_order, caller) != HT_NOTFOUND) + ins = set_next_edge(backedges, ins, invokeTypes, caller); + } + jl_array_del_end(backedges, n - ins); +} + +static void jl_prune_mt_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + for (i = 1; i < n; i += 2) { + jl_value_t *ci = jl_array_ptr_ref(backedges, i); + if (ptrhash_get(&serialization_order, ci) != HT_NOTFOUND) { + jl_array_ptr_set(backedges, ins++, jl_array_ptr_ref(backedges, i - 1)); + jl_array_ptr_set(backedges, ins++, ci); + } + } + jl_array_del_end(backedges, n - ins); +} + + +static void jl_prune_binding_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + for (i = 0; i < n; i++) { + jl_value_t *b = jl_array_ptr_ref(backedges, i); + if (ptrhash_get(&serialization_order, b) != HT_NOTFOUND) { + jl_array_ptr_set(backedges, ins, b); + ins++; + } + } + jl_array_del_end(backedges, n - ins); +} + uint_t bindingkey_hash(size_t idx, jl_value_t *data); @@ -3229,6 +3313,21 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); } + else if (jl_is_method_instance(v)) { + jl_method_instance_t *mi = (jl_method_instance_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + jl_prune_mi_backedges((jl_array_t*)backedges); + } + else if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + jl_prune_mt_backedges((jl_array_t*)backedges); + } + else if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + jl_prune_binding_backedges((jl_array_t*)backedges); + } } } From 334c3167e853daeb11e5121d2041de657e175726 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Sun, 20 Apr 2025 18:48:32 +0800 Subject: [PATCH 107/662] =?UTF-8?q?subtype:=20save=20some=20union=20stack?= =?UTF-8?q?=20space=20for=20=E2=88=83=20free=20cases.=20(#58159)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and avoid eager UnionAll unwrapping to hit more fast path. close #58129 (test passed locally) close #56350 (MWE returns `Tuple{Any, Any, Vararg}` now.) --- src/subtype.c | 34 +++++++++++++++++++++++++--------- test/subtype.jl | 14 ++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index c1c811d4e0ad5..ed6a8818dc3f2 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -139,7 +139,7 @@ static jl_varbinding_t *lookup(jl_stenv_t *e, jl_tvar_t *v) JL_GLOBALLY_ROOTED J static int statestack_get(jl_unionstate_t *st, int i) JL_NOTSAFEPOINT { - assert(i >= 0 && i <= 32767); // limited by the depth bit. + assert(i >= 0 && i < 32767); // limited by the depth bit. // get the `i`th bit in an array of 32-bit words jl_bits_stack_t *stack = &st->stack; while (i >= sizeof(stack->data) * 8) { @@ -153,7 +153,7 @@ static int statestack_get(jl_unionstate_t *st, int i) JL_NOTSAFEPOINT static void statestack_set(jl_unionstate_t *st, int i, int val) JL_NOTSAFEPOINT { - assert(i >= 0 && i <= 32767); // limited by the depth bit. + assert(i >= 0 && i < 32767); // limited by the depth bit. jl_bits_stack_t *stack = &st->stack; while (i >= sizeof(stack->data) * 8) { if (__unlikely(stack->next == NULL)) { @@ -1448,11 +1448,14 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param) } if (jl_is_unionall(y)) { jl_varbinding_t *xb = lookup(e, (jl_tvar_t*)x); - if (xb == NULL ? !e->ignore_free : !xb->right) { + jl_value_t *xub = xb == NULL ? ((jl_tvar_t *)x)->ub : xb->ub; + if ((xb == NULL ? !e->ignore_free : !xb->right) && xub != y) { // We'd better unwrap `y::UnionAll` eagerly if `x` isa ∀-var. // This makes sure the following cases work correct: // 1) `∀T <: Union{∃S, SomeType{P}} where {P}`: `S == Any` ==> `S >: T` // 2) `∀T <: Union{∀T, SomeType{P}} where {P}`: + // note: if xub == y we'd better try `subtype_var` as `subtype_left_var` + // hit `==` based fast path. return subtype_unionall(x, (jl_unionall_t*)y, e, 1, param); } } @@ -1590,6 +1593,8 @@ static int has_exists_typevar(jl_value_t *x, jl_stenv_t *e) JL_NOTSAFEPOINT return env != NULL && jl_has_bound_typevars(x, env); } +static int forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param); + static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param, int limit_slow) { int16_t oldRmore = e->Runions.more; @@ -1603,7 +1608,18 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t return jl_subtype(x, y); int has_exists = (!kindx && has_exists_typevar(x, e)) || (!kindy && has_exists_typevar(y, e)); - if (has_exists && (is_exists_typevar(x, e) != is_exists_typevar(y, e))) { + if (!has_exists) { + // We can use ∀_∃_subtype safely for ∃ free inputs. + // This helps to save some bits in union stack. + jl_saved_unionstate_t oldRunions; push_unionstate(&oldRunions, &e->Runions); + e->Lunions.used = e->Runions.used = 0; + e->Lunions.depth = e->Runions.depth = 0; + e->Lunions.more = e->Runions.more = 0; + sub = forall_exists_subtype(x, y, e, param); + pop_unionstate(&e->Runions, &oldRunions); + return sub; + } + if (is_exists_typevar(x, e) != is_exists_typevar(y, e)) { e->Lunions.used = 0; while (1) { e->Lunions.more = 0; @@ -1617,7 +1633,7 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t if (limit_slow == -1) limit_slow = kindx || kindy; jl_savedenv_t se; - save_env(e, &se, has_exists); + save_env(e, &se, 1); int count, limited = 0, ini_count = 0; jl_saved_unionstate_t latestLunions = {0, 0, 0, NULL}; while (1) { @@ -1635,13 +1651,13 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t limited = 1; if (!sub || !next_union_state(e, 0)) break; - if (limited || !has_exists || e->Runions.more == oldRmore) { + if (limited || e->Runions.more == oldRmore) { // re-save env and freeze the ∃decision for previous ∀Union // Note: We could ignore the rest `∃Union` decisions if `x` and `y` // contain no ∃ typevar, as they have no effect on env. ini_count = count; push_unionstate(&latestLunions, &e->Lunions); - re_save_env(e, &se, has_exists); + re_save_env(e, &se, 1); e->Runions.more = oldRmore; } } @@ -1649,12 +1665,12 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t break; assert(e->Runions.more > oldRmore); next_union_state(e, 1); - restore_env(e, &se, has_exists); // also restore Rdepth here + restore_env(e, &se, 1); // also restore Rdepth here e->Runions.more = oldRmore; } if (!sub) assert(e->Runions.more == oldRmore); - else if (limited || !has_exists) + else if (limited) e->Runions.more = oldRmore; free_env(&se); return sub; diff --git a/test/subtype.jl b/test/subtype.jl index d186262a6e1ba..da69022b1466e 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2768,3 +2768,17 @@ end Tuple{Type{Complex{T}} where T, Type{Complex{T}} where T, Type{String}}, Tuple{Type{Complex{T}}, Type{Complex{T}}, Type{String}} where T ) + +#issue 58129 +for k in 1:500 + @eval struct $(Symbol(:T58129, k)){T} end +end +let Tvar = TypeVar(:Tvar) + V = UnionAll(Tvar, Union{(@eval($(Symbol(:T58129, k)){$Tvar}) for k in 1:500)...}) + @test Set{<:V} <: AbstractSet{<:V} +end +let Tvar1 = TypeVar(:Tvar1), Tvar2 = TypeVar(:Tvar2) + V1 = UnionAll(Tvar1, Union{(@eval($(Symbol(:T58129, k)){$Tvar1}) for k in 1:100)...}) + V2 = UnionAll(Tvar2, Union{(@eval($(Symbol(:T58129, k)){$Tvar2}) for k in 1:100)...}) + @test Set{<:V2} <: AbstractSet{<:V1} +end From 39d748312492ae2cebc4a4103e42bfd5fa46fe0a Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Mon, 21 Apr 2025 04:07:46 +0900 Subject: [PATCH 108/662] inference: allow specialization of `scan_specified_partitions` (#58165) Changed the `::Function` signature to the `::F where F` pattern to allow specialization of `scan_specified_partitions`. The changes in `scan_leaf_partitions` and `scan_partitions` are not actually necessary because those methods are simple and are basically inlined and optimized down to calling `scan_specified_partitions`. However, considering the possibility of other code being added in the future and also for consistency, the same changes were applied. --- Compiler/src/abstractinterpretation.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 4f93f2d4df913..b14475edee613 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3609,7 +3609,8 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing} return RTEffects(rt, exct, effects) end -function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) +function scan_specified_partitions(query::F1, walk_binding_partition::F2, + interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) where {F1,F2} local total_validity, rte, binding_partition binding = convert(Core.Binding, g) lookup_world = max_world(wwr.valid_worlds) @@ -3642,12 +3643,12 @@ function scan_specified_partitions(query::Function, walk_binding_partition::Func return Pair{WorldRange, typeof(rte)}(total_validity, rte) end -scan_leaf_partitions(query::Function, ::Nothing, g::GlobalRef, wwr::WorldWithRange) = +scan_leaf_partitions(query::F, ::Nothing, g::GlobalRef, wwr::WorldWithRange) where F = scan_specified_partitions(query, walk_binding_partition, nothing, g, wwr) -scan_leaf_partitions(query::Function, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) = +scan_leaf_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F = scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) -function scan_partitions(query::Function, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) +function scan_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F walk_binding_partition = function (b::Core.Binding, partition::Core.BindingPartition, world::UInt) Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}( WorldRange(partition.min_world, partition.max_world), b=>partition) From 07dbfc665abf215592801d9c40d2f70594ac5294 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sun, 20 Apr 2025 18:07:39 -0300 Subject: [PATCH 109/662] Update stable version in README (#58174) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f1b8c8c3002f..cbd1b11e982cb 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ and then use the command prompt to change into the resulting julia directory. By Julia. However, most users should use the [most recent stable version](https://github.com/JuliaLang/julia/releases) of Julia. You can get this version by running: - git checkout v1.11.2 + git checkout v1.11.5 To build the `julia` executable, run `make` from within the julia directory. From f2641f86bd8de494c6a398ad561daffd12f82870 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 21 Apr 2025 01:29:29 +0000 Subject: [PATCH 110/662] Only define `libssp` if our CSL intsall actually has it Some CSL installs (such as those from `USE_BINARYBUILDER=0` builds) do not have a `libssp`, because their GCC was built without stack smash protection, or it was statically linked. This allows our CSL_jll to adapt to that, avoiding issues like those reported in [0]. [0]: https://github.com/JuliaLang/julia/issues/58173 --- .../src/CompilerSupportLibraries_jll.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl index b32a229d653ed..1f5c34c9c10ee 100644 --- a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl +++ b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl @@ -72,7 +72,10 @@ end const libgfortran = LazyLibrary(_libgfortran_path, dependencies=libgfortran_deps) const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=[libgcc_s]) const libgomp = LazyLibrary(_libgomp_path) -if @isdefined _libssp_path + +# Some installations (such as those from-source) may not have `libssp` +# So let's do a compile-time check to see if we've got it. +if @isdefined(_libssp_path) && isfile(string(_libssp_path)) const libssp = LazyLibrary(_libssp_path) end From fb31b3c8a3ba6ff6ed5bda0caf83694a3456796c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 21 Apr 2025 15:36:35 -0400 Subject: [PATCH 111/662] gf: make dispatch heuristic representation explicit (#58167) Makes this query more accurate for the rare nonfunction_mt dispatch, but otherwise not expected to be a visible change. (needed for future followup work, so splitting this out into a small change since it can be done independently) --- src/datatype.c | 1 + src/gf.c | 15 ++++++++++++--- src/jltypes.c | 14 ++++++++------ src/julia.h | 1 + test/core.jl | 2 +- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index 1fd7cf4b502f9..cbaf6e98993d9 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -80,6 +80,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; + jl_atomic_store_relaxed(&tn->cache_entry_count, 0); tn->max_methods = 0; tn->constprop_heustic = 0; return tn; diff --git a/src/gf.c b/src/gf.c index c8de6d03449cd..f01f48d6115cf 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1618,6 +1618,15 @@ jl_method_instance_t *cache_method( } else { jl_typemap_insert(cache, parent, newentry, offs); + if (mt) { + jl_datatype_t *dt = jl_nth_argument_datatype((jl_value_t*)tt, 1); + if (dt) { + jl_typename_t *tn = dt->name; + int cache_entry_count = jl_atomic_load_relaxed(&tn->cache_entry_count); + if (cache_entry_count < 31) + jl_atomic_store_relaxed(&tn->cache_entry_count, cache_entry_count + 1); + } + } } JL_GC_POP(); @@ -3620,9 +3629,9 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t mt = jl_gf_mtable(F); jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); entry = NULL; - if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && - jl_typetagis(jl_atomic_load_relaxed(&mt->cache), jl_typemap_level_type)) { - // hashing args is expensive, but looking at mt->cache is probably even more expensive + int cache_entry_count = jl_atomic_load_relaxed(&((jl_datatype_t*)FT)->name->cache_entry_count); + if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && (cache_entry_count == 0 || cache_entry_count >= 8)) { + // hashing args is expensive, but so do that only if looking at mc->cache is probably even more expensive tt = lookup_arg_type_tuple(F, args, nargs); if (tt != NULL) entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); diff --git a/src/jltypes.c b/src/jltypes.c index 98addd89e5ce5..e5341c2621df3 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3007,26 +3007,27 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name->mt = jl_nonfunction_mt; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 16 - 2; - jl_typename_type->name->names = jl_perm_symsvec(16, "name", "module", + jl_typename_type->name->n_uninitialized = 17 - 2; + jl_typename_type->name->names = jl_perm_symsvec(17, "name", "module", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", "mt", "partial", "hash", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", - "max_methods", "constprop_heuristic"); - const static uint32_t typename_constfields[1] = { 0x00003a27 }; // (1<<0)|(1<<1)|(1<<2)|(1<<5)|(1<<9)|(1<<11)|(1<<12)|(1<<13) ; TODO: put back (1<<3)|(1<<4) in this list - const static uint32_t typename_atomicfields[1] = { 0x00000180 }; // (1<<7)|(1<<8) + "cache_entry_count", "max_methods", "constprop_heuristic"); + const static uint32_t typename_constfields[1] = { 0b00011101000100111 }; // TODO: put back atomicfields and constfields in this list + const static uint32_t typename_atomicfields[1] = { 0b00100000110000000 }; jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(16, jl_symbol_type, jl_any_type /*jl_module_type*/, + jl_typename_type->types = jl_svec(17, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, jl_methtable_type, jl_any_type, jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, + jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/); jl_methtable_type->name = jl_new_typename_in(jl_symbol("MethodTable"), core, 0, 1); @@ -3856,6 +3857,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_typename_type->types, 13, jl_uint8_type); jl_svecset(jl_typename_type->types, 14, jl_uint8_type); jl_svecset(jl_typename_type->types, 15, jl_uint8_type); + jl_svecset(jl_typename_type->types, 16, jl_uint8_type); jl_svecset(jl_methtable_type->types, 4, jl_long_type); jl_svecset(jl_methtable_type->types, 5, jl_module_type); jl_svecset(jl_methtable_type->types, 6, jl_array_any_type); diff --git a/src/julia.h b/src/julia.h index 21af97deaf984..c8edf2d95afb8 100644 --- a/src/julia.h +++ b/src/julia.h @@ -536,6 +536,7 @@ typedef struct { uint8_t mutabl:1; uint8_t mayinlinealloc:1; uint8_t _reserved:5; + _Atomic(uint8_t) cache_entry_count; // (approximate counter of TypeMapEntry for heuristics) uint8_t max_methods; // override for inference's max_methods setting (0 = no additional limit or relaxation) uint8_t constprop_heustic; // override for inference's constprop heuristic } jl_typename_t; diff --git a/test/core.jl b/test/core.jl index f6cc5ae9f9f7a..15cb139601349 100644 --- a/test/core.jl +++ b/test/core.jl @@ -40,7 +40,7 @@ for (T, c) in ( (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), (Core.TypeMapEntry, [:next, :min_world, :max_world]), (Core.TypeMapLevel, [:arg1, :targ, :name1, :tname, :list, :any]), - (Core.TypeName, [:cache, :linearcache]), + (Core.TypeName, [:cache, :linearcache, :cache_entry_count]), (DataType, [:types, :layout]), (Core.Memory, []), (Core.GenericMemoryRef, []), From 473593343c1b2bf00e00775c4bf6b85eebb66c35 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 22 Apr 2025 15:23:03 +0900 Subject: [PATCH 112/662] fix performance regression introduced by #58027 (#58184) --- Compiler/src/abstractinterpretation.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index b14475edee613..3a2d04a127607 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3728,8 +3728,9 @@ struct AbstractEvalBasicStatementResult end end -function abstract_eval_basic_statement(interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState, - result::Union{Nothing,Future{RTEffects}}=nothing) +@inline function abstract_eval_basic_statement( + interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState, + result::Union{Nothing,Future{RTEffects}}=nothing) rt = nothing exct = Bottom changes = nothing From 4766133f6c1e7825f3fb35243e353a380eb425d5 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 22 Apr 2025 08:54:53 +0200 Subject: [PATCH 113/662] add showing a string to REPL precompile workload (#58157) --- stdlib/REPL/src/precompile.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl index 4f3b3a6eae083..f9f411566a719 100644 --- a/stdlib/REPL/src/precompile.jl +++ b/stdlib/REPL/src/precompile.jl @@ -58,6 +58,7 @@ function repl_workload() printstyled("a", "b") display([1]) display([1 2; 3 4]) + display("a string") foo(x) = 1 @time @eval foo(1) ; pwd From a7b8c833a3ec15c5b9adac2cc63caffe086b474c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 22 Apr 2025 08:11:09 -0400 Subject: [PATCH 114/662] Revert #57979 (and following #58083 #58082) (#58182) The point of #57979 was to make inference faster, but it made it instead much slower (https://github.com/JuliaLang/julia/commit/83524acfee6d1a99d175f0923005a229f9e885e5#commitcomment-155658124), so revert back to the fast behavior before it was "optimized" (and revert the bugfixes for the original commit). --- .../CompilerDevTools/src/CompilerDevTools.jl | 2 +- Compiler/src/optimize.jl | 98 ++++++++----- Compiler/src/ssair/inlining.jl | 2 +- Compiler/src/typeinfer.jl | 137 +++--------------- Compiler/test/codegen.jl | 1 - Compiler/test/inline.jl | 4 - 6 files changed, 90 insertions(+), 154 deletions(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index ddf202f378fb5..e08d08779e097 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -47,7 +47,7 @@ end function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) opt = result.src::Compiler.OptimizationState - ir = opt.optresult.ir::Compiler.IRCode + ir = opt.result.ir::Compiler.IRCode override = with_new_compiler for inst in ir.stmts stmt = inst[:stmt] diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index fdf97c447559d..612105061c38d 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -116,14 +116,11 @@ function inline_cost_clamp(x::Int) return convert(InlineCostType, x) end -const SRC_FLAG_DECLARED_INLINE = 0x1 -const SRC_FLAG_DECLARED_NOINLINE = 0x2 - is_declared_inline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 1 is_declared_noinline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 2 ##################### # OptimizationState # @@ -160,7 +157,6 @@ code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.wor mutable struct OptimizationResult ir::IRCode - inline_flag::UInt8 simplified::Bool # indicates whether the IR was processed with `cfg_simplify!` end @@ -172,7 +168,7 @@ end mutable struct OptimizationState{Interp<:AbstractInterpreter} linfo::MethodInstance src::CodeInfo - optresult::Union{Nothing, OptimizationResult} + result::Union{Nothing, OptimizationResult} stmt_info::Vector{CallInfo} mod::Module sptypes::Vector{VarState} @@ -240,29 +236,13 @@ include("ssair/EscapeAnalysis.jl") include("ssair/passes.jl") include("ssair/irinterp.jl") -function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector) - ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult)) -end - -function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType) - src = ir_to_codeinf!(opt, edges) - src.inlining_cost = inlining_cost - src -end - -function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector) - src = ir_to_codeinf!(opt) - src.edges = edges - src -end - function ir_to_codeinf!(opt::OptimizationState) - (; linfo, src, optresult) = opt - if optresult === nothing + (; linfo, src, result) = opt + if result === nothing return src end - src = ir_to_codeinf!(src, optresult.ir) - opt.optresult = nothing + src = ir_to_codeinf!(src, result.ir) + opt.result = nothing opt.src = src maybe_validate_code(linfo, src, "optimized") return src @@ -505,12 +485,63 @@ end abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = types(src)[s] """ - finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) + finish(interp::AbstractInterpreter, opt::OptimizationState, + ir::IRCode, caller::InferenceResult) -Called at the end of optimization to store the resulting IR back into the OptimizationState. +Post-process information derived by Julia-level optimizations for later use. +In particular, this function determines the inlineability of the optimized code. """ -function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) - opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false) +function finish(interp::AbstractInterpreter, opt::OptimizationState, + ir::IRCode, caller::InferenceResult) + (; src, linfo) = opt + (; def, specTypes) = linfo + + force_noinline = is_declared_noinline(src) + + # compute inlining and other related optimizations + result = caller.result + @assert !(result isa LimitedAccuracy) + result = widenslotwrapper(result) + + opt.result = OptimizationResult(ir, false) + + # determine and cache inlineability + if !force_noinline + sig = unwrap_unionall(specTypes) + if !(isa(sig, DataType) && sig.name === Tuple.name) + force_noinline = true + end + if !is_declared_inline(src) && result === Bottom + force_noinline = true + end + end + if force_noinline + set_inlineable!(src, false) + elseif isa(def, Method) + if is_declared_inline(src) && isdispatchtuple(specTypes) + # obey @inline declaration if a dispatch barrier would not help + set_inlineable!(src, true) + else + # compute the cost (size) of inlining this code + params = OptimizationParams(interp) + cost_threshold = default = params.inline_cost_threshold + if ⊑(optimizer_lattice(interp), result, Tuple) && !isconcretetype(widenconst(result)) + cost_threshold += params.inline_tupleret_bonus + end + # if the method is declared as `@inline`, increase the cost threshold 20x + if is_declared_inline(src) + cost_threshold += 19*default + end + # a few functions get special treatment + if def.module === _topmod(def.module) + name = def.name + if name === :iterate || name === :unsafe_convert || name === :cconvert + cost_threshold += 4*default + end + end + src.inlining_cost = inline_cost(ir, params, cost_threshold) + end + end return nothing end @@ -984,8 +1015,7 @@ end function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) - finishopt!(interp, opt, ir) - return nothing + return finish(interp, opt, ir, caller) end const ALL_PASS_NAMES = String[] @@ -1436,7 +1466,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod return thiscost end -function inline_cost_model(ir::IRCode, params::OptimizationParams, cost_threshold::Int) +function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Int) bodycost = 0 for i = 1:length(ir.stmts) stmt = ir[SSAValue(i)][:stmt] diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index c7e052ed17218..3830197aef458 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -976,7 +976,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, ir::IRCode, preserve_local return ir, spec_info, DebugInfo(ir.debuginfo, length(ir.stmts)) end function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, preserve_local_sources::Bool) - result = opt.optresult + result = opt.result if result !== nothing !result.simplified && simplify_ir!(result) return retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 1afe4c297e5ba..2b4ada7805140 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -104,10 +104,7 @@ end function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) - if caller.cache_mode === CACHE_MODE_LOCAL - @assert !isdefined(result, :ci) - result.src = transform_result_for_local_cache(interp, result) - elseif isdefined(result, :ci) + if isdefined(result, :ci) edges = result_edges(interp, caller) ci = result.ci # if we aren't cached, we don't need this edge @@ -118,25 +115,21 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation store_backedges(ci, edges) end inferred_result = nothing - uncompressed = result.src + uncompressed = inferred_result const_flag = is_result_constabi_eligible(result) - debuginfo = get_debuginfo(result.src) discard_src = caller.cache_mode === CACHE_MODE_NULL || const_flag if !discard_src inferred_result = transform_result_for_cache(interp, result, edges) - if inferred_result !== nothing - uncompressed = inferred_result - debuginfo = get_debuginfo(inferred_result) - # Inlining may fast-path the global cache via `VolatileInferenceResult`, so store it back here - result.src = inferred_result - end # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? if inferred_result isa CodeInfo + result.src = inferred_result if may_compress(interp) nslots = length(inferred_result.slotflags) resize!(inferred_result.slottypes::Vector{Any}, nslots) resize!(inferred_result.slotnames, nslots) end + di = inferred_result.debuginfo + uncompressed = inferred_result inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing @@ -144,21 +137,18 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation inferred_result = nothing end end - if debuginfo === nothing - debuginfo = DebugInfo(result.linfo) + if !@isdefined di + di = DebugInfo(result.linfo) end time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), - result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) + result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, di, edges) engine_reject(interp, ci) codegen = codegen_cache(interp) - if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) - if isa(uncompressed, OptimizationState) - uncompressed = ir_to_codeinf!(uncompressed, edges) - end + if !discard_src && codegen !== nothing && uncompressed isa CodeInfo # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work codegen[ci] = uncompressed if bootstrapping_compiler && inferred_result == nothing @@ -309,116 +299,36 @@ function adjust_cycle_frame!(sv::InferenceState, cycle_valid_worlds::WorldRange, return nothing end -function get_debuginfo(src) - isa(src, CodeInfo) && return src.debuginfo - isa(src, OptimizationState) && return src.src.debuginfo - return nothing -end - function is_result_constabi_eligible(result::InferenceResult) result_type = result.result return isa(result_type, Const) && is_foldable_nothrow(result.ipo_effects) && is_inlineable_constant(result_type.val) end -function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult) - src = result.src - isa(src, OptimizationState) || return MAX_INLINE_COST - compute_inlining_cost(interp, result, src.optresult) -end - -function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult, optresult#=::OptimizationResult=#) - return inline_cost_model(interp, result, optresult.inline_flag, optresult.ir) -end - -function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, - inline_flag::UInt8, ir::IRCode) - - inline_flag === SRC_FLAG_DECLARED_NOINLINE && return MAX_INLINE_COST - - mi = result.linfo - (; def, specTypes) = mi - if !isa(def, Method) - return MAX_INLINE_COST - end - - declared_inline = inline_flag === SRC_FLAG_DECLARED_INLINE - - rt = result.result - @assert !(rt isa LimitedAccuracy) - rt = widenslotwrapper(rt) - - sig = unwrap_unionall(specTypes) - if !(isa(sig, DataType) && sig.name === Tuple.name) - return MAX_INLINE_COST - end - if !declared_inline && rt === Bottom - return MAX_INLINE_COST - end - - if declared_inline && isdispatchtuple(specTypes) - # obey @inline declaration if a dispatch barrier would not help - return MIN_INLINE_COST - else - # compute the cost (size) of inlining this code - params = OptimizationParams(interp) - cost_threshold = default = params.inline_cost_threshold - if ⊑(optimizer_lattice(interp), rt, Tuple) && !isconcretetype(widenconst(rt)) - cost_threshold += params.inline_tupleret_bonus - end - # if the method is declared as `@inline`, increase the cost threshold 20x - if declared_inline - cost_threshold += 19*default - end - # a few functions get special treatment - if def.module === _topmod(def.module) - name = def.name - if name === :iterate || name === :unsafe_convert || name === :cconvert - cost_threshold += 4*default - end - end - return inline_cost_model(ir, params, cost_threshold) - end -end - -function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) - if is_result_constabi_eligible(result) - return nothing - end +function transform_result_for_cache(::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) src = result.src if isa(src, OptimizationState) - # Compute and store any information required to determine the inlineability of the callee. - opt = src - opt.src.inlining_cost = compute_inlining_cost(interp, result) - end - return src -end - -function transform_result_for_cache(interp::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) - inlining_cost = nothing - src = result.src - if isa(src, OptimizationState) - opt = src - inlining_cost = compute_inlining_cost(interp, result, opt.optresult) - discard_optimized_result(interp, opt, inlining_cost) && return nothing - src = ir_to_codeinf!(opt) + src = ir_to_codeinf!(src) end if isa(src, CodeInfo) src.edges = edges - src.inlining_cost = inlining_cost !== nothing ? inlining_cost : compute_inlining_cost(interp, result) end return src end -function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) - may_discard_trees(interp) || return false - return inlining_cost == MAX_INLINE_COST -end - function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo) def = mi.def isa(def, Method) || return ci # don't compress toplevel code - may_compress(interp) && return ccall(:jl_compress_ir, String, (Any, Any), def, ci) - return ci + can_discard_trees = may_discard_trees(interp) + cache_the_tree = !can_discard_trees || is_inlineable(ci) + if cache_the_tree + if may_compress(interp) + return ccall(:jl_compress_ir, String, (Any, Any), def, ci) + else + return ci + end + else + return nothing + end end function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) @@ -1193,7 +1103,8 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - src = ir_to_codeinf!(opt, frame, Core.svec(opt.inlining.edges...)) + src = ir_to_codeinf!(opt) + src.edges = Core.svec(opt.inlining.edges...) end result.src = frame.src = src end diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 45db9e73d5a3f..fd8bbae70a346 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -4,7 +4,6 @@ using Random using InteractiveUtils -using InteractiveUtils: code_llvm, code_native using Libdl using Test diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index 0a88907965f5a..92e389ff0dc04 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -1,7 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -module inline_tests - using Test using Base.Meta using Core: ReturnNode @@ -2313,5 +2311,3 @@ g_noinline_invoke(x) = f_noinline_invoke(x) let src = code_typed1(g_noinline_invoke, (Union{Symbol,Nothing},)) @test !any(@nospecialize(x)->isa(x,GlobalRef), src.code) end - -end # module inline_tests From f3264e7e392b70286de7b9d242672564ed9ab03a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 22 Apr 2025 08:12:56 -0400 Subject: [PATCH 115/662] staticdata: stop compressing and storing large IR (#58172) Compressing IR adds Methods roots, which tend to become quite expensive to store after some time. We also used to store the inferred code, but since we never actually look at that again now (since now the compiler avoids using --output-ji or --sysimage-native-code=no or --pkgimages=no) it is now quite a bit of wasted space also. Roughly 40 MB saved on disk (165 MB -> 124 MB) if I measured correctly: ``` usr/lib/julia/sys.so : section size addr .note.gnu.build-id 36 680 .gnu.hash 52 720 .dynsym 2760 776 .dynstr 2070 3536 .gnu.version 230 5606 .gnu.version_r 160 5840 .rela.dyn 943440 6000 .rela.plt 2256 949440 .init 27 954368 .plt 1520 954400 .plt.got 8 955920 .text 17207656 955936 .fini 13 18163592 .rodata 18388 18165760 .eh_frame_hdr 315940 18184148 .eh_frame 1525972 18500088 .init_array 8 20032896 .fini_array 8 20032904 .dynamic 496 20032912 .got 128 20033408 .got.plt 776 20033536 .data 8 20034312 .bss 8 20034320 .lbss 80336 20034336 .lrodata 303064 20118768 .ldata 123852168 20425928 .comment 43 0 .debug_info 10028208 0 .debug_abbrev 18197 0 .debug_line 6581659 0 .debug_str 578898 0 .debug_ranges 10854832 0 .debug_gnu_pubnames 3065385 0 .debug_gnu_pubtypes 352332 0 Total 175737082 ``` --- src/aotcompile.cpp | 14 -------------- src/gf.c | 20 -------------------- src/julia_internal.h | 1 - src/staticdata.c | 26 ++++++++++++++------------ 4 files changed, 14 insertions(+), 47 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index d687f44808409..5cdecda9d8582 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -675,20 +675,6 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[0] = (jl_value_t*)codeinfos; void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, external_linkage); - // examine everything just emitted and save it to the caches - if (!external_linkage) { - for (size_t i = 0, l = jl_array_nrows(codeinfos); i < l; i++) { - jl_value_t *item = jl_array_ptr_ref(codeinfos, i); - if (jl_is_code_instance(item)) { - // now add it to our compilation results - jl_code_instance_t *codeinst = (jl_code_instance_t*)item; - jl_code_info_t *src = (jl_code_info_t*)jl_array_ptr_ref(codeinfos, ++i); - assert(jl_is_code_info(src)); - jl_add_codeinst_to_cache(codeinst, src); - } - } - } - // move everything inside, now that we've merged everything // (before adding the exported headers) ((jl_native_code_desc_t*)data)->M.withModuleDo([&](Module &M) { diff --git a/src/gf.c b/src/gf.c index f01f48d6115cf..2f4b838f04908 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2846,30 +2846,10 @@ void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_c jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_PROPAGATES_ROOT); -JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src) -{ - assert(jl_is_code_info(src)); - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - if (jl_generating_output() && jl_is_method(mi->def.method) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { - jl_value_t *compressed = jl_compress_ir(mi->def.method, src); - // These should already be compatible (and should be an assert), but make sure of it anyways - if (jl_is_svec(src->edges)) { - jl_atomic_store_release(&codeinst->edges, (jl_svec_t*)src->edges); - jl_gc_wb(codeinst, src->edges); - } - jl_atomic_store_release(&codeinst->debuginfo, src->debuginfo); - jl_gc_wb(codeinst, src->debuginfo); - jl_atomic_store_release(&codeinst->inferred, compressed); - jl_gc_wb(codeinst, compressed); - } -} - - JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src) { assert(jl_is_code_info(src)); jl_emit_codeinst_to_jit(codeinst, src); - jl_add_codeinst_to_cache(codeinst, src); } jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) diff --git a/src/julia_internal.h b/src/julia_internal.h index 26380768c0b60..864097d8d22fe 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -688,7 +688,6 @@ JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROP JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile); JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_info_t *src); -JL_DLLEXPORT void jl_add_codeinst_to_cache(jl_code_instance_t *codeinst, jl_code_info_t *src); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( diff --git a/src/staticdata.c b/src/staticdata.c index 6916bfe00ba5f..64f66e87252aa 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -954,24 +954,26 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } } - if (s->incremental && jl_is_code_instance(v)) { + if (jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; jl_method_instance_t *mi = jl_get_ci_mi(ci); - // make sure we don't serialize other reachable cache entries of foreign methods - // Should this now be: - // if (ci !in ci->defs->cache) - // record_field_change((jl_value_t**)&ci->next, NULL); - // Why are we checking that the method/module this originates from is in_image? - // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)mi->def.value)) { - // TODO: if (ci in ci->defs->cache) - record_field_change((jl_value_t**)&ci->next, NULL); + if (s->incremental) { + // make sure we don't serialize other reachable cache entries of foreign methods + // Should this now be: + // if (ci !in ci->defs->cache) + // record_field_change((jl_value_t**)&ci->next, NULL); + // Why are we checking that the method/module this originates from is in_image? + // and then disconnect this CI? + if (jl_object_in_image((jl_value_t*)mi->def.value)) { + // TODO: if (ci in ci->defs->cache) + record_field_change((jl_value_t**)&ci->next, NULL); + } } jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred); if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) jl_method_t *def = mi->def.method; if (jl_is_method(def)) { // don't delete toplevel code - int is_relocatable = jl_is_code_info(inferred) || + int is_relocatable = !s->incremental || jl_is_code_info(inferred) || (jl_is_string(inferred) && jl_string_len(inferred) > 0 && jl_string_data(inferred)[jl_string_len(inferred) - 1]); if (!is_relocatable) { inferred = jl_nothing; @@ -993,7 +995,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (inferred == jl_nothing) { record_field_change((jl_value_t**)&ci->inferred, jl_nothing); } - else if (jl_is_string(inferred)) { + else if (s->incremental && jl_is_string(inferred)) { // New roots for external methods if (jl_object_in_image((jl_value_t*)def)) { void **pfound = ptrhash_bp(&s->method_roots_index, def); From ff0a9313de7542c08279dd7925f5468cf54f6e92 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Tue, 22 Apr 2025 07:02:15 -0700 Subject: [PATCH 116/662] JuliaSyntax parser-based REPL completions overhaul (#57767) # Overview As we add REPL features, bugs related to the ad-hoc parsing done by `REPLCompletions.completions` have crept in. This pull request replaces most of the manual parsing (regex, `find_start_brace`) with a new approach that parses the entire input buffer once, before and after the cursor, using JuliaSyntax. We then query the parsed syntax tree to determine the kind of completion to be done. # Changes - New, JuliaSyntax-based completions mechanism. - The `complete_line` interface now has the option of replacing arbitrary regions of text in the input buffer by returning a `Region` (`Pair{Int, Int}` for consistency with the convention in LineEdit, and `pos` being a 0-based byte offset). - Fixes parsing-related bugs: - fix #55420 - fix #55429 - fix #55518 - fix #55520 - fix #55842 - fix #56389 - fix #57307 - fix #57611 - fix #57624 - fix #58099 - Fixes some bugs that exist on 28d3bd56d36 that were found by fuzzing: - `x \"` + `TAB` throws a `FieldError` exception - String completion would sometimes delete the entire input buffer. - Completions should not happen inside comments. - The duplicate code for path completion in strings, `Cmd`-strings, and the shell has been removed, causing paths to complete the same way for all three. Now, `~` is expanded in two situations: - If `foo` exists, or if `foo` does not exist but there are no possible completions: ``` "~/foo/b|" =TAB=> "~/foo/bar|" "~/foo/bar|" =TAB=> "/home/user/foo/bar|" OR "~/foo/bar"| =TAB=> "/home/user/foo/bar"| ``` - If the current path ends with a `/` and you hit TAB again: ``` "~/foo/|" =TAB=> "/home/user/foo/|" OR "~/foo/"| =TAB=> "/home/user/foo/"| ``` # Future work - Method completions could be changed to look for methods with exactly the given number of arguments if the closing `)` is present, and search for signatures with the right prefix otherwise. - It would be nice to be able to search by type as well as value (perhaps by putting `::T` in place of arguments). - Other REPL features could benefit from JuliaSyntax, so it might be worth sharing the parse tree between completions and other features: - Emacs-style sexpr navigation: `C-M-f`/`C-M-b`/`C-M-u`, etc. - Improved auto-indent. - It would be nice if hints worked even when the cursor is between text. - `CursorNode` is a slightly tweaked copy of `SyntaxNode` from JuliaSyntax that tracks the parent node but includes all trivia. It is used with `seek_pos`, which navigates to the innermost node at a given position so we can examine nearby nodes and the parent. This could probably duplicate less code from JuliaSyntax. --- stdlib/REPL/src/LineEdit.jl | 60 +- stdlib/REPL/src/REPL.jl | 23 +- stdlib/REPL/src/REPLCompletions.jl | 983 +++++++++++----------------- stdlib/REPL/src/SyntaxUtil.jl | 111 ++++ stdlib/REPL/test/replcompletions.jl | 270 ++++++-- 5 files changed, 759 insertions(+), 688 deletions(-) create mode 100644 stdlib/REPL/src/SyntaxUtil.jl diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 288a9cb1ea91f..53e497a6548ee 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -391,16 +391,21 @@ function complete_line(s::MIState) end end +# Old complete_line return type: Vector{String}, String, Bool +# New complete_line return type: NamedCompletion{String}, String, Bool +# OR NamedCompletion{String}, Region, Bool +# # due to close coupling of the Pkg ReplExt `complete_line` can still return a vector of strings, # so we convert those in this helper -function complete_line_named(args...; kwargs...)::Tuple{Vector{NamedCompletion},String,Bool} - result = complete_line(args...; kwargs...)::Union{Tuple{Vector{NamedCompletion},String,Bool},Tuple{Vector{String},String,Bool}} - if result isa Tuple{Vector{NamedCompletion},String,Bool} - return result - else - completions, partial, should_complete = result - return map(NamedCompletion, completions), partial, should_complete - end +function complete_line_named(c, s, args...; kwargs...)::Tuple{Vector{NamedCompletion},Region,Bool} + r1, r2, should_complete = complete_line(c, s, args...; kwargs...)::Union{ + Tuple{Vector{String}, String, Bool}, + Tuple{Vector{NamedCompletion}, String, Bool}, + Tuple{Vector{NamedCompletion}, Region, Bool}, + } + completions = (r1 isa Vector{String} ? map(NamedCompletion, r1) : r1) + r = (r2 isa String ? (position(s)-sizeof(r2) => position(s)) : r2) + completions, r, should_complete end # checks for a hint and shows it if appropriate. @@ -426,14 +431,14 @@ function check_show_hint(s::MIState) return end t_completion = Threads.@spawn :default begin - named_completions, partial, should_complete = nothing, nothing, nothing + named_completions, reg, should_complete = nothing, nothing, nothing # only allow one task to generate hints at a time and check around lock # if the user has pressed a key since the hint was requested, to skip old completions next_key_pressed() && return @lock s.hint_generation_lock begin next_key_pressed() && return - named_completions, partial, should_complete = try + named_completions, reg, should_complete = try complete_line_named(st.p.complete, st, s.active_module; hint = true) catch lock_clear_hint() @@ -448,21 +453,19 @@ function check_show_hint(s::MIState) return end # Don't complete for single chars, given e.g. `x` completes to `xor` - if length(partial) > 1 && should_complete + if reg.second - reg.first > 1 && should_complete singlecompletion = length(completions) == 1 p = singlecompletion ? completions[1] : common_prefix(completions) if singlecompletion || p in completions # i.e. complete `@time` even though `@time_imports` etc. exists - # The completion `p` and the input `partial` may not share the same initial + # The completion `p` and the region `reg` may not share the same initial # characters, for instance when completing to subscripts or superscripts. # So, in general, make sure that the hint starts at the correct position by # incrementing its starting position by as many characters as the input. - startind = 1 # index of p from which to start providing the hint - maxind = ncodeunits(p) - for _ in partial - startind = nextind(p, startind) - startind > maxind && break - end + maxind = lastindex(p) + startind = sizeof(content(s, reg)) if startind ≤ maxind # completion on a complete name returns itself so check that there's something to hint + # index of p from which to start providing the hint + startind = nextind(p, startind) hint = p[startind:end] next_key_pressed() && return @lock s.line_modify_lock begin @@ -491,7 +494,7 @@ function clear_hint(s::ModeState) end function complete_line(s::PromptState, repeats::Int, mod::Module; hint::Bool=false) - completions, partial, should_complete = complete_line_named(s.p.complete, s, mod; hint) + completions, reg, should_complete = complete_line_named(s.p.complete, s, mod; hint) isempty(completions) && return false if !should_complete # should_complete is false for cases where we only want to show @@ -499,17 +502,16 @@ function complete_line(s::PromptState, repeats::Int, mod::Module; hint::Bool=fal show_completions(s, completions) elseif length(completions) == 1 # Replace word by completion - prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, reg, completions[1].completion) else p = common_prefix(completions) + partial = content(s, reg.first => min(bufend(s), reg.first + sizeof(p))) if !isempty(p) && p != partial # All possible completions share the same prefix, so we might as - # well complete that - prev_pos = position(s) + # well complete that. push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, p) + edit_splice!(s, reg, p) elseif repeats > 0 show_completions(s, completions) end @@ -830,12 +832,12 @@ function edit_move_right(m::MIState) refresh_line(s) return true else - completions, partial, should_complete = complete_line(s.p.complete, s, m.active_module) - if should_complete && eof(buf) && length(completions) == 1 && length(partial) > 1 + completions, reg, should_complete = complete_line(s.p.complete, s, m.active_module) + if should_complete && eof(buf) && length(completions) == 1 && reg.second - reg.first > 1 # Replace word by completion prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, (prev_pos - reg.second + reg.first) => prev_pos, completions[1].completion) refresh_line(state(s)) return true else @@ -2255,12 +2257,12 @@ setmodifiers!(c) = nothing # Search Mode completions function complete_line(s::SearchState, repeats, mod::Module; hint::Bool=false) - completions, partial, should_complete = complete_line(s.histprompt.complete, s, mod; hint) + completions, reg, should_complete = complete_line(s.histprompt.complete, s, mod; hint) # For now only allow exact completions in search mode if length(completions) == 1 prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, (prev_pos - reg.second - reg.first) => prev_pos, completions[1].completion) return true end return false diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index ae2fec4ee10e0..cfb6e88a2663c 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -86,6 +86,7 @@ import .LineEdit: PromptState, mode_idx +include("SyntaxUtil.jl") include("REPLCompletions.jl") using .REPLCompletions @@ -799,27 +800,29 @@ end beforecursor(buf::IOBuffer) = String(buf.data[1:buf.ptr-1]) +# Convert inclusive-inclusive 1-based char indexing to inclusive-exclusive byte Region. +to_region(s, r) = first(r)-1 => (length(r) > 0 ? nextind(s, last(r))-1 : first(r)-1) + function complete_line(c::REPLCompletionProvider, s::PromptState, mod::Module; hint::Bool=false) - partial = beforecursor(s.input_buffer) full = LineEdit.input_string(s) - ret, range, should_complete = completions(full, lastindex(partial), mod, c.modifiers.shift, hint) + ret, range, should_complete = completions(full, thisind(full, position(s)), mod, c.modifiers.shift, hint) + range = to_region(full, range) c.modifiers = LineEdit.Modifiers() - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end function complete_line(c::ShellCompletionProvider, s::PromptState; hint::Bool=false) - # First parse everything up to the current position - partial = beforecursor(s.input_buffer) full = LineEdit.input_string(s) - ret, range, should_complete = shell_completions(full, lastindex(partial), hint) - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + ret, range, should_complete = shell_completions(full, thisind(full, position(s)), hint) + range = to_region(full, range) + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end function complete_line(c::LatexCompletions, s; hint::Bool=false) - partial = beforecursor(LineEdit.buffer(s)) full = LineEdit.input_string(s)::String - ret, range, should_complete = bslash_completions(full, lastindex(partial), hint)[2] - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + ret, range, should_complete = bslash_completions(full, thisind(full, position(s)), hint)[2] + range = to_region(full, range) + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end with_repl_linfo(f, repl) = f(outstream(repl)) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index d4e33e8ff5136..c56f8b9653cbf 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -12,8 +12,10 @@ const CC = Base.Compiler using Base.Meta using Base: propertynames, something, IdSet using Base.Filesystem: _readdirx +using Base.JuliaSyntax: @K_str, @KSet_str, parseall, byte_range, children, is_prefix_call, is_trivia, kind using ..REPL.LineEdit: NamedCompletion +using ..REPL.SyntaxUtil: CursorNode, find_parent, seek_pos, char_range, char_last, children_nt, find_delim abstract type Completion end @@ -302,9 +304,8 @@ const sorted_keyvals = ["false", "true"] complete_keyval!(suggestions::Vector{Completion}, s::String) = complete_from_list!(suggestions, KeyvalCompletion, sorted_keyvals, s) -function do_raw_escape(s) - # escape_raw_string with delim='`' and ignoring the rule for the ending \ - return replace(s, r"(\\+)`" => s"\1\\`") +function do_cmd_escape(s) + return Base.escape_raw_string(Base.shell_escape_posixly(s), '`') end function do_shell_escape(s) return Base.shell_escape_posixly(s) @@ -312,6 +313,20 @@ end function do_string_escape(s) return escape_string(s, ('\"','$')) end +function do_string_unescape(s) + s = replace(s, "\\\$"=>"\$") + try + unescape_string(s) + catch e + e isa ArgumentError || rethrow() + s # it is unlikely, but if it isn't a valid string, maybe it was a valid path, and just needs escape_string called? + end +end + +function joinpath_withsep(dir, path; dirsep) + dir == "" && return path + dir[end] == dirsep ? dir * path : dir * dirsep * path +end const PATH_cache_lock = Base.ReentrantLock() const PATH_cache = Set{String}() @@ -409,9 +424,10 @@ end function complete_path(path::AbstractString; use_envpath=false, shell_escape=false, - raw_escape=false, + cmd_escape=false, string_escape=false, - contract_user=false) + contract_user=false, + dirsep=Sys.iswindows() ? '\\' : '/') @assert !(shell_escape && string_escape) if Base.Sys.isunix() && occursin(r"^~(?:/|$)", path) # if the path is just "~", don't consider the expanded username as a prefix @@ -440,7 +456,7 @@ function complete_path(path::AbstractString; for entry in entries if startswith(entry.name, prefix) is_dir = try isdir(entry) catch ex; ex isa Base.IOError ? false : rethrow() end - push!(matches, is_dir ? entry.name * "/" : entry.name) + push!(matches, is_dir ? joinpath_withsep(entry.name, ""; dirsep) : entry.name) end end @@ -456,7 +472,7 @@ function complete_path(path::AbstractString; end matches = ((shell_escape ? do_shell_escape(s) : string_escape ? do_string_escape(s) : s) for s in matches) - matches = ((raw_escape ? do_raw_escape(s) : s) for s in matches) + matches = ((cmd_escape ? do_cmd_escape(s) : s) for s in matches) matches = Completion[PathCompletion(contract_user ? contractuser(s) : s) for s in matches] return matches, dir, !isempty(matches) end @@ -469,7 +485,8 @@ function complete_path(path::AbstractString, contract_user=false) ## TODO: enable this depwarn once Pkg is fixed #Base.depwarn("complete_path with pos argument is deprecated because the return value [2] is incorrect to use", :complete_path) - paths, dir, success = complete_path(path; use_envpath, shell_escape, string_escape) + paths, dir, success = complete_path(path; use_envpath, shell_escape, string_escape, dirsep='/') + if Base.Sys.isunix() && occursin(r"^~(?:/|$)", path) # if the path is just "~", don't consider the expanded username as a prefix if path == "~" @@ -490,91 +507,6 @@ function complete_path(path::AbstractString, return paths, startpos:pos, success end -function complete_expanduser(path::AbstractString, r) - expanded = - try expanduser(path) - catch e - e isa ArgumentError || rethrow() - path - end - return Completion[PathCompletion(expanded)], r, path != expanded -end - -# Returns a range that includes the method name in front of the first non -# closed start brace from the end of the string. -function find_start_brace(s::AbstractString; c_start='(', c_end=')') - r = reverse(s) - i = firstindex(r) - braces = in_comment = 0 - in_single_quotes = in_double_quotes = in_back_ticks = false - num_single_quotes_in_string = count('\'', s) - while i <= ncodeunits(r) - c, i = iterate(r, i) - if c == '#' && i <= ncodeunits(r) && iterate(r, i)[1] == '=' - c, i = iterate(r, i) # consume '=' - new_comments = 1 - # handle #=#=#=#, by counting =# pairs - while i <= ncodeunits(r) && iterate(r, i)[1] == '#' - c, i = iterate(r, i) # consume '#' - iterate(r, i)[1] == '=' || break - c, i = iterate(r, i) # consume '=' - new_comments += 1 - end - if c == '=' - in_comment += new_comments - else - in_comment -= new_comments - end - elseif !in_single_quotes && !in_double_quotes && !in_back_ticks && in_comment == 0 - if c == c_start - braces += 1 - elseif c == c_end - braces -= 1 - elseif c == '\'' && num_single_quotes_in_string % 2 == 0 - # ' can be a transpose too, so check if there are even number of 's in the string - # TODO: This probably needs to be more robust - in_single_quotes = true - elseif c == '"' - in_double_quotes = true - elseif c == '`' - in_back_ticks = true - end - else - if in_single_quotes && - c == '\'' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_single_quotes = false - elseif in_double_quotes && - c == '"' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_double_quotes = false - elseif in_back_ticks && - c == '`' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_back_ticks = false - elseif in_comment > 0 && - c == '=' && i <= ncodeunits(r) && iterate(r, i)[1] == '#' - # handle =#=#=#=, by counting #= pairs - c, i = iterate(r, i) # consume '#' - old_comments = 1 - while i <= ncodeunits(r) && iterate(r, i)[1] == '=' - c, i = iterate(r, i) # consume '=' - iterate(r, i)[1] == '#' || break - c, i = iterate(r, i) # consume '#' - old_comments += 1 - end - if c == '#' - in_comment -= old_comments - else - in_comment += old_comments - end - end - end - braces == 1 && break - end - braces != 1 && return 0:-1, -1 - method_name_end = reverseind(s, i) - startind = nextind(s, something(findprev(in(non_identifier_chars), s, method_name_end), 0))::Int - return (startind:lastindex(s), method_name_end) -end - struct REPLCacheToken end struct REPLInterpreter <: CC.AbstractInterpreter @@ -729,6 +661,7 @@ end # lower `ex` and run type inference on the resulting top-level expression function repl_eval_ex(@nospecialize(ex), context_module::Module; limit_aggressive_inference::Bool=false) + expr_has_error(ex) && return nothing if (isexpr(ex, :toplevel) || isexpr(ex, :tuple)) && !isempty(ex.args) # get the inference result for the last expression ex = ex.args[end] @@ -778,6 +711,7 @@ code_typed(CC.typeinf, (REPLInterpreter, CC.InferenceState)) # Method completion on function call expression that look like :(max(1)) MAX_METHOD_COMPLETIONS::Int = 40 function _complete_methods(ex_org::Expr, context_module::Module, shift::Bool) + isempty(ex_org.args) && return 2, nothing, [], Set{Symbol}() funct = repl_eval_ex(ex_org.args[1], context_module) funct === nothing && return 2, nothing, [], Set{Symbol}() funct = CC.widenconst(funct) @@ -935,50 +869,6 @@ const subscript_regex = Regex("^\\\\_[" * join(isdigit(k) || isletter(k) ? "$k" const superscripts = Dict(k[3]=>v[1] for (k,v) in latex_symbols if startswith(k, "\\^") && length(k)==3) const superscript_regex = Regex("^\\\\\\^[" * join(isdigit(k) || isletter(k) ? "$k" : "\\$k" for k in keys(superscripts)) * "]+\\z") -# Aux function to detect whether we're right after a using or import keyword -function get_import_mode(s::String) - # allow all of these to start with leading whitespace and macros like @eval and @eval( - # ^\s*(?:@\w+\s*(?:\(\s*)?)? - - # match simple cases like `using |` and `import |` - mod_import_match_simple = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s*$", s) - if mod_import_match_simple !== nothing - if mod_import_match_simple[1] == "using" - return :using_module - else - return :import_module - end - end - # match module import statements like `using Foo|`, `import Foo, Bar|` and `using Foo.Bar, Baz, |` - mod_import_match = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s+([\w\.]+(?:\s*,\s*[\w\.]+)*),?\s*$", s) - if mod_import_match !== nothing - if mod_import_match.captures[1] == "using" - return :using_module - else - return :import_module - end - end - # now match explicit name import statements like `using Foo: |` and `import Foo: bar, baz|` - name_import_match = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s+([\w\.]+)\s*:\s*([\w@!\s,]+)$", s) - if name_import_match !== nothing - if name_import_match[1] == "using" - return :using_name - else - return :import_name - end - end - return nothing -end - -function close_path_completion(dir, path, str, pos) - path = unescape_string(replace(path, "\\\$"=>"\$")) - path = joinpath(dir, path) - # ...except if it's a directory... - Base.isaccessibledir(path) && return false - # ...and except if there's already a " at the cursor. - return lastindex(str) <= pos || str[nextind(str, pos)] != '"' -end - function bslash_completions(string::String, pos::Int, hint::Bool=false) slashpos = something(findprev(isequal('\\'), string, pos), 0) if (something(findprev(in(bslash_separators), string, pos), 0) < slashpos && @@ -1006,29 +896,7 @@ function bslash_completions(string::String, pos::Int, hint::Bool=false) completions = Completion[BslashCompletion(name, "$(symbol_dict[name]) $name") for name in sort!(collect(namelist))] return (true, (completions, slashpos:pos, true)) end - return (false, (Completion[], 0:-1, false)) -end - -function dict_identifier_key(str::String, tag::Symbol, context_module::Module=Main) - if tag === :string - str_close = str*"\"" - elseif tag === :cmd - str_close = str*"`" - else - str_close = str - end - frange, end_of_identifier = find_start_brace(str_close, c_start='[', c_end=']') - isempty(frange) && return (nothing, nothing, nothing) - objstr = str[1:end_of_identifier] - objex = Meta.parse(objstr, raise=false, depwarn=false) - objt = repl_eval_ex(objex, context_module) - isa(objt, Core.Const) || return (nothing, nothing, nothing) - obj = objt.val - isa(obj, AbstractDict) || return (nothing, nothing, nothing) - (Base.haslength(obj) && length(obj)::Int < 1_000_000) || return (nothing, nothing, nothing) - begin_of_key = something(findnext(!isspace, str, nextind(str, end_of_identifier) + 1), # +1 for [ - lastindex(str)+1) - return (obj, str[begin_of_key:end], begin_of_key) + return (false, (Completion[], 1:0, false)) end # This needs to be a separate non-inlined function, see #19441 @@ -1041,45 +909,12 @@ end return matches end -# Identify an argument being completed in a method call. If the argument is empty, method -# suggestions will be provided instead of argument completions. -function identify_possible_method_completion(partial, last_idx) - fail = 0:-1, Expr(:nothing), 0:-1, 0 - - # First, check that the last punctuation is either ',', ';' or '(' - idx_last_punct = something(findprev(x -> ispunct(x) && x != '_' && x != '!', partial, last_idx), 0)::Int - idx_last_punct == 0 && return fail - last_punct = partial[idx_last_punct] - last_punct == ',' || last_punct == ';' || last_punct == '(' || return fail - - # Then, check that `last_punct` is only followed by an identifier or nothing - before_last_word_start = something(findprev(in(non_identifier_chars), partial, last_idx), 0) - before_last_word_start == 0 && return fail - all(isspace, @view partial[nextind(partial, idx_last_punct):before_last_word_start]) || return fail - - # Check that `last_punct` is either the last '(' or placed after a previous '(' - frange, method_name_end = find_start_brace(@view partial[1:idx_last_punct]) - method_name_end ∈ frange || return fail - - # Strip the preceding ! operators, if any, and close the expression with a ')' - s = replace(partial[frange], r"\G\!+([^=\(]+)" => s"\1"; count=1) * ')' - ex = Meta.parse(s, raise=false, depwarn=false) - isa(ex, Expr) || return fail - - # `wordrange` is the position of the last argument to complete - wordrange = nextind(partial, before_last_word_start):last_idx - return frange, ex, wordrange, method_name_end -end - # Provide completion for keyword arguments in function calls -function complete_keyword_argument(partial::String, last_idx::Int, context_module::Module; - shift::Bool=false) - frange, ex, wordrange, = identify_possible_method_completion(partial, last_idx) - fail = Completion[], 0:-1, frange - ex.head === :call || is_broadcasting_expr(ex) || return fail - +function complete_keyword_argument!(suggestions::Vector{Completion}, + ex::Expr, last_word::String, + context_module::Module; shift::Bool=false) kwargs_flag, funct, args_ex, kwargs_ex = _complete_methods(ex, context_module, true)::Tuple{Int, Any, Vector{Any}, Set{Symbol}} - kwargs_flag == 2 && return fail # one of the previous kwargs is invalid + kwargs_flag == 2 && false # one of the previous kwargs is invalid methods = Completion[] complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, shift ? -1 : MAX_METHOD_COMPLETIONS, kwargs_flag == 1) @@ -1091,7 +926,6 @@ function complete_keyword_argument(partial::String, last_idx::Int, context_modul # previously in the expression. The corresponding suggestion is "kwname=". # If the keyword corresponds to an existing name, also include "kwname" as a suggestion # since the syntax "foo(; kwname)" is equivalent to "foo(; kwname=kwname)". - last_word = partial[wordrange] # the word to complete kwargs = Set{String}() for m in methods # if MAX_METHOD_COMPLETIONS is hit a single TextCompletion is return by complete_methods! with an explanation @@ -1102,22 +936,18 @@ function complete_keyword_argument(partial::String, last_idx::Int, context_modul current_kwarg_candidates = String[] for _kw in possible_kwargs kw = String(_kw) - if !endswith(kw, "...") && startswith(kw, last_word) && _kw ∉ kwargs_ex + # HACK: Should consider removing current arg from AST. + if !endswith(kw, "...") && startswith(kw, last_word) && (_kw ∉ kwargs_ex || kw == last_word) push!(current_kwarg_candidates, kw) end end union!(kwargs, current_kwarg_candidates) end - suggestions = Completion[KeywordArgumentCompletion(kwarg) for kwarg in kwargs] - - # Only add these if not in kwarg space. i.e. not in `foo(; ` - if kwargs_flag == 0 - complete_symbol!(suggestions, #=prefix=#nothing, last_word, context_module; shift) - complete_keyval!(suggestions, last_word) + for kwarg in kwargs + push!(suggestions, KeywordArgumentCompletion(kwarg)) end - - return sort!(suggestions, by=named_completion_completion), wordrange + return kwargs_flag != 0 end function get_loading_candidates(pkgstarts::String, project_file::String) @@ -1136,411 +966,367 @@ function get_loading_candidates(pkgstarts::String, project_file::String) return loading_candidates end -function complete_loading_candidates!(suggestions::Vector{Completion}, pkgstarts::String, project_file::String) - for name in get_loading_candidates(pkgstarts, project_file) - push!(suggestions, PackageCompletion(name)) +function complete_loading_candidates!(suggestions::Vector{Completion}, s::String) + for name in ("Core", "Base") + startswith(name, s) && push!(suggestions, PackageCompletion(name)) end - return suggestions -end -function complete_identifiers!(suggestions::Vector{Completion}, - context_module::Module, string::String, name::String, - pos::Int, separatorpos::Int, startpos::Int; - comp_keywords::Bool=false, - complete_modules_only::Bool=false, - shift::Bool=false) - if comp_keywords - complete_keyword!(suggestions, name) - complete_keyval!(suggestions, name) - end - if separatorpos > 1 && (string[separatorpos] == '.' || string[separatorpos] == ':') - s = string[1:prevind(string, separatorpos)] - # First see if the whole string up to `pos` is a valid expression. If so, use it. - prefix = Meta.parse(s, raise=false, depwarn=false) - if isexpr(prefix, :incomplete) - s = string[startpos:pos] - # Heuristic to find the start of the expression. TODO: This would be better - # done with a proper error-recovering parser. - if 0 < startpos <= lastindex(string) && string[startpos] == '.' - i = prevind(string, startpos) - while 0 < i - c = string[i] - if c in (')', ']') - if c == ')' - c_start = '(' - c_end = ')' - elseif c == ']' - c_start = '[' - c_end = ']' - end - frange, end_of_identifier = find_start_brace(string[1:prevind(string, i)], c_start=c_start, c_end=c_end) - isempty(frange) && break # unbalanced parens - startpos = first(frange) - i = prevind(string, startpos) - elseif c in ('\'', '\"', '\`') - s = "$c$c"*string[startpos:pos] - break + # If there's no dot, we're in toplevel, so we should + # also search for packages + for dir in Base.load_path() + if basename(dir) in Base.project_names && isfile(dir) + for name in get_loading_candidates(s, dir) + push!(suggestions, PackageCompletion(name)) + end + end + isdir(dir) || continue + for entry in _readdirx(dir) + pname = entry.name + if pname[1] != '.' && pname != "METADATA" && + pname != "REQUIRE" && startswith(pname, s) + # Valid file paths are + # .jl + # /src/.jl + # .jl/src/.jl + if isfile(entry) + endswith(pname, ".jl") && push!(suggestions, + PackageCompletion(pname[1:prevind(pname, end-2)])) + else + mod_name = if endswith(pname, ".jl") + pname[1:prevind(pname, end-2)] else - break + pname end - s = string[startpos:pos] - end - end - if something(findlast(in(non_identifier_chars), s), 0) < something(findlast(isequal('.'), s), 0) - lookup_name, name = rsplit(s, ".", limit=2) - name = String(name) - prefix = Meta.parse(lookup_name, raise=false, depwarn=false) - end - isexpr(prefix, :incomplete) && (prefix = nothing) - elseif isexpr(prefix, (:using, :import)) - arglast = prefix.args[end] # focus on completion to the last argument - if isexpr(arglast, :.) - # We come here for cases like: - # - `string`: "using Mod1.Mod2.M" - # - `ex`: :(using Mod1.Mod2) - # - `name`: "M" - # Now we transform `ex` to `:(Mod1.Mod2)` to allow `complete_symbol!` to - # complete for inner modules whose name starts with `M`. - # Note that `complete_modules_only=true` is set within `completions` - prefix = nothing - firstdot = true - for arg = arglast.args - if arg === :. - # override `context_module` if multiple `.` accessors are used - if firstdot - firstdot = false - else - context_module = parentmodule(context_module) - end - elseif arg isa Symbol - if prefix === nothing - prefix = arg - else - prefix = Expr(:., prefix, QuoteNode(arg)) - end - else # invalid expression - prefix = nothing - break + if isfile(joinpath(entry, "src", + "$mod_name.jl")) + push!(suggestions, PackageCompletion(mod_name)) end end end - elseif isexpr(prefix, :call) && length(prefix.args) > 1 - isinfix = s[end] != ')' - # A complete call expression that does not finish with ')' is an infix call. - if !isinfix - # Handle infix call argument completion of the form bar + foo(qux). - frange, end_of_identifier = find_start_brace(@view s[1:prevind(s, end)]) - if !isempty(frange) # if find_start_brace fails to find the brace just continue - isinfix = Meta.parse(@view(s[frange[1]:end]), raise=false, depwarn=false) == prefix.args[end] - end - end - if isinfix - prefix = prefix.args[end] - end - elseif isexpr(prefix, :macrocall) && length(prefix.args) > 1 - # allow symbol completions within potentially incomplete macrocalls - if s[end] ≠ '`' && s[end] ≠ ')' - prefix = prefix.args[end] - end end - else - prefix = nothing end - complete_symbol!(suggestions, prefix, name, context_module; complete_modules_only, shift) - return suggestions end function completions(string::String, pos::Int, context_module::Module=Main, shift::Bool=true, hint::Bool=false) - # First parse everything up to the current position - partial = string[1:pos] - inc_tag = Base.incomplete_tag(Meta.parse(partial, raise=false, depwarn=false)) - - if !hint # require a tab press for completion of these - # ?(x, y)TAB lists methods you can call with these objects - # ?(x, y TAB lists methods that take these objects as the first two arguments - # MyModule.?(x, y)TAB restricts the search to names in MyModule - rexm = match(r"(\w+\.|)\?\((.*)$", partial) - if rexm !== nothing - # Get the module scope - if isempty(rexm.captures[1]) - callee_module = context_module - else - modname = Symbol(rexm.captures[1][1:end-1]) - if isdefined(context_module, modname) - callee_module = getfield(context_module, modname) - if !isa(callee_module, Module) - callee_module = context_module - end - else - callee_module = context_module - end - end - moreargs = !endswith(rexm.captures[2], ')') - callstr = "_(" * rexm.captures[2] - if moreargs - callstr *= ')' - end - ex_org = Meta.parse(callstr, raise=false, depwarn=false) - if isa(ex_org, Expr) - return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:length(rexm.captures[1])+1) .+ rexm.offset, false - end - end - end + # filename needs to be string so macro can be evaluated + node = parseall(CursorNode, string, ignore_errors=true, keep_parens=true, filename="none") + cur = @something seek_pos(node, pos) node - # if completing a key in a Dict - identifier, partial_key, loc = dict_identifier_key(partial, inc_tag, context_module) - if identifier !== nothing - matches = find_dict_matches(identifier, partial_key) - length(matches)==1 && (lastindex(string) <= pos || string[nextind(string,pos)] != ']') && (matches[1]*=']') - length(matches)>0 && return Completion[DictCompletion(identifier, match) for match in sort!(matches)], loc::Int:pos, true - end + # Back up before whitespace to get a more useful AST node. + pos_not_ws = findprev(!isspace, string, pos) + cur_not_ws = something(seek_pos(node, pos_not_ws), node) suggestions = Completion[] + sort_suggestions() = sort!(unique!(named_completion, suggestions), by=named_completion_completion) + + # Search for methods (requires tab press): + # ?(x, y)TAB lists methods you can call with these objects + # ?(x, y TAB lists methods that take these objects as the first two arguments + # MyModule.?(x, y)TAB restricts the search to names in MyModule + if !hint + cs = method_search(view(string, 1:pos), context_module, shift) + cs !== nothing && return cs + end - # Check if this is a var"" string macro that should be completed like - # an identifier rather than a string. - # TODO: It would be nice for the parser to give us more information here - # so that we can lookup the macro by identity rather than pattern matching - # its invocation. - varrange = findprev("var\"", string, pos) - - expanded = nothing - was_expanded = false - - if varrange !== nothing - ok, ret = bslash_completions(string, pos) - ok && return ret - startpos = first(varrange) + 4 - separatorpos = something(findprev(isequal('.'), string, first(varrange)-1), 0) - name = string[startpos:pos] - complete_identifiers!(suggestions, context_module, string, name, - pos, separatorpos, startpos; - shift) - return sort!(unique!(named_completion, suggestions), by=named_completion_completion), (separatorpos+1):pos, true - elseif inc_tag === :cmd - # TODO: should this call shell_completions instead of partially reimplementing it? - let m = match(r"[\t\n\r\"`><=*?|]| (?!\\)", reverse(partial)) # fuzzy shell_parse in reverse - startpos = nextind(partial, reverseind(partial, m.offset)) - r = startpos:pos - scs::String = string[r] - - expanded = complete_expanduser(scs, r) - was_expanded = expanded[3] - if was_expanded - scs = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(scs) && return expanded::Completions - end - - path::String = replace(scs, r"(\\+)\g1(\\?)`" => "\1\2`") # fuzzy unescape_raw_string: match an even number of \ before ` and replace with half as many - # This expansion with "\\ "=>' ' replacement and shell_escape=true - # assumes the path isn't further quoted within the cmd backticks. - path = replace(path, r"\\ " => " ", r"\$" => "\$") # fuzzy shell_parse (reversed by shell_escape_posixly) - paths, dir, success = complete_path(path, shell_escape=true, raw_escape=true) - - if success && !isempty(dir) - let dir = do_raw_escape(do_shell_escape(dir)) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - if endswith(dir, "/") && startswith(scs, dir) - r = (startpos + sizeof(dir)):pos - elseif startswith(scs, dir * "/") - r = nextind(string, startpos + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - p = dir * "/" * c.path - was_expanded && (p = contractuser(p)) - return PathCompletion(p) - end - end - end - end - if isempty(paths) && !hint && was_expanded - # if not able to provide completions, not hinting, and ~ expansion was possible, return ~ expansion - return expanded::Completions - else - return sort!(paths, by=p->p.path), r::UnitRange{Int}, success + # Complete keys in a Dict: + # my_dict[ TAB + n, key, closed = find_ref_key(cur_not_ws, pos) + if n !== nothing + key::UnitRange{Int} + obj = dict_eval(Expr(n), context_module) + if obj !== nothing + # Skip leading whitespace inside brackets. + i = @something findnext(!isspace, string, first(key)) nextind(string, last(key)) + key = i:last(key) + s = string[intersect(key, 1:pos)] + matches = find_dict_matches(obj, s) + length(matches) == 1 && !closed && (matches[1] *= ']') + if length(matches) > 0 + ret = Completion[DictCompletion(obj, match) for match in sort!(matches)] + return ret, key, true end end - elseif inc_tag === :string - # Find first non-escaped quote - let m = match(r"\"(?!\\)", reverse(partial)) - startpos = nextind(partial, reverseind(partial, m.offset)) - r = startpos:pos - scs::String = string[r] - - expanded = complete_expanduser(scs, r) - was_expanded = expanded[3] - if was_expanded - scs = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(scs) && return expanded::Completions - end - - path = try - unescape_string(replace(scs, "\\\$"=>"\$")) - catch ex - ex isa ArgumentError || rethrow() - nothing - end - if !isnothing(path) - paths, dir, success = complete_path(path::String, string_escape=true) - - if length(paths) == 1 - p = (paths[1]::PathCompletion).path - hint && was_expanded && (p = contractuser(p)) - if close_path_completion(dir, p, path, pos) - paths[1] = PathCompletion(p * "\"") - end - end + end - if success && !isempty(dir) - let dir = do_string_escape(dir) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - if endswith(dir, "/") && startswith(scs, dir) - r = (startpos + sizeof(dir)):pos - elseif startswith(scs, dir * "/") && dir != dirname(homedir()) - was_expanded && (dir = contractuser(dir)) - r = nextind(string, startpos + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - p = dir * "/" * c.path - hint && was_expanded && (p = contractuser(p)) - return PathCompletion(p) - end - end - end - end + # Complete Cmd strings: + # `fil TAB => `file + # `file ~/exa TAB => `file ~/example.txt + # `file ~/example.txt TAB => `file /home/user/example.txt + if (n = find_parent(cur, K"CmdString")) !== nothing + off = n.position - 1 + ret, r, success = shell_completions(string[char_range(n)], pos - off, hint, cmd_escape=true) + success && return ret, r .+ off, success + end - # Fallthrough allowed so that Latex symbols can be completed in strings - if success - return sort!(paths, by=p->p.path), r::UnitRange{Int}, success - elseif !hint && was_expanded - # if not able to provide completions, not hinting, and ~ expansion was possible, return ~ expansion - return expanded::Completions - end - end + # Complete ordinary strings: + # "~/exa TAB => "~/example.txt" + # "~/example.txt TAB => "/home/user/example.txt" + r, closed = find_str(cur) + if r !== nothing + s = do_string_unescape(string[r]) + ret, success = complete_path_string(s, hint; string_escape=true, + dirsep=Sys.iswindows() ? '\\' : '/') + if length(ret) == 1 && !closed && close_path_completion(ret[1].path) + ret[1] = PathCompletion(ret[1].path * '"') end + success && return ret, r, success end - # if path has ~ and we didn't find any paths to complete just return the expanded path - was_expanded && return expanded::Completions + # Backlash symbols: + # \pi => π + # Comes after string completion so backslash escapes are not misinterpreted. ok, ret = bslash_completions(string, pos) ok && return ret - # Make sure that only bslash_completions is working on strings - inc_tag === :string && return Completion[], 0:-1, false - if inc_tag === :other - frange, ex, wordrange, method_name_end = identify_possible_method_completion(partial, pos) - if last(frange) != -1 && all(isspace, @view partial[wordrange]) # no last argument to complete - if ex.head === :call - return complete_methods(ex, context_module, shift), first(frange):method_name_end, false - elseif is_broadcasting_expr(ex) - return complete_methods(ex, context_module, shift), first(frange):(method_name_end - 1), false - end + # Don't fall back to symbol completion inside strings or comments. + inside_cmdstr = find_parent(cur, K"cmdstring") !== nothing + (kind(cur) in KSet"String Comment ErrorEofMultiComment" || inside_cmdstr) && + return Completion[], 1:0, false + + if (n = find_prefix_call(cur_not_ws)) !== nothing + func = first(children_nt(n)) + e = Expr(n) + # Remove arguments past the first parse error (allows unclosed parens) + if is_broadcasting_expr(e) + i = findfirst(x -> x isa Expr && x.head == :error, e.args[2].args) + i !== nothing && deleteat!(e.args[2].args, i:lastindex(e.args[2].args)) + else + i = findfirst(x -> x isa Expr && x.head == :error, e.args) + i !== nothing && deleteat!(e.args, i:lastindex(e.args)) end - elseif inc_tag === :comment - return Completion[], 0:-1, false - end - # Check whether we can complete a keyword argument in a function call - kwarg_completion, wordrange = complete_keyword_argument(partial, pos, context_module; shift) - isempty(wordrange) || return kwarg_completion, wordrange, !isempty(kwarg_completion) + # Method completion: + # foo( TAB => list of method signatures for foo + # foo(x, TAB => list of methods signatures for foo with x as first argument + if kind(cur_not_ws) in KSet"( , ;" + # Don't provide method completions unless the cursor is after: '(' ',' ';' + return complete_methods(e, context_module, shift), char_range(func), false + + # Keyword argument completion: + # foo(ar TAB => keyword arguments like `arg1=` + elseif kind(cur) == K"Identifier" + r = char_range(cur) + s = string[intersect(r, 1:pos)] + # Return without adding more suggestions if kwargs only + complete_keyword_argument!(suggestions, e, s, context_module; shift) && + return sort_suggestions(), r, true + end + end - startpos = nextind(string, something(findprev(in(non_identifier_chars), string, pos), 0)) - # strip preceding ! operator - if (m = match(r"\G\!+", partial, startpos)) isa RegexMatch - startpos += length(m.match) + # Symbol completion + # TODO: Should completions replace the identifier at the cursor? + if cur.parent !== nothing && kind(cur.parent) == K"var" + # Replace the entire var"foo", but search using only "foo". + r = intersect(char_range(cur.parent), 1:pos) + r2 = char_range(children_nt(cur.parent)[1]) + s = string[intersect(r2, 1:pos)] + elseif kind(cur) in KSet"Identifier @" + r = intersect(char_range(cur), 1:pos) + s = string[r] + elseif kind(cur) == K"MacroName" + # Include the `@` + r = intersect(prevind(string, cur.position):char_last(cur), 1:pos) + s = string[r] + else + r = nextind(string, pos):pos + s = "" end - separatorpos = something(findprev(isequal('.'), string, pos), 0) - namepos = max(startpos, separatorpos+1) - name = string[namepos:pos] - import_mode = get_import_mode(string) - if import_mode === :using_module || import_mode === :import_module + complete_modules_only = false + prefix = node_prefix(cur, context_module) + comp_keywords = prefix === nothing + + # Complete loadable module names: + # import Mod TAB + # import Mod1, Mod2 TAB + # using Mod TAB + if (n = find_parent(cur, K"importpath")) !== nothing # Given input lines like `using Foo|`, `import Foo, Bar|` and `using Foo.Bar, Baz, |`: # Let's look only for packages and modules we can reach from here - - # If there's no dot, we're in toplevel, so we should - # also search for packages - s = string[startpos:pos] - if separatorpos <= startpos - for dir in Base.load_path() - if basename(dir) in Base.project_names && isfile(dir) - complete_loading_candidates!(suggestions, s, dir) - end - isdir(dir) || continue - for entry in _readdirx(dir) - pname = entry.name - if pname[1] != '.' && pname != "METADATA" && - pname != "REQUIRE" && startswith(pname, s) - # Valid file paths are - # .jl - # /src/.jl - # .jl/src/.jl - if isfile(entry) - endswith(pname, ".jl") && push!(suggestions, - PackageCompletion(pname[1:prevind(pname, end-2)])) - else - mod_name = if endswith(pname, ".jl") - pname[1:prevind(pname, end-2)] - else - pname - end - if isfile(joinpath(entry, "src", - "$mod_name.jl")) - push!(suggestions, PackageCompletion(mod_name)) - end - end - end - end - end + if prefix == nothing + complete_loading_candidates!(suggestions, s) + return sort_suggestions(), r, true end + + # Allow completion for `import Mod.name` (where `name` is not a module) + complete_modules_only = prefix == nothing || kind(n.parent) == K"using" comp_keywords = false - complete_modules_only = import_mode === :using_module # allow completion for `import Mod.name` (where `name` is not a module) - elseif import_mode === :using_name || import_mode === :import_name - # `using Foo: |` and `import Foo: bar, baz|` - separatorpos = findprev(isequal(':'), string, pos)::Int - comp_keywords = false - complete_modules_only = false + end + + if comp_keywords + complete_keyword!(suggestions, s) + complete_keyval!(suggestions, s) + end + + complete_symbol!(suggestions, prefix, s, context_module; complete_modules_only, shift) + return sort_suggestions(), r, true +end + +function close_path_completion(path) + path = expanduser(path) + path = do_string_unescape(path) + !Base.isaccessibledir(path) +end + +# Lowering can misbehave with nested error expressions. +function expr_has_error(@nospecialize(e)) + e isa Expr || return false + e.head === :error && return true + any(expr_has_error, e.args) +end + +# Is the cursor inside the square brackets of a ref expression? If so, returns: +# - The ref node +# - The range of characters for the brackets +# - A flag indicating if the closing bracket is present +function find_ref_key(cur::CursorNode, pos::Int) + n = find_parent(cur, K"ref") + n !== nothing || return nothing, nothing, nothing + key, closed = find_delim(n, K"[", K"]") + first(key) - 1 <= pos <= last(key) || return nothing, nothing, nothing + n, key, closed +end + +# If the cursor is in a literal string, return the contents and char range +# inside the quotes. Ignores triple strings. +function find_str(cur::CursorNode) + n = find_parent(cur, K"string") + n !== nothing || return nothing, nothing + find_delim(n, K"\"", K"\"") +end + +# Is the cursor directly inside of the arguments of a prefix call (no nested +# expressions)? +function find_prefix_call(cur::CursorNode) + n = cur.parent + n !== nothing || return nothing + is_call(n) = kind(n) in KSet"call dotcall" && is_prefix_call(n) + if kind(n) == K"parameters" + is_call(n.parent) || return nothing + n.parent else - comp_keywords = !isempty(name) && startpos > separatorpos - complete_modules_only = false + # Check that we are beyond the function name. + is_call(n) && cur.index > children_nt(n)[1].index || return nothing + n + end +end + +# If node is the field in a getfield-like expression, return the value +# complete_symbol! should use as the prefix. +function node_prefix(node::CursorNode, context_module::Module) + node.parent !== nothing || return nothing + p = node.parent + # In x.var"y", the parent is the "var" when the cursor is on "y". + kind(p) == K"var" && (p = p.parent) + + # expr.node => expr + if kind(p) == K"." + n = children_nt(p)[1] + # Don't use prefix if we are the value + n !== node || return nothing + return Expr(n) end - complete_identifiers!(suggestions, context_module, string, name, - pos, separatorpos, startpos; - comp_keywords, complete_modules_only, shift) - return sort!(unique!(named_completion, suggestions), by=named_completion_completion), namepos:pos, true + if kind(p) == K"importpath" + if p.parent !== nothing && kind(p.parent) == K":" && p.index_nt > 1 + # import A.B: C.node + chain = children_nt(children_nt(p.parent)[1]) + append!(chain, children_nt(p)[1:end-1]) + else + # import A.node + # import A.node: ... + chain = children_nt(p)[1:node.index_nt] + # Don't include the node under cursor in prefix unless it is `.` + kind(chain[end]) != K"." && deleteat!(chain, lastindex(chain)) + end + length(chain) > 0 || return nothing + + # (:importpath :x :y :z) => (:. (:. :x :y) :z) + # (:importpath :. :. :z) => (:. (parentmodule context_module) :z) + if (i = findlast(x -> kind(x) == K".", chain)) !== nothing + init = context_module + for j in 2:i + init = parentmodule(init) + end + deleteat!(chain, 1:i) + else + # No leading `.`, init is the first element of the path + init = chain[1].val + deleteat!(chain, 1) + end + + # Convert the "chain" into nested (. a b) expressions. + all(x -> kind(x) == K"Identifier", chain) || return nothing + return foldl((x, y) -> Expr(:., x, Expr(:quote, y.val)), chain; init) + end + + nothing +end + +function dict_eval(@nospecialize(e), context_module::Module=Main) + objt = repl_eval_ex(e.args[1], context_module) + isa(objt, Core.Const) || return nothing + obj = objt.val + isa(obj, AbstractDict) || return nothing + (Base.haslength(obj) && length(obj)::Int < 1_000_000) || return nothing + return obj end -function shell_completions(string, pos, hint::Bool=false) +function method_search(partial::AbstractString, context_module::Module, shift::Bool) + rexm = match(r"(\w+\.|)\?\((.*)$", partial) + if rexm !== nothing + # Get the module scope + if isempty(rexm.captures[1]) + callee_module = context_module + else + modname = Symbol(rexm.captures[1][1:end-1]) + if isdefined(context_module, modname) + callee_module = getfield(context_module, modname) + if !isa(callee_module, Module) + callee_module = context_module + end + else + callee_module = context_module + end + end + moreargs = !endswith(rexm.captures[2], ')') + callstr = "_(" * rexm.captures[2] + if moreargs + callstr *= ')' + end + ex_org = Meta.parse(callstr, raise=false, depwarn=false) + if isa(ex_org, Expr) + return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:length(rexm.captures[1])+1) .+ rexm.offset, false + end + end +end + +function shell_completions(str, pos, hint::Bool=false; cmd_escape::Bool=false) # First parse everything up to the current position - scs = string[1:pos] + scs = str[1:pos] args, last_arg_start = try Base.shell_parse(scs, true)::Tuple{Expr,Int} catch ex ex isa ArgumentError || ex isa ErrorException || rethrow() - return Completion[], 0:-1, false + return Completion[], 1:0, false end ex = args.args[end]::Expr # Now look at the last thing we parsed - isempty(ex.args) && return Completion[], 0:-1, false - lastarg = ex.args[end] + isempty(ex.args) && return Completion[], 1:0, false + # Concatenate every string fragment so dir\file completes correctly. + lastarg = all(x -> x isa String, ex.args) ? string(ex.args...) : ex.args[end] + # As Base.shell_parse throws away trailing spaces (unless they are escaped), # we need to special case here. # If the last char was a space, but shell_parse ignored it search on "". if isexpr(lastarg, :incomplete) || isexpr(lastarg, :error) - partial = string[last_arg_start:pos] + partial = str[last_arg_start:pos] ret, range = completions(partial, lastindex(partial), Main, true, hint) range = range .+ (last_arg_start - 1) return ret, range, true elseif endswith(scs, ' ') && !endswith(scs, "\\ ") r = pos+1:pos - paths, dir, success = complete_path("", use_envpath=false, shell_escape=true) + paths, dir, success = complete_path(""; use_envpath=false, shell_escape=!cmd_escape, cmd_escape, dirsep='/') return paths, r, success elseif all(@nospecialize(arg) -> arg isa AbstractString, ex.args) # Join these and treat this as a path @@ -1550,44 +1336,63 @@ function shell_completions(string, pos, hint::Bool=false) # Also try looking into the env path if the user wants to complete the first argument use_envpath = length(args.args) < 2 - expanded = complete_expanduser(path, r) - was_expanded = expanded[3] - if was_expanded - path = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(path) && return expanded::Completions - end + paths, success = complete_path_string(path, hint; use_envpath, shell_escape=!cmd_escape, cmd_escape, dirsep='/') + return paths, r, success + end + return Completion[], 1:0, false +end - paths, dir, success = complete_path(path, use_envpath=use_envpath, shell_escape=true, contract_user=was_expanded) - - if success && !isempty(dir) - let dir = do_shell_escape(dir) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - partial = string[last_arg_start:pos] - if endswith(dir, "/") && startswith(partial, dir) - r = (last_arg_start + sizeof(dir)):pos - elseif startswith(partial, dir * "/") - r = nextind(string, last_arg_start + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - return PathCompletion(dir * "/" * c.path) - end - end - end +function complete_path_string(path, hint::Bool=false; + shell_escape::Bool=false, + cmd_escape::Bool=false, + string_escape::Bool=false, + dirsep='/', + kws...) + # Expand "~" and remember if we expanded it. + local expanded + try + let p = expanduser(path) + expanded = path != p + path = p end - # if ~ was expanded earlier and the incomplete string isn't a path - # return the path with contracted user to match what the hint shows. Otherwise expand ~ - # i.e. require two tab presses to expand user - if was_expanded && !ispath(path) - map!(paths, paths) do c::PathCompletion - PathCompletion(contractuser(c.path)) - end + catch e + e isa ArgumentError || rethrow() + expanded = false + end + + function escape(p) + shell_escape && (p = do_shell_escape(p)) + string_escape && (p = do_string_escape(p)) + cmd_escape && (p = do_cmd_escape(p)) + p + end + + paths, dir, success = complete_path(path; dirsep, kws...) + + # Expand '~' if the user hits TAB after exhausting completions (either + # because we have found an existing file, or there is no such file). + full_path = try + ispath(path) || isempty(paths) + catch err + # access(2) errors unhandled by ispath: EACCES, EIO, ELOOP, ENAMETOOLONG + if err isa Base.IOError + false + elseif err isa Base.ArgumentError && occursin("embedded NULs", err.msg) + false + else + rethrow() end - return paths, r, success end - return Completion[], 0:-1, false + expanded && !hint && full_path && return Completion[PathCompletion(escape(path))], true + + # Expand '~' if the user hits TAB on a path ending in '/'. + expanded && (hint || path != dir * "/") && (dir = contractuser(dir)) + + map!(paths) do c::PathCompletion + p = joinpath_withsep(dir, c.path; dirsep) + PathCompletion(escape(p)) + end + return sort!(paths, by=p->p.path), success end function __init__() diff --git a/stdlib/REPL/src/SyntaxUtil.jl b/stdlib/REPL/src/SyntaxUtil.jl new file mode 100644 index 0000000000000..6b455aa16dc9f --- /dev/null +++ b/stdlib/REPL/src/SyntaxUtil.jl @@ -0,0 +1,111 @@ +module SyntaxUtil + +import Base.JuliaSyntax: build_tree +using Base.JuliaSyntax: + AbstractSyntaxData, GreenNode, Kind, ParseStream, SourceFile, SyntaxHead, SyntaxNode, TreeNode, + byte_range, children, first_byte, head, is_leaf, is_trivia, kind, parse_julia_literal, span, + @K_str, _unsafe_wrap_substring + +export CursorNode, char_range, char_last, children_nt, find_delim, seek_pos + +# Like SyntaxNode, but keeps trivia, and tracks each child's index in its parent. +# Extracted from JuliaSyntax/src/syntax_tree.jl +# TODO: don't duplicate so much code? +struct CursorData <: AbstractSyntaxData + source::SourceFile + raw::GreenNode{SyntaxHead} + position::Int + index::Int + index_nt::Int # nth non-trivia in parent + val::Any +end + +const CursorNode = TreeNode{CursorData} + +function CursorNode(source::SourceFile, raw::GreenNode{SyntaxHead}; + position::Integer=1) + GC.@preserve source begin + raw_offset, txtbuf = _unsafe_wrap_substring(source.code) + offset = raw_offset - source.byte_offset + _to_CursorNode(source, txtbuf, offset, raw, convert(Int, position)) + end +end + +function _to_CursorNode(source::SourceFile, txtbuf::Vector{UInt8}, offset::Int, + raw::GreenNode{SyntaxHead}, + position::Int, index::Int=-1, index_nt::Int=-1) + if is_leaf(raw) + valrange = position:position + span(raw) - 1 + val = parse_julia_literal(txtbuf, head(raw), valrange .+ offset) + return CursorNode(nothing, nothing, CursorData(source, raw, position, index, index_nt, val)) + else + cs = CursorNode[] + pos = position + i_nt = 1 + for (i,rawchild) in enumerate(children(raw)) + push!(cs, _to_CursorNode(source, txtbuf, offset, rawchild, pos, i, i_nt)) + pos += Int(rawchild.span) + i_nt += !is_trivia(rawchild) + end + node = CursorNode(nothing, cs, CursorData(source, raw, position, index, index_nt, nothing)) + for c in cs + c.parent = node + end + return node + end +end + +function build_tree(::Type{CursorNode}, stream::ParseStream; + filename=nothing, first_line=1, kws...) + green_tree = build_tree(GreenNode, stream; kws...) + source = SourceFile(stream, filename=filename, first_line=first_line) + CursorNode(source, green_tree, position=first_byte(stream)) +end + +Base.show(io::IO, node::CursorNode) = show(io, MIME("text/plain"), node.raw) +Base.show(io::IO, mime::MIME{Symbol("text/plain")}, node::CursorNode) = show(io, mime, node.raw) + +function Base.Expr(node::CursorNode) + (; filename, first_line) = node.source + src = SourceFile(node.source[byte_range(node)]; filename, first_line) + Expr(SyntaxNode(src, node.raw)) +end + +char_range(node) = node.position:char_last(node) +char_last(node) = thisind(node.source, node.position + span(node) - 1) + +children_nt(node) = [n for n in children(node) if !is_trivia(n)] + +function seek_pos(node, pos) + pos in byte_range(node) || return nothing + (cs = children(node)) === nothing && return node + for n in cs + c = seek_pos(n, pos) + c === nothing || return c + end + node +end + +find_parent(node, k::Kind) = find_parent(node, n -> kind(n) == k) +function find_parent(node, f::Function) + while node !== nothing && !f(node) + node = node.parent + end + node +end + +# Return the character range between left_kind and right_kind in node. The left +# delimiter must be present, while the range will extend to the rest of the node +# if the right delimiter is missing. +function find_delim(node, left_kind, right_kind) + cs = children(node) + left = findfirst(c -> kind(c) == left_kind, cs) + left !== nothing || return nothing, nothing + right = findlast(c -> kind(c) == right_kind, cs) + closed = right !== nothing && right != left + right = closed ? thisind(node.source, cs[right].position - 1) : char_last(node) + left = nextind(node.source, char_last(cs[left])) + return left:right, closed +end + +end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index d9aa11cf609dd..442dff90cc039 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -158,6 +158,10 @@ let ex = export exported_symbol exported_symbol(::WeirdNames) = nothing + macro ignoremacro(e...) + nothing + end + end # module CompletionFoo test_repl_comp_dict = CompletionFoo.test_dict test_repl_comp_customdict = CompletionFoo.test_customdict @@ -180,9 +184,11 @@ end test_complete(s) = map_completion_text(@inferred(completions(s, lastindex(s)))) test_scomplete(s) = map_completion_text(@inferred(shell_completions(s, lastindex(s)))) +# | is reserved in test_complete_pos +test_complete_pos(s) = map_completion_text(@inferred(completions(replace(s, '|' => ""), findfirst('|', s)-1))) test_complete_context(s, m=@__MODULE__; shift::Bool=true) = map_completion_text(@inferred(completions(s,lastindex(s), m, shift))) -test_complete_foo(s) = test_complete_context(s, Main.CompletionFoo) +test_complete_foo(s; shift::Bool=true) = test_complete_context(s, Main.CompletionFoo; shift) test_complete_noshift(s) = map_completion_text(@inferred(completions(s, lastindex(s), Main, false))) test_bslashcomplete(s) = map_named_completion(@inferred(bslash_completions(s, lastindex(s)))[2]) @@ -290,10 +296,11 @@ let s = "Main.CompletionFoo.bar.no_val_available" @test length(c)==0 end -#cannot do dot completion on infix operator -let s = "+." - c, r = test_complete(s) - @test length(c)==0 +#cannot do dot completion on infix operator (get default completions) +let s1 = "", s2 = "+." + c1, r1 = test_complete(s1) + c2, r2 = test_complete(s2) + @test length(c1)==length(c2) end # To complete on a variable of a type, the type T of the variable @@ -458,13 +465,13 @@ let c, r, res = test_complete(s) @test !res @test all(m -> string(m) in c, methods(isnothing)) - @test s[r] == s[1:end-1] + @test s[r] == s[2:end-1] s = "!!isnothing(" c, r, res = test_complete(s) @test !res @test all(m -> string(m) in c, methods(isnothing)) - @test s[r] == s[1:end-1] + @test s[r] == s[3:end-1] end # Test completion of methods with input concrete args and args where typeinference determine their type @@ -1052,7 +1059,7 @@ end let c, r, res c, r, res = test_scomplete("\$a") @test c == String[] - @test r === 0:-1 + @test r === 1:0 @test res === false end @@ -1064,38 +1071,38 @@ let s, c, r # Issue #8047 s = "@show \"/dev/nul" c,r = test_complete(s) - @test "null\"" in c - @test r == 13:15 - @test s[r] == "nul" + @test "/dev/null\"" in c + @test r == 8:15 + @test s[r] == "/dev/nul" # Tests path in Julia code and not closing " if it's a directory # Issue #8047 s = "@show \"/tm" c,r = test_complete(s) - @test "tmp/" in c - @test r == 9:10 - @test s[r] == "tm" + @test "/tmp/" in c + @test r == 8:10 + @test s[r] == "/tm" # Tests path in Julia code and not double-closing " # Issue #8047 s = "@show \"/dev/nul\"" c,r = completions(s, 15) c = map(named_completion, c) - @test "null\"" in [_c.completion for _c in c] - @test r == 13:15 - @test s[r] == "nul" + @test "/dev/null" in [_c.completion for _c in c] + @test r == 8:15 + @test s[r] == "/dev/nul" s = "/t" c,r = test_scomplete(s) - @test "tmp/" in c - @test r == 2:2 - @test s[r] == "t" + @test "/tmp/" in c + @test r == 1:2 + @test s[r] == "/t" s = "/tmp" c,r = test_scomplete(s) - @test "tmp/" in c - @test r == 2:4 - @test s[r] == "tmp" + @test "/tmp/" in c + @test r == 1:4 + @test s[r] == "/tmp" # This should match things that are inside the tmp directory s = tempdir() @@ -1106,7 +1113,7 @@ let s, c, r c,r = test_scomplete(s) @test !("tmp/" in c) @test !("$s/tmp/" in c) - @test r === (sizeof(s) + 1):sizeof(s) + @test r === 1:sizeof(s) end s = "cd \$(Iter" @@ -1131,8 +1138,8 @@ let s, c, r touch(file) s = string(tempdir(), "/repl\\ ") c,r = test_scomplete(s) - @test ["'repl completions'"] == c - @test s[r] == "repl\\ " + @test [Base.shell_escape_posixly(joinpath(tempdir(), "repl completions"))] == c + @test s[r] == string(tempdir(), "/repl\\ ") rm(file) end @@ -1144,12 +1151,19 @@ let s, c, r mkdir(dir) s = "\"" * path * "/tmpfoob" c,r = test_complete(s) - @test "tmpfoobar/" in c - l = 3 + length(path) - @test r == l:l+6 - @test s[r] == "tmpfoob" + @test string(dir, "/") in c + @test r == 2:sizeof(s) + @test s[r] == joinpath(path, "tmpfoob") + + # Homedir expansion inside Cmd string (#57624) + s = "`ls " * path * "/tmpfoob" + c,r = test_complete(s) + @test string(dir, "/") in c + @test r == 5:sizeof(s) + @test s[r] == joinpath(path, "tmpfoob") + s = "\"~" - @test "tmpfoobar/" in c + @test joinpath(path, "tmpfoobar/") in c c,r = test_complete(s) s = "\"~user" c, r = test_complete(s) @@ -1248,7 +1262,7 @@ let current_dir, forbidden e isa Base.IOError && occursin("ELOOP", e.msg) end c, r = test_complete("\"$(escape_string(path))/selfsym") - @test c == ["selfsymlink\""] + @test c == [escape_string(joinpath(path, "selfsymlink")) * "\""] end end @@ -1285,47 +1299,47 @@ mktempdir() do path s = Sys.iswindows() ? "cd $dir_space\\\\space" : "cd $dir_space/space" c, r = test_scomplete(s) @test s[r] == (Sys.iswindows() ? "$dir_space\\\\space" : "$dir_space/space") - @test "'$space_folder'/'space .file'" in c + @test "'$space_folder/space .file'" in c # Also use shell escape rules within cmd backticks s = "`$s" c, r = test_scomplete(s) @test s[r] == (Sys.iswindows() ? "$dir_space\\\\space" : "$dir_space/space") - @test "'$space_folder'/'space .file'" in c + @test "'$space_folder/space .file'" in c # escape string according to Julia escaping rules julia_esc(str) = REPL.REPLCompletions.do_string_escape(str) # For normal strings the string should be properly escaped according to # the usual rules for Julia strings. - s = "cd(\"" * julia_esc(joinpath(path, space_folder) * "/space") + s = "cd(\"" * julia_esc(joinpath(path, space_folder, "space")) c, r = test_complete(s) - @test s[r] == "space" - @test "space .file\"" in c + @test s[r] == julia_esc(joinpath(path, space_folder, "space")) + @test julia_esc(joinpath(path, space_folder, "space .file")) * "\"" in c # '$' is the only character which can appear in a windows filename and # which needs to be escaped in Julia strings (on unix we could do this # test with all sorts of special chars) touch(joinpath(space_folder, "needs_escape\$.file")) - escpath = julia_esc(joinpath(path, space_folder) * "/needs_escape\$") + escpath = julia_esc(joinpath(path, space_folder, "needs_escape\$")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape\\\$" - @test "needs_escape\\\$.file\"" in c + @test s[r] == julia_esc(joinpath(path, space_folder, "needs_escape\$")) + @test julia_esc(joinpath(path, space_folder, "needs_escape\$.file")) * "\"" in c if !Sys.iswindows() touch(joinpath(space_folder, "needs_escape2\n\".file")) escpath = julia_esc(joinpath(path, space_folder, "needs_escape2\n\"")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape2\\n\\\"" - @test "needs_escape2\\n\\\".file\"" in c + @test s[r] == joinpath(path, space_folder, "needs_escape2\\n\\\"") + @test joinpath(path, space_folder, "needs_escape2\\n\\\".file\"") in c touch(joinpath(space_folder, "needs_escape3\\.file")) escpath = julia_esc(joinpath(path, space_folder, "needs_escape3\\")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape3\\\\" - @test "needs_escape3\\\\.file\"" in c + @test s[r] == joinpath(path, space_folder, "needs_escape3\\\\") + @test joinpath(path, space_folder, "needs_escape3\\\\.file\"") in c end # Test for issue #10324 @@ -1339,16 +1353,17 @@ mktempdir() do path test_dir = "test$(c)test" mkdir(joinpath(path, test_dir)) try - if !(c in ['\'','$']) # As these characters hold special meaning + # TODO: test on Windows when backslash-paths fixed + if !Sys.iswindows() && !(c in ['\'','$']) # As these characters hold special meaning # in shell commands the shell path completion cannot complete # paths with these characters c, r, res = test_scomplete(test_dir) - @test c[1] == "'$test_dir/'" + @test c[1] == "'$(joinpath(test_dir, ""))'" @test res end escdir = julia_esc(test_dir) c, r, res = test_complete("\""*escdir) - @test c[1] == escdir * "/" + @test c[1] == julia_esc(joinpath(test_dir, "")) @test res finally rm(joinpath(path, test_dir), recursive=true) @@ -1361,7 +1376,7 @@ end # Test tilde path completion let (c, r, res) = test_complete("\"~/ka8w5rsz") if !Sys.iswindows() - @test res && c == String[homedir() * "/ka8w5rsz"] + @test res && c == String[homedir() * "/ka8w5rsz\""] else @test !res end @@ -1376,7 +1391,7 @@ if !Sys.iswindows() try let (c, r, res) = test_complete("\"~/Zx6Wa0GkC") @test res - @test c == String["Zx6Wa0GkC0/"] + @test c == String["~/Zx6Wa0GkC0/"] end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0") @test res @@ -1384,11 +1399,11 @@ if !Sys.iswindows() end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0/my_") @test res - @test c == String["my_file\""] + @test c == String["~/Zx6Wa0GkC0/my_file\""] end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0/my_file") @test res - @test c == String[homedir() * "/Zx6Wa0GkC0/my_file"] + @test c == String[homedir() * "/Zx6Wa0GkC0/my_file\""] end finally rm(path, recursive=true) @@ -1414,8 +1429,8 @@ if Sys.iswindows() s = "cd ../" c,r = test_scomplete(s) - @test r == lastindex(s)+1:lastindex(s) - @test "$temp_name/" in c + @test r == 4:6 + @test "../$temp_name/" in c s = "ls $(file[1:2])" c,r = test_scomplete(s) @@ -1425,12 +1440,12 @@ if Sys.iswindows() s = "cd(\"..\\\\" c,r = test_complete(s) @test r == lastindex(s)-3:lastindex(s) - @test "../$temp_name/" in c + @test "..\\\\$temp_name\\\\" in c s = "cd(\"../" c,r = test_complete(s) - @test r == lastindex(s)+1:lastindex(s) - @test "$temp_name/" in c + @test r == 5:7 + @test "..\\\\$temp_name\\\\" in c s = "cd(\"$(file[1:2])" c,r = test_complete(s) @@ -1485,10 +1500,10 @@ function test_dict_completion(dict_name) s = "$dict_name[ \"abcd" # leading whitespace c, r = test_complete(s) @test c == Any["\"abcd\"]"] - s = "$dict_name[\"abcd]" # trailing close bracket + s = "$dict_name[Bas]" # trailing close bracket c, r = completions(s, lastindex(s) - 1) c = map(x -> named_completion(x).completion, c) - @test c == Any["\"abcd\""] + @test c == Any["Base"] s = "$dict_name[:b" c, r = test_complete(s) @test c == Any[":bar", ":bar2"] @@ -1542,9 +1557,13 @@ test_dict_completion("test_repl_comp_customdict") @testset "dict_identifier_key" begin # Issue #23004: this should not throw: - @test REPLCompletions.dict_identifier_key("test_dict_ℂ[\\", :other) isa Tuple + let s = "test_dict_ℂ[\\" + @test REPLCompletions.completions(s, sizeof(s), Main.CompletionFoo) isa Tuple + end # Issue #55931: neither should this: - @test REPLCompletions.dict_identifier_key("test_dict_no_length[", :other) isa NTuple{3,Nothing} + let s = "test_dict_no_length[" + @test REPLCompletions.completions(s, sizeof(s), Main.CompletionFoo) isa Tuple + end end @testset "completion of string/cmd macros (#22577)" begin @@ -2485,8 +2504,139 @@ let (c, r, res) = test_complete_context("global xxx::Number = Base.", Main) @test "pi" ∈ c end +# #55842 +let (c, r) = test_complete_pos("@tim| using Date") + @test "@time" in c + @test r == 1:4 +end + +# #56389 +let s = "begin\n using Ran" + c, r = test_complete(s) + @test "Random" in c + @test r == 15:17 + @test s[r] == "Ran" +end +let s = "using .CompletionFoo: bar, type_" + c, r = test_complete(s) + @test "type_test" in c + @test r == 28:32 + @test s[r] == "type_" +end + +# #55518 +let s = "CompletionFoo.@barfoo nothi" + c, r = test_complete(s) + @test "nothing" in c + @test r == 23:27 +end +let s = "CompletionFoo.@barfoo kwtest" + c, r = test_complete(s) + @test isempty(c) +end +let s = "CompletionFoo.kwtest(x=type" + c, r = test_complete(s) + @test "typeof" in c + @test !("type_test" in c) + @test r == 24:27 +end +let s = "CompletionFoo.bar; nothi" + c, r = test_complete(s) + @test "nothing" in c + @test r == 20:24 +end +let s = "CompletionFoo.bar; @ti" + c, r = test_complete(s) + @test "@time" in c + @test r == 20:22 +end +let s = "x = sin.([1]); y = ex" + c, r = test_complete(s) + @test "exp" in c + @test r == 20:21 +end + +# #57611 +let s = "x = Base.BinaryPlatforms.ar" + c, r = test_complete(s) + @test "arch" in c + @test r == 26:27 +end + +# #55520 +let s = "@ignoremacro A .= A setup=(A=ident" + c, r = test_complete(s) + @test "identity"in c + @test r == 30:34 +end + +# #57307 +let s = "unicode_αβγ.yy = named.len" + c, r = test_complete_foo(s) + @test "len2" in c + @test r == 27:29 +end + +# #55429 +let s = "@time @eval CompletionFoo.Compl" + c, r = test_complete(s) + @test "CompletionFoo2" in c + @test r == 27:31 +end + +# #55420 +let s = "CompletionFoo.test(iden" + c, r = test_complete(s) + @test "identity" in c + @test r == 20:23 +end + +# #57772 +let s = "sum(!ismis" + c, r = test_complete(s) + @test "ismissing" in c + @test r == 6:10 +end +let s = "sum(!!ismis" + c, r = test_complete(s) + @test "ismissing" in c + @test r == 7:11 +end + +# Don't trigger complete_methods! when the cursor is on the function name. +let s = "prin|(\"hello\")" + c, r = test_complete_pos(s) + @test "print" in c + @test r == 1:4 +end + +# Don't crash when tab-completing paths that cause ispath() to throw +let s = "include(\"" * repeat("a", 5000) # ENAMETOOLONG + c, r = test_complete(s) + @test isempty(c) +end + # JuliaLang/julia#57780 const issue57780 = ["a", "b", "c"] const issue57780_orig = copy(issue57780) test_complete_context("empty!(issue57780).", Main) @test issue57780 == issue57780_orig + +# Completion inside string interpolation +let s = "\"example: \$varflo" + c, r = test_complete_foo(s) + @test "varfloat" in c + @test r == 12:17 +end + +let s = "\"example: \$(3 + findfir" + c, r = test_complete(s) + @test "findfirst" in c + @test r == 17:23 +end + +let s = "\"example: \$(named.len" + c, r = test_complete_foo(s) + @test "len2" in c + @test r == 19:21 +end From d7163dafee7bbca5b1d78c54932924a8aad3a330 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 16 Apr 2025 10:45:04 -0700 Subject: [PATCH 117/662] Create single entry point for lowering `jl_lower` and redirect all calls to lowering through it. `jl_fl_lower` is essentially a renamed `jl_expand_with_loc_warn`, but change the structure of the return value of its callee `jl-expand-to-thunk-warn` so we can ignore the warnings if they aren't relevant. --- src/ast.c | 97 ++++++++++++++++++--------------------- src/jl_exported_funcs.inc | 2 +- src/jlfrontend.scm | 14 +++--- src/julia.h | 5 +- src/toplevel.c | 8 ++-- 5 files changed, 59 insertions(+), 67 deletions(-) diff --git a/src/ast.c b/src/ast.c index ab6b04efa526a..8c65b832e9f62 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1278,95 +1278,88 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule // Lower an expression tree into Julia's intermediate-representation. JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule) { - return jl_expand_with_loc(expr, inmodule, "none", 0); + return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 0); } // Lowering, with starting program location specified JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line) { - return jl_expand_in_world(expr, inmodule, file, line, ~(size_t)0); + return jl_lower(expr, inmodule, file, line, ~(size_t)0, 0, 0); } // Lowering, with starting program location and worldage specified JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world) { - JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); - JL_GC_PUSH1(&expr); - expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1); - expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line); - JL_GC_POP(); - return expr; + return jl_lower(expr, inmodule, file, line, world, 0, 0); } -// Same as the above, but printing warnings when applicable -JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) +// Main entry point to flisp lowering +// warn: Print any lowering warnings returned; otherwise ignore +// stmt: Lower knowing that the value of expr is unused +JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *file, int line, size_t world, bool_t warn, bool_t stmt) { JL_TIMING(LOWERING, LOWERING); jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_array_t *kwargs = NULL; - JL_GC_PUSH2(&expr, &kwargs); + JL_GC_PUSH3(&expr, &kwargs, &inmodule); expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); + expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1); jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); fl_context_t *fl_ctx = &ctx->fl; value_t arg = julia_to_scm(fl_ctx, expr); - value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg, - symbol(fl_ctx, file), fixnum(line), fl_ctx->F); - expr = scm_to_julia(fl_ctx, e, inmodule); + value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, + symbol(fl_ctx, file), fixnum(line), stmt ? fl_ctx->T : fl_ctx->F); + value_t lwr = car_(e); + value_t warnings = car_(cdr_(e)); + expr = scm_to_julia(fl_ctx, lwr, inmodule); jl_ast_ctx_leave(ctx); jl_sym_t *warn_sym = jl_symbol("warn"); - if (jl_is_expr(expr) && ((jl_expr_t*)expr)->head == warn_sym) { - size_t nargs = jl_expr_nargs(expr); - for (int i = 0; i < nargs - 1; i++) { - jl_value_t *warning = jl_exprarg(expr, i); - size_t nargs = 0; - if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) - nargs = jl_expr_nargs(warning); - int kwargs_len = (int)nargs - 6; - if (nargs < 6 || kwargs_len % 2 != 0) { - jl_error("julia-logmsg: bad argument list - expected " - ":warn level (symbol) group (symbol) id file line msg . kwargs"); - } - jl_value_t *level = jl_exprarg(warning, 0); - jl_value_t *group = jl_exprarg(warning, 1); - jl_value_t *id = jl_exprarg(warning, 2); - jl_value_t *file = jl_exprarg(warning, 3); - jl_value_t *line = jl_exprarg(warning, 4); - jl_value_t *msg = jl_exprarg(warning, 5); - kwargs = jl_alloc_vec_any(kwargs_len); - for (int i = 0; i < kwargs_len; ++i) { - jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); - } - JL_TYPECHK(logmsg, long, level); - jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + for (; warn && iscons(warnings); warnings = cdr_(warnings)) { + jl_value_t *warning = scm_to_julia(fl_ctx, car_(warnings), inmodule); + size_t nargs = 0; + if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) + nargs = jl_expr_nargs(warning); + int kwargs_len = (int)nargs - 6; + if (nargs < 6 || kwargs_len % 2 != 0) { + jl_error("julia-logmsg: bad argument list - expected " + ":warn level (symbol) group (symbol) id file line msg . kwargs"); + } + jl_value_t *level = jl_exprarg(warning, 0); + jl_value_t *group = jl_exprarg(warning, 1); + jl_value_t *id = jl_exprarg(warning, 2); + jl_value_t *file = jl_exprarg(warning, 3); + jl_value_t *line = jl_exprarg(warning, 4); + jl_value_t *msg = jl_exprarg(warning, 5); + kwargs = jl_alloc_vec_any(kwargs_len); + for (int i = 0; i < kwargs_len; ++i) { + jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); } - expr = jl_exprarg(expr, nargs - 1); + JL_TYPECHK(logmsg, long, level); + jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); } JL_GC_POP(); return expr; } -// expand in a context where the expression value is unused +JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *file, int line, size_t world, bool_t warn, bool_t stmt) +{ + // TODO: Allow change of lowerer + return jl_fl_lower(expr, inmodule, file, line, world, warn, stmt); +} + JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line) { - JL_TIMING(LOWERING, LOWERING); - JL_GC_PUSH1(&expr); - expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); - expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line); - JL_GC_POP(); - return expr; + return jl_lower(expr, inmodule, file, line, ~(size_t)0, 0, 1); } JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule) { - return jl_expand_stmt_with_loc(expr, inmodule, "none", 0); + return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 1); } jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 883a8d74df1bb..c5378b1aa5403 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -129,7 +129,6 @@ XX(jl_expand_stmt) \ XX(jl_expand_stmt_with_loc) \ XX(jl_expand_with_loc) \ - XX(jl_expand_with_loc_warn) \ XX(jl_field_index) \ XX(jl_field_isdefined) \ XX(jl_gc_add_finalizer) \ @@ -289,6 +288,7 @@ XX(jl_load_dynamic_library) \ XX(jl_load_file_string) \ XX(jl_lookup_code_address) \ + XX(jl_lower) \ XX(jl_lseek) \ XX(jl_lstat) \ XX(jl_macroexpand) \ diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index c313a1e9b0db5..12e1e6402646a 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -173,7 +173,11 @@ `(block ,expr (null))) file line)) -(define (jl-expand-to-thunk-warn expr file line stmt) +;; Returns a list `(,lowered-code ,warnings) where +;; - warnings (currently only ambiguous soft scope assignments) may be ignored, +;; e.g. when running interactively +;; - more items may be added to the list later +(define (jl-lower-to-thunk expr file line stmt) (let ((warnings '())) (with-bindings ;; Abuse scm_to_julia here to convert arguments to warn. This is meant for @@ -186,13 +190,7 @@ (let ((thunk (if stmt (expand-to-thunk-stmt- expr file line) (expand-to-thunk- expr file line)))) - (if (pair? warnings) `(warn ,@(reverse warnings) ,thunk) thunk))))) - -(define (jl-expand-to-thunk expr file line) - (expand-to-thunk- expr file line)) - -(define (jl-expand-to-thunk-stmt expr file line) - (expand-to-thunk-stmt- expr file line)) + `(,thunk ,(reverse warnings)))))) (define (jl-expand-macroscope expr) (error-wrap (lambda () diff --git a/src/julia.h b/src/julia.h index c8edf2d95afb8..a5e807ce9c859 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2260,8 +2260,9 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule); JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line); -JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); +JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *file, int line, size_t world, + bool_t warn, bool_t stmt); JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world); JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule); diff --git a/src/toplevel.c b/src/toplevel.c index f1fff694926ba..eca0cf20af14c 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -808,7 +808,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val size_t last_age = ct->world_age; if (!expanded && jl_needs_lowering(e)) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ex = (jl_expr_t*)jl_expand_with_loc_warn(e, m, *toplevel_filename, *toplevel_lineno); + ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1, 0); ct->world_age = last_age; } jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL; @@ -972,7 +972,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val root = jl_array_ptr_ref(ex->args, i); if (jl_needs_lowering(root)) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - root = jl_expand_with_loc_warn(root, m, *toplevel_filename, *toplevel_lineno); + root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1, 0); } ct->world_age = jl_atomic_load_acquire(&jl_world_counter); res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno); @@ -1151,8 +1151,8 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, continue; } ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); - expression = jl_expand_with_loc_warn(expression, module, - jl_string_data(filename), lineno); + expression = jl_lower(expression, module, + jl_string_data(filename), lineno, ~(size_t)0, 1, 0); ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } From 70f80e7828fa879b6b0b918b9aa395aeda8a1c76 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 16 Apr 2025 12:45:56 -0700 Subject: [PATCH 118/662] Remove some mostly-unused lowering functions `jl_expand_stmt`, `jl_expand_stmt_with_loc`, `jl_expand_with_loc` --- src/ast.c | 27 +-------------------------- src/jl_exported_funcs.inc | 3 --- src/julia.h | 7 ------- src/toplevel.c | 2 +- stdlib/REPL/src/REPL.jl | 2 +- test/meta.jl | 2 +- 6 files changed, 4 insertions(+), 39 deletions(-) diff --git a/src/ast.c b/src/ast.c index 8c65b832e9f62..ec13bd5eec9bc 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1281,21 +1281,7 @@ JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule) return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 0); } -// Lowering, with starting program location specified -JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) -{ - return jl_lower(expr, inmodule, file, line, ~(size_t)0, 0, 0); -} - -// Lowering, with starting program location and worldage specified -JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world) -{ - return jl_lower(expr, inmodule, file, line, world, 0, 0); -} - -// Main entry point to flisp lowering +// Main entry point to flisp lowering. Most arguments are optional; see `jl_expand`. // warn: Print any lowering warnings returned; otherwise ignore // stmt: Lower knowing that the value of expr is unused JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, @@ -1351,17 +1337,6 @@ JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, return jl_fl_lower(expr, inmodule, file, line, world, warn, stmt); } -JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) -{ - return jl_lower(expr, inmodule, file, line, ~(size_t)0, 0, 1); -} - -JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 1); -} - jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) { JL_TIMING(LOWERING, LOWERING); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index c5378b1aa5403..8ceb1b8add816 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -126,9 +126,6 @@ XX(jl_exit_on_sigint) \ XX(jl_exit_threaded_region) \ XX(jl_expand) \ - XX(jl_expand_stmt) \ - XX(jl_expand_stmt_with_loc) \ - XX(jl_expand_with_loc) \ XX(jl_field_index) \ XX(jl_field_isdefined) \ XX(jl_gc_add_finalizer) \ diff --git a/src/julia.h b/src/julia.h index a5e807ce9c859..11dc8a14b07a7 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2258,16 +2258,9 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, int offset, int greedy); // lowering JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule); -JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, bool_t warn, bool_t stmt); -JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world); -JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule); -JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, const char *filename, size_t filename_len); diff --git a/src/toplevel.c b/src/toplevel.c index eca0cf20af14c..2a938afa1c085 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -196,7 +196,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - form = jl_expand_stmt_with_loc(jl_array_ptr_ref(exprs, i), newm, filename, lineno); + form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0, 1); ct->world_age = jl_atomic_load_acquire(&jl_world_counter); (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno); } diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index cfb6e88a2663c..6cb8991677c27 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -298,7 +298,7 @@ const install_packages_hooks = Any[] # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro # code to run (and error). __repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_expand_with_loc, Any, (Any, Any, Ptr{UInt8}, Cint), ast, mod, toplevel_file[], toplevel_line[]) + ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0, 0) __repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line) diff --git a/test/meta.jl b/test/meta.jl index e9e344bba2e22..0191e23a49718 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar) @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar end -_lower(m::Module, ex, world::UInt) = ccall(:jl_expand_in_world, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t), ex, m, "none", 0, world) +_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint, Cint), ex, m, "none", 0, world, 0, 0) module TestExpandInWorldModule macro m() 1 end From 818311209c73aa7bf8e7669bb467d1219b6d9a9a Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 16 Apr 2025 12:58:47 -0700 Subject: [PATCH 119/662] Rename `jl_expand` to `jl_lower_expr_mod` This function has always been a convenience wrapper providing default arguments to another lowering function. We can give it a more accurate name now that there aren't five other wrappers we would also need to rename in a consistent way. --- base/expr.jl | 2 +- base/meta.jl | 2 +- doc/src/devdocs/eval.md | 2 +- src/ast.c | 14 +++++++------- src/jl_exported_funcs.inc | 2 +- src/julia.h | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/base/expr.jl b/base/expr.jl index 91b37af17c231..dd357736bc529 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1669,7 +1669,7 @@ end # Implementation of generated functions function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) - ci = ccall(:jl_expand, Any, (Any, Any), ex, defmod) + ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod) if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error msg = ci.args[1] diff --git a/base/meta.jl b/base/meta.jl index 4807b910c494a..32c78ce0639f8 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form for executing in module `m`. See also [`code_lowered`](@ref). """ -lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m) +lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m) """ @lower [m] x diff --git a/doc/src/devdocs/eval.md b/doc/src/devdocs/eval.md index 8f2fd68159676..e2c4adb741fcc 100644 --- a/doc/src/devdocs/eval.md +++ b/doc/src/devdocs/eval.md @@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what else - Julia) via `fl_invoke_julia_macro()`, and back. -Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_expand()`, +Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`, although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`. ## [Type Inference](@id dev-type-inference) diff --git a/src/ast.c b/src/ast.c index ec13bd5eec9bc..aeea7933dd24f 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1275,13 +1275,7 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule return expr; } -// Lower an expression tree into Julia's intermediate-representation. -JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 0); -} - -// Main entry point to flisp lowering. Most arguments are optional; see `jl_expand`. +// Main entry point to flisp lowering. Most arguments are optional; see `jl_lower_expr_mod`. // warn: Print any lowering warnings returned; otherwise ignore // stmt: Lower knowing that the value of expr is unused JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, @@ -1330,6 +1324,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, return expr; } +// Lower an expression tree into Julia's intermediate-representation. JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, bool_t warn, bool_t stmt) { @@ -1337,6 +1332,11 @@ JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, return jl_fl_lower(expr, inmodule, file, line, world, warn, stmt); } +JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule) +{ + return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 0); +} + jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) { JL_TIMING(LOWERING, LOWERING); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 8ceb1b8add816..6cb9ab630a4b1 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -125,7 +125,6 @@ XX(jl_exit) \ XX(jl_exit_on_sigint) \ XX(jl_exit_threaded_region) \ - XX(jl_expand) \ XX(jl_field_index) \ XX(jl_field_isdefined) \ XX(jl_gc_add_finalizer) \ @@ -286,6 +285,7 @@ XX(jl_load_file_string) \ XX(jl_lookup_code_address) \ XX(jl_lower) \ + XX(jl_lower_expr_mod) \ XX(jl_lseek) \ XX(jl_lstat) \ XX(jl_macroexpand) \ diff --git a/src/julia.h b/src/julia.h index 11dc8a14b07a7..afa9a04f777c4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2257,10 +2257,10 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len, JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, int offset, int greedy); // lowering -JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule); JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, bool_t warn, bool_t stmt); +JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, const char *filename, size_t filename_len); From a042b68a4eb5cf9cbf1eb382bc38472acaea64a8 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 16 Apr 2025 13:17:42 -0700 Subject: [PATCH 120/662] Rename top-level lisp `expand` functions to `lower` This seems to be more accurate. --- src/jlfrontend.scm | 36 ++++++++++++++++++------------------ src/julia-syntax.scm | 10 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index 12e1e6402646a..d2e41aaf9370a 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -109,7 +109,7 @@ ;; return a lambda expression representing a thunk for a top-level expression ;; note: expansion of stuff inside module is delayed, so the contents obey ;; toplevel expansion order (don't expand until stuff before is evaluated). -(define (expand-toplevel-expr-- e file line) +(define (lower-toplevel-expr-- e file line) (let ((lno (first-lineno e)) (ex0 (julia-expand-macroscope e))) (if (and lno (or (not (length= lno 3)) (not (atom? (caddr lno))))) (set! lno #f)) @@ -118,8 +118,8 @@ ex0 (if lno `(toplevel ,lno ,ex0) ex0)) (let* ((linenode (if (and lno (or (= line 0) (eq? file 'none))) lno `(line ,line ,file))) - (ex (julia-expand0 ex0 linenode)) - (th (julia-expand1 + (ex (julia-lower0 ex0 linenode)) + (th (julia-lower1 `(lambda () () (scope-block ,(blockify ex lno))) @@ -142,20 +142,20 @@ error incomplete)) (and (memq (car e) '(global const)) (every symbol? (cdr e)))))) -(define *in-expand* #f) +(define *in-lowering* #f) -(define (expand-toplevel-expr e file line) +(define (lower-toplevel-expr e file line) (cond ((or (atom? e) (toplevel-only-expr? e)) (if (underscore-symbol? e) (error "all-underscore identifiers are write-only and their values cannot be used in expressions")) e) (else - (let ((last *in-expand*)) + (let ((last *in-lowering*)) (if (not last) (begin (reset-gensyms) - (set! *in-expand* #t))) - (begin0 (expand-toplevel-expr-- e file line) - (set! *in-expand* last)))))) + (set! *in-lowering* #t))) + (begin0 (lower-toplevel-expr-- e file line) + (set! *in-lowering* last)))))) ;; used to collect warnings during lowering, which are usually discarded ;; unless logging is requested @@ -163,12 +163,12 @@ ;; expand a piece of raw surface syntax to an executable thunk -(define (expand-to-thunk- expr file line) +(define (lower-to-thunk- expr file line) (error-wrap (lambda () - (expand-toplevel-expr expr file line)))) + (lower-toplevel-expr expr file line)))) -(define (expand-to-thunk-stmt- expr file line) - (expand-to-thunk- (if (toplevel-only-expr? expr) +(define (lower-to-thunk-stmt- expr file line) + (lower-to-thunk- (if (toplevel-only-expr? expr) expr `(block ,expr (null))) file line)) @@ -188,8 +188,8 @@ (file (if (eq? warn_file 'none) file warn_file))) (set! warnings (cons (list* 'warn level group (symbol (string file line)) file line lst) warnings)))))) (let ((thunk (if stmt - (expand-to-thunk-stmt- expr file line) - (expand-to-thunk- expr file line)))) + (lower-to-thunk-stmt- expr file line) + (lower-to-thunk- expr file line)))) `(,thunk ,(reverse warnings)))))) (define (jl-expand-macroscope expr) @@ -197,14 +197,14 @@ (julia-expand-macroscope expr)))) (define (jl-default-inner-ctor-body field-kinds file line) - (expand-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line)) + (lower-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line)) (define (jl-default-outer-ctor-body args file line) - (expand-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line)) + (lower-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line)) ; run whole frontend on a string. useful for testing. (define (fe str) - (expand-toplevel-expr (julia-parse str) 'none 0)) + (lower-toplevel-expr (julia-parse str) 'none 0)) (define (profile-e s) (with-exception-catcher diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index ae5a4148da88d..4fa6dc2e14a70 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -5331,7 +5331,7 @@ f(x) = yt(x) ;; expander entry point -(define (julia-expand1 ex file line) +(define (julia-lower1 ex file line) (compact-and-renumber (linearize (closure-convert @@ -5340,7 +5340,7 @@ f(x) = yt(x) (define *current-desugar-loc* #f) -(define (julia-expand0 ex lno) +(define (julia-lower0 ex lno) (with-bindings ((*current-desugar-loc* lno)) (trycatch (expand-forms ex) (lambda (e) @@ -5353,7 +5353,7 @@ f(x) = yt(x) (error (string (cadr e) (format-loc *current-desugar-loc*)))) (raise e))))) -(define (julia-expand ex (file 'none) (line 0)) - (julia-expand1 - (julia-expand0 +(define (julia-lower ex (file 'none) (line 0)) + (julia-lower1 + (julia-lower0 (julia-expand-macroscope ex) `(line ,line ,file)) file line)) From 8d82bec7a7ead1b6dc69b8dc1d1f63b48af50171 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Thu, 17 Apr 2025 08:38:28 -0700 Subject: [PATCH 121/662] Try removing `stmt` lowering option --- src/ast.c | 13 ++++++------- src/jlfrontend.scm | 14 +++----------- src/julia.h | 2 +- src/toplevel.c | 9 ++++----- stdlib/REPL/src/REPL.jl | 2 +- test/meta.jl | 2 +- 6 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/ast.c b/src/ast.c index aeea7933dd24f..b7edaac43f134 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1277,9 +1277,8 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule // Main entry point to flisp lowering. Most arguments are optional; see `jl_lower_expr_mod`. // warn: Print any lowering warnings returned; otherwise ignore -// stmt: Lower knowing that the value of expr is unused JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn, bool_t stmt) + const char *file, int line, size_t world, bool_t warn) { JL_TIMING(LOWERING, LOWERING); jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); @@ -1290,8 +1289,8 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); fl_context_t *fl_ctx = &ctx->fl; value_t arg = julia_to_scm(fl_ctx, expr); - value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, - symbol(fl_ctx, file), fixnum(line), stmt ? fl_ctx->T : fl_ctx->F); + value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, + symbol(fl_ctx, file), fixnum(line)); value_t lwr = car_(e); value_t warnings = car_(cdr_(e)); expr = scm_to_julia(fl_ctx, lwr, inmodule); @@ -1326,15 +1325,15 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, // Lower an expression tree into Julia's intermediate-representation. JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn, bool_t stmt) + const char *file, int line, size_t world, bool_t warn) { // TODO: Allow change of lowerer - return jl_fl_lower(expr, inmodule, file, line, world, warn, stmt); + return jl_fl_lower(expr, inmodule, file, line, world, warn); } JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule) { - return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0, 0); + return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0); } jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index d2e41aaf9370a..a678d481eab1f 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -167,17 +167,11 @@ (error-wrap (lambda () (lower-toplevel-expr expr file line)))) -(define (lower-to-thunk-stmt- expr file line) - (lower-to-thunk- (if (toplevel-only-expr? expr) - expr - `(block ,expr (null))) - file line)) - ;; Returns a list `(,lowered-code ,warnings) where ;; - warnings (currently only ambiguous soft scope assignments) may be ignored, ;; e.g. when running interactively ;; - more items may be added to the list later -(define (jl-lower-to-thunk expr file line stmt) +(define (jl-lower-to-thunk expr file line) (let ((warnings '())) (with-bindings ;; Abuse scm_to_julia here to convert arguments to warn. This is meant for @@ -187,10 +181,8 @@ (let ((line (if (= warn_line 0) line warn_line)) (file (if (eq? warn_file 'none) file warn_file))) (set! warnings (cons (list* 'warn level group (symbol (string file line)) file line lst) warnings)))))) - (let ((thunk (if stmt - (lower-to-thunk-stmt- expr file line) - (lower-to-thunk- expr file line)))) - `(,thunk ,(reverse warnings)))))) + `(,(lower-to-thunk- expr file line) + ,(reverse warnings))))) (define (jl-expand-macroscope expr) (error-wrap (lambda () diff --git a/src/julia.h b/src/julia.h index afa9a04f777c4..195c23f20662f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2259,7 +2259,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, // lowering JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, - bool_t warn, bool_t stmt); + bool_t warn); JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, diff --git a/src/toplevel.c b/src/toplevel.c index 2a938afa1c085..868c4e28a2849 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -196,7 +196,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0, 1); + form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0); ct->world_age = jl_atomic_load_acquire(&jl_world_counter); (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno); } @@ -808,7 +808,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val size_t last_age = ct->world_age; if (!expanded && jl_needs_lowering(e)) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1, 0); + ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); ct->world_age = last_age; } jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL; @@ -972,7 +972,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val root = jl_array_ptr_ref(ex->args, i); if (jl_needs_lowering(root)) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1, 0); + root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); } ct->world_age = jl_atomic_load_acquire(&jl_world_counter); res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno); @@ -1151,8 +1151,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, continue; } ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); - expression = jl_lower(expression, module, - jl_string_data(filename), lineno, ~(size_t)0, 1, 0); + expression = jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1); ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 6cb8991677c27..d349e75d128d4 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -298,7 +298,7 @@ const install_packages_hooks = Any[] # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro # code to run (and error). __repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0, 0) + ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0) __repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line) diff --git a/test/meta.jl b/test/meta.jl index 0191e23a49718..f8707bc36791c 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar) @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar end -_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint, Cint), ex, m, "none", 0, world, 0, 0) +_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint), ex, m, "none", 0, world, 0) module TestExpandInWorldModule macro m() 1 end From 6d78a4ae91078cea43470f8cd6d895e0c9b3d5e7 Mon Sep 17 00:00:00 2001 From: Timothy Date: Wed, 23 Apr 2025 01:25:55 +0800 Subject: [PATCH 122/662] Workaround StyledStrings type-piracy (#58112) After discussion with Cody, this is an attempt at a more minimal version of #56194. The intent is to work around the invalidation issue introduced by the split design with AnnotatedStrings, resolving the headache for 1.12. Paired with https://github.com/JuliaLang/StyledStrings.jl/pull/115 Many thanks to @topolarity for working out this approach, the discussion on balancing the short/long term fixes for this issue. ------ From my understanding of the problem, this should fix the `write`/`print`/`show` invalidations, but this needs to be checked. --------- Co-authored-by: Cody Tapscott --- base/strings/annotated_io.jl | 75 +++++++++++++++++++ .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/StyledStrings.version | 2 +- 6 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 87db57b8030c9..9698fd5909b68 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -199,3 +199,78 @@ function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionA push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) end end + +# NOTE: This is an interim solution to the invalidations caused +# by the split styled display implementation. This should be +# replaced by a more robust solution (such as a consolidation of +# the type and method definitions) in the near future. +module AnnotatedDisplay + +using ..Base: IO, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer +using ..Base: eachregion, invoke_in_world, tls_world_age + +# Write + +ansi_write(f::Function, io::IO, x::Any) = f(io, String(x)) + +ansi_write_(f::Function, io::IO, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), ansi_write, f, io, x) + +Base.write(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + ansi_write_(write, io, s)::Int + +Base.write(io::IO, c::AnnotatedChar) = + ansi_write_(write, io, c)::Int + +function Base.write(io::IO, aio::AnnotatedIOBuffer) + if get(io, :color, false) == true + # This does introduce an overhead that technically + # could be avoided, but I'm not sure that it's currently + # worth the effort to implement an efficient version of + # writing from a AnnotatedIOBuffer with style. + # In the meantime, by converting to an `AnnotatedString` we can just + # reuse all the work done to make that work. + ansi_write_(write, io, read(aio, AnnotatedString))::Int + else + write(io, aio.io) + end +end + +# Print + +Base.print(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + (ansi_write_(write, io, s); nothing) + +Base.print(io::IO, s::AnnotatedChar) = + (ansi_write_(write, io, s); nothing) + +Base.print(io::AnnotatedIOBuffer, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + (write(io, s); nothing) + +Base.print(io::AnnotatedIOBuffer, c::AnnotatedChar) = + (write(io, c); nothing) + +# Escape + +Base.escape_string(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}, + esc = ""; keep = (), ascii::Bool=false, fullhex::Bool=false) = + (ansi_write_((io, s) -> escape_string(io, s, esc; keep, ascii, fullhex), io, s); nothing) + +# Show + +show_annot(io::IO, ::Any) = nothing +show_annot(io::IO, ::MIME, ::Any) = nothing + +show_annot_(io::IO, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), show_annot, io, x)::Nothing + +show_annot_(io::IO, m::MIME, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), show_annot, io, m, x)::Nothing + +Base.show(io::IO, m::MIME"text/html", s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + show_annot_(io, m, s) + +Base.show(io::IO, m::MIME"text/html", c::AnnotatedChar) = + show_annot_(io, m, c) + +end diff --git a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 new file mode 100644 index 0000000000000..46d5cacf788df --- /dev/null +++ b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 @@ -0,0 +1 @@ +1cb6007a66d3f74cbe5b27ee449aa9c8 diff --git a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 new file mode 100644 index 0000000000000..724b2d311c123 --- /dev/null +++ b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 @@ -0,0 +1 @@ +1fa95646fdf4cc7ea282bd355fded9464e7572792912942ea1c45f6ed126eead2333fdeed92e7db3efbcd6c3a171a04e5c9562dab2685bb39947136284ae1da3 diff --git a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 b/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 deleted file mode 100644 index 0fd8e8966e068..0000000000000 --- a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -411277f3701cc3e286ec8a84ccdf6f11 diff --git a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 b/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 deleted file mode 100644 index 0b495aefef55d..0000000000000 --- a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -95a7e92389f6fd02d3bec17ec0201ba41316aa2d7c321b14af88ccce8246fd0000ed2c0cc818f87cb81f7134304233db897f656426a00caac1bc7635056260c2 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index c72f7a8399725..55a4a08c17ea0 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = main -STYLEDSTRINGS_SHA1 = 8985a37ac054c37d084a03ad2837208244824877 +STYLEDSTRINGS_SHA1 = 3fe829fcf611b5fefaefb64df7e61f2ae82db117 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 From 16eca6e2658de5cffa8d98583ad325d52566dbab Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Wed, 23 Apr 2025 02:42:12 -0400 Subject: [PATCH 123/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?LinearAlgebra=20stdlib=20from=201ce8426=20to=2007725da=20(#5818?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: LinearAlgebra URL: https://github.com/JuliaLang/LinearAlgebra.jl.git Stdlib branch: master Julia branch: master Old commit: 1ce8426 New commit: 07725da Julia version: 1.13.0-DEV LinearAlgebra version: 1.12.0(Does not match) Bump invoked by: @jishnub Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/LinearAlgebra.jl/compare/1ce842652c07b33289046236b09bc59bad43dbeb...07725da80be389e170cff75cf7157399c2449643 ``` $ git log --oneline 1ce8426..07725da 07725da Branch on `Bool` `alpha` in scaling `mul!` (#1286) 61e444d Fix exponentiation with immutable matrix (#1289) 77475c1 Fix `lmul!`/`rmul!` for 0-sized matrices (#1290) c3d35c0 Test for `versioninfo` with `ENV` variable (#1279) 830ea2f Fit broken matmul test (#1288) 222f7f2 Propagate inbounds in diagzero (#1285) e65e75c Clean up `herk_wrapper!` and add 5-arg tests (#1254) ece1962 `versioninfo`: simplify, improve type stability (#1274) 3e525a8 Speicalize `copy!` for triangular, and use `copy!` in `ldiv` (#1263) 763f19f Forward scaling `lmul!`/`rmul!` for `Tridiagonal` to bands (#1284) 2d27d1c Explicit loop in converting `Bidiagonal`/`Tridiagonal` to `Matrix` (#1283) 0671a7b Simplify small Bidiagonal-AbstractVecOrMat multiplication (#1278) a8fd121 Test for empty `Symmetric` and `BlasFlag.NONE` (#1280) 6e2de14 Remove bounds-checking in `Bidiagonal` `rdiv!` (#1281) 4a8fc62 Test for 5-arg `mul!` c437beb Test for empty `Symmetric` and `BlasFlag.NONE` 7a891e9 Test for `versioninfo` with `ENV` variable 84fd21b Test for `versioninfo` (#1276) 243efdb `AbstractQ` bugfix: define `Base.IteratorSize` (#1277) 6073f28 Ease inference in `Bidiagonal` divmul tests (#1273) 30cb5d6 Avoid copy in empty bidiag `ldiv!` (#1275) 5d3ef46 Skip symmetry check in converting Symmetric to SymTridiagonal (#1269) ae5385b Specialize `isreal` for banded matrices (#1271) 0a253be Explicit loop in `_iszero` for strided views (#1264) 3f46f5f `istril`/`istriu` for opposite triangularity (#1265) cd048eb Tests for `mul!` scaling with mismatched axes (#1266) 26db4c8 Restrict tests to `Number` eltypes 8b21ca6 Tests for `mul!` scaling with mismatched axes 9675c38 Fix minimum band for `UpperTriangular`/`Lowertriangular` aeeed7d Add unit triangular tests 3e1aaac Add banded tests 7fdfb3b Add tests d866014 istril/istriu for opposite triangularity 925acef Return view in `_diag` for `Bidiagonal` (#1258) b1bcca1 Branch on `Bool` `alpha` in diagonal matmul (#1256) e3e9987 `peakflops`: make `eltype` a static parameter of the method (#1253) 781eb5d `peakflops`: make the element type a static parameter of the method e53b50c use smaller matrix size in `peakflops` on 32-bit (#1255) 0c87d0d use smaller matrix size in `peakflops` on 32-bit 0cbf85f Check approx equality in bunchkaufman docstring (#1251) 5e53d12 Avoid accessing unset indices in symmetric copyto! (#1244) e64a3df fix dispatch to herk (#1247) bfdb742 Destination zeros from eltype in zero-length generic_matvecmul (#1241) 16dedb5 Remove redundant adjoint/transpose methods for Symmetric/Hermitian (#1245) 7f1dfe4 stop assuming that you can mutate data structures in other packages at precompile time (#1252) ``` Co-authored-by: jishnub <10461665+jishnub@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/LinearAlgebra.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 create mode 100644 deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 delete mode 100644 deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 delete mode 100644 deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 diff --git a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 new file mode 100644 index 0000000000000..13eaae101948e --- /dev/null +++ b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 @@ -0,0 +1 @@ +3a0109056e5135fc5f9c9ddb2c43c531 diff --git a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 new file mode 100644 index 0000000000000..f1ae59d2204c3 --- /dev/null +++ b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 @@ -0,0 +1 @@ +9543ff47af4fe439e29119e73759ef06a0bbdd263a39cd323243c6f05ee44eadf015c335aab60b43d0f83327714bb71d316b2f4ebfaee205fa5368f2ab856a78 diff --git a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 b/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 deleted file mode 100644 index cf11a21c45995..0000000000000 --- a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -33933c31906af2086702d38b9d8eb90b diff --git a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 b/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 deleted file mode 100644 index e11b0c3957407..0000000000000 --- a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -3783b8c3a46c4db5cf750a99c6753e1bf377b2d979a85df8e26c81f41cb3c75a41c28a487fce332e19ba8287c2531d440b248a6f5aedc93e8fa6673d60456c33 diff --git a/stdlib/LinearAlgebra.version b/stdlib/LinearAlgebra.version index 57683cd38f060..243522ef799ae 100644 --- a/stdlib/LinearAlgebra.version +++ b/stdlib/LinearAlgebra.version @@ -1,4 +1,4 @@ LINEARALGEBRA_BRANCH = master -LINEARALGEBRA_SHA1 = 1ce842652c07b33289046236b09bc59bad43dbeb +LINEARALGEBRA_SHA1 = 07725da80be389e170cff75cf7157399c2449643 LINEARALGEBRA_GIT_URL := https://github.com/JuliaLang/LinearAlgebra.jl.git LINEARALGEBRA_TAR_URL = https://api.github.com/repos/JuliaLang/LinearAlgebra.jl/tarball/$1 From ec70a2c6108b637dafa9eba2821e6b1bbf83411e Mon Sep 17 00:00:00 2001 From: Omar Elrefaei Date: Wed, 23 Apr 2025 17:53:24 -0400 Subject: [PATCH 124/662] Fix typo in testset docs (#58204) --- stdlib/Test/src/Test.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index e3a52baf4fafa..da102e37e7b80 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1650,9 +1650,9 @@ trigonometric identities | 4 4 0.2s # `@testset for` -When `@testset for` is used, the macro starts a new test for each iteration of +When `@testset for` is used, the macro starts a new test set for each iteration of the provided loop. The semantics of each test set are otherwise identical to that -of that `begin/end` case (as if used for each loop iteration). +of the `begin/end` case (as if used for each loop iteration). # `@testset let` From 6498664d66c419c832f9bf2799a9a88d86420cae Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Thu, 24 Apr 2025 15:09:29 +0200 Subject: [PATCH 125/662] faster `rand(1:n)` by outlining unlikely branch (#58089) It's hard to measure the improvement with single calls, but this change substantially improve the situation in #50509, such that these new versions of `randperm` etc are almost always faster (even for big n). Here are some example benchmarks. Note that biggest ranges like `UInt(0):UInt(2)^64-2` are the ones exercising the most the "unlikely" branch: ```julia julia> const xx = Xoshiro(); using Chairmarks julia> rands(rng, ns) = for i=ns rand(rng, zero(i):i) end julia> rands(ns) = for i=ns rand(zero(i):i) end julia> @b rand(xx, 1:100), rand(xx, UInt(0):UInt(2)^63), rand(xx, UInt(0):UInt(2)^64-3), rand(xx, UInt(0):UInt(2)^64-2), rand(xx, UInt(0):UInt(2)^64-1) (1.968 ns, 8.000 ns, 3.321 ns, 3.321 ns, 2.152 ns) # PR (2.151 ns, 7.284 ns, 2.151 ns, 2.151 ns, 2.151 ns) # master julia> @b rand(1:100), rand(UInt(0):UInt(2)^63), rand(UInt(0):UInt(2)^64-3), rand(UInt(0):UInt(2)^64-2),rand(UInt(0):UInt(2)^64-1) # with TaskLocalRNG (2.148 ns, 7.837 ns, 3.317 ns, 3.085 ns, 1.957 ns) # PR (3.128 ns, 8.275 ns, 3.324 ns, 3.324 ns, 1.955 ns) # master julia> rands(xx, 1:100), rands(xx, UInt(2)^62:UInt(2)^59:UInt(2)^64-1), rands(xx, UInt(2)^64-4:UInt(2)^64-2) (95.315 ns, 132.144 ns, 7.486 ns) # PR (217.169 ns, 143.519 ns, 8.065 ns) # master julia> rands(1:100), rands(UInt(2)^62:UInt(2)^59:UInt(2)^64-1), rands(UInt(2)^64-4:UInt(2)^64-2) (235.882 ns, 162.809 ns, 10.603 ns) # PR (202.524 ns, 132.869 ns, 7.631 ns) # master ``` So it's a bit tricky: with an explicit RNG, `rands(xx, 1:100)` becomes much faster, but without, `rands(1:100)` becomes slower. Assuming #50509 was merged, `shuffle` is a good function to benchmark `rand(1:n)`, and the changes here consistently improve performance, as shown by this graph (when `TaskLocalRNG` is mentioned, it means *no* RNG argument was passed to the function): ![new-rand-ndl](https://github.com/user-attachments/assets/b7a6229a-f5d9-408e-9102-4056b796d22c) So although there can be slowdowns, I think this change is overall a win. --- stdlib/Random/src/generation.jl | 22 +++++++++++++--------- test/testhelpers/coverage_file.info | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/stdlib/Random/src/generation.jl b/stdlib/Random/src/generation.jl index b605dff9e5d80..61d0dd74eaa5c 100644 --- a/stdlib/Random/src/generation.jl +++ b/stdlib/Random/src/generation.jl @@ -375,16 +375,20 @@ function rand(rng::AbstractRNG, sp::SamplerRangeNDL{U,T}) where {U,T} s = sp.s x = widen(rand(rng, U)) m = x * s - l = m % U - if l < s - t = mod(-s, s) # as s is unsigned, -s is equal to 2^L - s in the paper - while l < t - x = widen(rand(rng, U)) - m = x * s - l = m % U - end + r::T = (m % U) < s ? rand_unlikely(rng, s, m) % T : + iszero(s) ? x % T : + (m >> (8*sizeof(U))) % T + r + sp.a +end + +# similar to `randn_unlikely` : splitting this unlikely path out results in faster code +@noinline function rand_unlikely(rng, s::U, m)::U where {U} + t = mod(-s, s) # as s is unsigned, -s is equal to 2^L - s in the paper + while (m % U) < t + x = widen(rand(rng, U)) + m = x * s end - (s == 0 ? x : m >> (8*sizeof(U))) % T + sp.a + (m >> (8*sizeof(U))) % U end diff --git a/test/testhelpers/coverage_file.info b/test/testhelpers/coverage_file.info index b03b0e07e6977..61410a72bc849 100644 --- a/test/testhelpers/coverage_file.info +++ b/test/testhelpers/coverage_file.info @@ -1,6 +1,6 @@ SF: DA:3,1 -DA:4,1 +DA:4,2 DA:5,0 DA:7,1 DA:8,1 From 907b201cedec1b4d00e27f27535cce7e192beb26 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 24 Apr 2025 10:25:48 -0400 Subject: [PATCH 126/662] reduce places where Builtins are listed (#58205) DRY code somewhat by consolidating builtin declarations to use a table or macro definition to auto-generate the required reflection metadata. Mostly NFC, but does include a couple bugfixes caused by the consistency this enforces. Adding a new `Builtin` is now simply a matter of declaring it in `builtin_proto.h` and defining it in `builtins.c` and any relevant declarations are handled automatically. --- src/builtin_proto.h | 152 +++++++------ src/builtins.c | 143 ++++-------- src/cgutils.cpp | 2 +- src/codegen.cpp | 312 +++++++++++--------------- src/common_symbols2.inc | 2 +- src/gf.c | 12 +- src/init.c | 2 - src/julia_internal.h | 10 +- src/llvm-codegen-shared.h | 3 - src/llvm-pass-helpers.cpp | 3 + src/method.c | 5 +- src/signals-mach.c | 2 - src/staticdata.c | 456 +++++++++++++++++--------------------- src/toplevel.c | 4 +- 14 files changed, 472 insertions(+), 636 deletions(-) diff --git a/src/builtin_proto.h b/src/builtin_proto.h index c82ec77414129..586d948f722c1 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -8,82 +8,88 @@ extern "C" { #endif // declarations for julia-callable builtin functions +#define JL_BUILTIN_FUNCTIONS(XX) \ + XX(_abstracttype,"_abstracttype") \ + XX(_apply_iterate,"_apply_iterate") \ + XX(_call_in_world_total,"_call_in_world_total") \ + XX(_compute_sparams,"_compute_sparams") \ + XX(_defaultctors,"_defaultctors") \ + XX(_equiv_typedef,"_equiv_typedef") \ + XX(_expr,"_expr") \ + XX(_primitivetype,"_primitivetype") \ + XX(_setsuper,"_setsuper!") \ + XX(_structtype,"_structtype") \ + XX(_svec_ref,"_svec_ref") \ + XX(_typebody,"_typebody!") \ + XX(_typevar,"_typevar") \ + XX(applicable,"applicable") \ + XX(apply_type,"apply_type") \ + XX(compilerbarrier,"compilerbarrier") \ + XX(current_scope,"current_scope") \ + XX(donotdelete,"donotdelete") \ + XX(fieldtype,"fieldtype") \ + XX(finalizer,"finalizer") \ + XX(get_binding_type,"get_binding_type") \ + XX(getfield,"getfield") \ + XX(getglobal,"getglobal") \ + XX(ifelse,"ifelse") \ + XX(intrinsic_call,"intrinsic_call") \ + XX(invoke,"invoke") \ + XX(invoke_in_world,"invoke_in_world") \ + XX(invokelatest,"invokelatest") \ + XX(is,"===") \ + XX(isa,"isa") \ + XX(isdefined,"isdefined") \ + XX(isdefinedglobal,"isdefinedglobal") \ + XX(issubtype,"<:") \ + XX(memorynew,"memorynew") \ + XX(memoryrefnew,"memoryrefnew") \ + XX(memoryref_isassigned,"memoryref_isassigned") \ + XX(memoryrefget,"memoryrefget") \ + XX(memoryrefmodify,"memoryrefmodify!") \ + XX(memoryrefoffset,"memoryrefoffset") \ + XX(memoryrefreplace,"memoryrefreplace!") \ + XX(memoryrefset,"memoryrefset!") \ + XX(memoryrefsetonce,"memoryrefsetonce!") \ + XX(memoryrefswap,"memoryrefswap!") \ + XX(modifyfield,"modifyfield!") \ + XX(modifyglobal,"modifyglobal!") \ + XX(nfields,"nfields") \ + XX(opaque_closure_call,"opaque_closure_call") \ + XX(replacefield,"replacefield!") \ + XX(replaceglobal,"replaceglobal!") \ + XX(setfield,"setfield!") \ + XX(setfieldonce,"setfieldonce!") \ + XX(setglobal,"setglobal!") \ + XX(setglobalonce,"setglobalonce!") \ + XX(sizeof,"sizeof") \ + XX(svec,"svec") \ + XX(swapfield,"swapfield!") \ + XX(swapglobal,"swapglobal!") \ + XX(throw,"throw") \ + XX(throw_methoderror,"throw_methoderror") \ + XX(tuple,"tuple") \ + XX(typeassert,"typeassert") \ + XX(typeof,"typeof") \ -#ifdef DEFINE_BUILTIN_GLOBALS -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT jl_fptr_args_t jl_f_##name##_addr = &jl_f_##name -#else -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT extern jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT extern jl_fptr_args_t jl_f_##name##_addr -#endif +#define DECLARE_BUILTIN(cname,jlname) \ + JL_CALLABLE(jl_f_##cname); +JL_BUILTIN_FUNCTIONS(DECLARE_BUILTIN) +#undef DECLARE_BUILTIN + +#define BUILTIN(cname) (jl_builtin_instances[jl_builtin_id_##cname]) + +enum jl_builtin_ids { +#define BUILTIN_IDS(cname,jlname) jl_builtin_id_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_IDS) +#undef BUILTIN_IDS + jl_n_builtins +}; -DECLARE_BUILTIN(_apply_iterate); -DECLARE_BUILTIN(invoke_in_world); -DECLARE_BUILTIN(_call_in_world_total); -DECLARE_BUILTIN(invokelatest); -DECLARE_BUILTIN(_compute_sparams); -DECLARE_BUILTIN(_expr); -DECLARE_BUILTIN(_svec_ref); -DECLARE_BUILTIN(_typebody); -DECLARE_BUILTIN(_typevar); -DECLARE_BUILTIN(applicable); -DECLARE_BUILTIN(apply_type); -DECLARE_BUILTIN(compilerbarrier); -DECLARE_BUILTIN(current_scope); -DECLARE_BUILTIN(donotdelete); -DECLARE_BUILTIN(fieldtype); -DECLARE_BUILTIN(finalizer); -DECLARE_BUILTIN(getfield); -DECLARE_BUILTIN(getglobal); -DECLARE_BUILTIN(ifelse); -DECLARE_BUILTIN(invoke); -DECLARE_BUILTIN(is); -DECLARE_BUILTIN(isa); -DECLARE_BUILTIN(isdefined); -DECLARE_BUILTIN(isdefinedglobal); -DECLARE_BUILTIN(issubtype); -DECLARE_BUILTIN(memorynew); -DECLARE_BUILTIN(memoryref); -DECLARE_BUILTIN(memoryref_isassigned); -DECLARE_BUILTIN(memoryrefget); -DECLARE_BUILTIN(memoryrefmodify); -DECLARE_BUILTIN(memoryrefoffset); -DECLARE_BUILTIN(memoryrefreplace); -DECLARE_BUILTIN(memoryrefset); -DECLARE_BUILTIN(memoryrefsetonce); -DECLARE_BUILTIN(memoryrefswap); -DECLARE_BUILTIN(modifyfield); -DECLARE_BUILTIN(modifyglobal); -DECLARE_BUILTIN(nfields); -DECLARE_BUILTIN(replacefield); -DECLARE_BUILTIN(replaceglobal); -DECLARE_BUILTIN(setfield); -DECLARE_BUILTIN(setfieldonce); -DECLARE_BUILTIN(setglobal); -DECLARE_BUILTIN(setglobalonce); -DECLARE_BUILTIN(sizeof); -DECLARE_BUILTIN(svec); -DECLARE_BUILTIN(swapfield); -DECLARE_BUILTIN(swapglobal); -DECLARE_BUILTIN(throw); -DECLARE_BUILTIN(throw_methoderror); -DECLARE_BUILTIN(tuple); -DECLARE_BUILTIN(typeassert); -DECLARE_BUILTIN(typeof); +JL_DLLEXPORT extern jl_fptr_args_t const jl_builtin_f_addrs[]; +JL_DLLEXPORT extern const char *const jl_builtin_f_names[]; +JL_DLLEXPORT extern jl_value_t *jl_builtin_instances[]; -JL_CALLABLE(jl_f__structtype); -JL_CALLABLE(jl_f__abstracttype); -JL_CALLABLE(jl_f__primitivetype); -JL_CALLABLE(jl_f__setsuper); -JL_CALLABLE(jl_f__defaultctors); -JL_CALLABLE(jl_f__equiv_typedef); -JL_CALLABLE(jl_f_get_binding_type); -JL_CALLABLE(jl_f__compute_sparams); -JL_CALLABLE(jl_f__svec_ref); #ifdef __cplusplus } #endif diff --git a/src/builtins.c b/src/builtins.c index a2cae857f26b4..e3a0380182e15 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -30,6 +30,27 @@ extern "C" { #endif +jl_fptr_args_t const jl_builtin_f_addrs[jl_n_builtins] = { +#define BUILTIN_ADDRS(cname,jlname) &jl_f_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_ADDRS) +#undef BUILTIN_ADDRS +}; + +const char *const jl_builtin_f_names[jl_n_builtins] = { +#define BUILTIN_F_NAMES(cname,jlname) XSTR(jl_f_##cname), +JL_BUILTIN_FUNCTIONS(BUILTIN_F_NAMES) +#undef BUILTIN_F_NAMES +}; + +jl_value_t *jl_builtin_instances[jl_n_builtins]; + +static const char *const jl_builtin_names[jl_n_builtins] = { +#define BUILTIN_NAMES(cname,jlname) jlname, +JL_BUILTIN_FUNCTIONS(BUILTIN_NAMES) +#undef BUILTIN_NAMES +}; + + // egal and object_id --------------------------------------------------------- static int bits_equal(const void *a, const void *b, int sz) JL_NOTSAFEPOINT @@ -647,7 +668,7 @@ JL_CALLABLE(jl_f__apply_iterate) nargs -= 1; if (nargs == 2) { // some common simple cases - if (f == jl_builtin_svec) { + if (f == BUILTIN(svec)) { if (jl_is_svec(args[1])) return args[1]; if (jl_is_genericmemory(args[1])) { @@ -672,7 +693,7 @@ JL_CALLABLE(jl_f__apply_iterate) return (jl_value_t*)t; } } - else if (f == jl_builtin_tuple && jl_is_tuple(args[1])) { + else if (f == BUILTIN(tuple) && jl_is_tuple(args[1])) { return args[1]; } } @@ -1691,11 +1712,11 @@ JL_CALLABLE(jl_f_memorynew) return (jl_value_t*)jl_alloc_genericmemory(args[0], nel); } -JL_CALLABLE(jl_f_memoryref) +JL_CALLABLE(jl_f_memoryrefnew) { - JL_NARGS(memoryref, 1, 3); + JL_NARGS(memoryrefnew, 1, 3); if (nargs == 1) { - JL_TYPECHK(memoryref, genericmemory, args[0]); + JL_TYPECHK(memoryrefnew, genericmemory, args[0]); jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(((jl_datatype_t*)jl_typetagof(m))->parameters), 3); JL_GC_PROMISE_ROOTED(typ); // it is a concrete type @@ -1705,10 +1726,10 @@ JL_CALLABLE(jl_f_memoryref) return (jl_value_t*)jl_new_memoryref(typ, m, m->ptr); } else { - JL_TYPECHK(memoryref, genericmemoryref, args[0]); - JL_TYPECHK(memoryref, long, args[1]); + JL_TYPECHK(memoryrefnew, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefnew, long, args[1]); if (nargs == 3) - JL_TYPECHK(memoryref, bool, args[2]); + JL_TYPECHK(memoryrefnew, bool, args[2]); jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)args[0]; size_t i = jl_unbox_long(args[1]) - 1; const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m->mem))->layout; @@ -1735,7 +1756,7 @@ JL_CALLABLE(jl_f_memoryref) JL_CALLABLE(jl_f_memoryrefoffset) { JL_NARGS(memoryrefoffset, 1, 1); - JL_TYPECHK(memoryref, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefoffest, genericmemoryref, args[0]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; size_t offset; @@ -2415,10 +2436,10 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED { jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1); jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0); - jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call); + jl_mk_builtin_func(jl_intrinsic_type, jl_symbol("IntrinsicFunction"), jl_f_intrinsic_call); jl_mk_builtin_func( (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type), - "OpaqueClosure", jl_f_opaque_closure_call); + jl_symbol("OpaqueClosure"), jl_f_opaque_closure_call); // Save a reference to the just created OpaqueClosure method, so we can provide special // codegen for it later. @@ -2439,93 +2460,21 @@ static void add_builtin(const char *name, jl_value_t *v) jl_set_initial_const(jl_core_module, jl_symbol(name), v, 0); } -jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt) -{ - assert(jl_subtype((jl_value_t*)dt, (jl_value_t*)jl_builtin_type)); - jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_atomic_load_relaxed(&dt->name->mt->defs); - jl_method_instance_t *mi = jl_atomic_load_relaxed(&entry->func.method->unspecialized); - jl_code_instance_t *ci = jl_atomic_load_relaxed(&mi->cache); - assert(ci->owner == jl_nothing); - return jl_atomic_load_relaxed(&ci->specptr.fptr1); -} - -static jl_value_t *add_builtin_func(const char *name, jl_fptr_args_t fptr) -{ - return jl_mk_builtin_func(NULL, name, fptr)->instance; -} - void jl_init_primitives(void) JL_GC_DISABLED { - jl_builtin_is = add_builtin_func("===", jl_f_is); - jl_builtin_typeof = add_builtin_func("typeof", jl_f_typeof); - jl_builtin_sizeof = add_builtin_func("sizeof", jl_f_sizeof); - jl_builtin_issubtype = add_builtin_func("<:", jl_f_issubtype); - jl_builtin_isa = add_builtin_func("isa", jl_f_isa); - jl_builtin_typeassert = add_builtin_func("typeassert", jl_f_typeassert); - jl_builtin_throw = add_builtin_func("throw", jl_f_throw); - jl_builtin_tuple = add_builtin_func("tuple", jl_f_tuple); - jl_builtin_ifelse = add_builtin_func("ifelse", jl_f_ifelse); - - // field access - jl_builtin_getfield = add_builtin_func("getfield", jl_f_getfield); - jl_builtin_setfield = add_builtin_func("setfield!", jl_f_setfield); - jl_builtin_setfieldonce = add_builtin_func("setfieldonce!", jl_f_setfieldonce); - jl_builtin_swapfield = add_builtin_func("swapfield!", jl_f_swapfield); - jl_builtin_modifyfield = add_builtin_func("modifyfield!", jl_f_modifyfield); - jl_builtin_replacefield = add_builtin_func("replacefield!", jl_f_replacefield); - jl_builtin_fieldtype = add_builtin_func("fieldtype", jl_f_fieldtype); - jl_builtin_nfields = add_builtin_func("nfields", jl_f_nfields); - jl_builtin_isdefined = add_builtin_func("isdefined", jl_f_isdefined); - - // module bindings - jl_builtin_getglobal = add_builtin_func("getglobal", jl_f_getglobal); - jl_builtin_setglobal = add_builtin_func("setglobal!", jl_f_setglobal); - jl_builtin_isdefinedglobal = add_builtin_func("isdefinedglobal", jl_f_isdefinedglobal); - add_builtin_func("get_binding_type", jl_f_get_binding_type); - jl_builtin_swapglobal = add_builtin_func("swapglobal!", jl_f_swapglobal); - jl_builtin_replaceglobal = add_builtin_func("replaceglobal!", jl_f_replaceglobal); - jl_builtin_modifyglobal = add_builtin_func("modifyglobal!", jl_f_modifyglobal); - jl_builtin_setglobalonce = add_builtin_func("setglobalonce!", jl_f_setglobalonce); - - // memory primitives - jl_builtin_memorynew = add_builtin_func("memorynew", jl_f_memorynew); - jl_builtin_memoryref = add_builtin_func("memoryrefnew", jl_f_memoryref); - jl_builtin_memoryrefoffset = add_builtin_func("memoryrefoffset", jl_f_memoryrefoffset); - jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); - jl_builtin_memoryrefset = add_builtin_func("memoryrefset!", jl_f_memoryrefset); - jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); - jl_builtin_memoryrefswap = add_builtin_func("memoryrefswap!", jl_f_memoryrefswap); - jl_builtin_memoryrefreplace = add_builtin_func("memoryrefreplace!", jl_f_memoryrefreplace); - jl_builtin_memoryrefmodify = add_builtin_func("memoryrefmodify!", jl_f_memoryrefmodify); - jl_builtin_memoryrefsetonce = add_builtin_func("memoryrefsetonce!", jl_f_memoryrefsetonce); - - // method table utils - jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); - jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke); - - // internal functions - jl_builtin_apply_type = add_builtin_func("apply_type", jl_f_apply_type); - jl_builtin__apply_iterate = add_builtin_func("_apply_iterate", jl_f__apply_iterate); - jl_builtin__expr = add_builtin_func("_expr", jl_f__expr); - jl_builtin_svec = add_builtin_func("svec", jl_f_svec); - add_builtin_func("invokelatest", jl_f_invokelatest); - add_builtin_func("invoke_in_world", jl_f_invoke_in_world); - add_builtin_func("_call_in_world_total", jl_f__call_in_world_total); - add_builtin_func("_typevar", jl_f__typevar); - add_builtin_func("_structtype", jl_f__structtype); - add_builtin_func("_abstracttype", jl_f__abstracttype); - add_builtin_func("_primitivetype", jl_f__primitivetype); - add_builtin_func("_setsuper!", jl_f__setsuper); - add_builtin_func("_defaultctors", jl_f__defaultctors); - jl_builtin__typebody = add_builtin_func("_typebody!", jl_f__typebody); - add_builtin_func("_equiv_typedef", jl_f__equiv_typedef); - jl_builtin_donotdelete = add_builtin_func("donotdelete", jl_f_donotdelete); - jl_builtin_compilerbarrier = add_builtin_func("compilerbarrier", jl_f_compilerbarrier); - add_builtin_func("finalizer", jl_f_finalizer); - add_builtin_func("_compute_sparams", jl_f__compute_sparams); - add_builtin_func("_svec_ref", jl_f__svec_ref); - jl_builtin_current_scope = add_builtin_func("current_scope", jl_f_current_scope); - add_builtin_func("throw_methoderror", jl_f_throw_methoderror); + // Builtins are specially considered available from world 0 + for (int i = 0; i < jl_n_builtins; i++) { + if (i == jl_builtin_id_intrinsic_call || + i == jl_builtin_id_opaque_closure_call) + continue; + jl_sym_t *sname = jl_symbol(jl_builtin_names[i]); + jl_value_t *builtin = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); + jl_set_initial_const(jl_core_module, sname, builtin, 0); + jl_mk_builtin_func((jl_datatype_t*)jl_typeof(builtin), sname, jl_builtin_f_addrs[i]); + jl_builtin_instances[i] = builtin; + } + add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); + add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); // builtin types add_builtin("Any", (jl_value_t*)jl_any_type); @@ -2558,14 +2507,12 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("PartialOpaque", (jl_value_t*)jl_partial_opaque_type); add_builtin("InterConditional", (jl_value_t*)jl_interconditional_type); add_builtin("MethodMatch", (jl_value_t*)jl_method_match_type); - add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); add_builtin("Function", (jl_value_t*)jl_function_type); add_builtin("Builtin", (jl_value_t*)jl_builtin_type); add_builtin("MethodInstance", (jl_value_t*)jl_method_instance_type); add_builtin("CodeInfo", (jl_value_t*)jl_code_info_type); add_builtin("LLVMPtr", (jl_value_t*)jl_llvmpointer_type); add_builtin("Task", (jl_value_t*)jl_task_type); - add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); add_builtin("AddrSpace", (jl_value_t*)jl_addrspace_type); add_builtin("Ref", (jl_value_t*)jl_ref_type); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d9b7b98e40ef4..cfa40b130eb97 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4662,7 +4662,7 @@ static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, const jl_datatype_layout_t *layout) { ++EmittedArrayNdIndex; - emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryref"); + emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryrefnew"); idx = update_julia_type(ctx, idx, (jl_value_t*)jl_long_type); if (idx.typ == jl_bottom_type) return jl_cgval_t(); diff --git a/src/codegen.cpp b/src/codegen.cpp index c89fc423bd948..a4817aa0e9bc8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -341,7 +341,7 @@ struct jl_tbaacache_t { MDNode *tbaa_ptrarraybuf; // Data in an array of boxed values MDNode *tbaa_arraybuf; // Data in an array of POD MDNode *tbaa_array; // jl_array_t or jl_genericmemory_t - MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to memoryref) + MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to a memoryref) MDNode *tbaa_arraysize; // A size in a jl_array_t MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_genericmemory_t MDNode *tbaa_memoryptr; // The pointer inside a jl_genericmemory_t @@ -555,7 +555,14 @@ FunctionType *invoke_type(TypeFnContextAndTriple f, Module &M) template struct JuliaFunction { public: - llvm::StringLiteral name; + template + constexpr JuliaFunction(const char (&cname)[N], TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(StringRef(cname, N-1)), _type(_type), _attrs(_attrs) {} + JuliaFunction(StringRef cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(cname), _type(_type), _attrs(_attrs) {} + JuliaFunction(char *cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) = delete; + + llvm::StringRef name; TypeFn_t _type; llvm::AttributeList (*_attrs)(llvm::LLVMContext &C); @@ -604,10 +611,6 @@ static FunctionType *get_func_sig(LLVMContext &C) { return JuliaType::get_jlfunc static FunctionType *get_func2_sig(LLVMContext &C) { return JuliaType::get_jlfunc2_ty(C); } static FunctionType *get_func3_sig(LLVMContext &C) { return JuliaType::get_jlfunc3_ty(C); } -static FunctionType *get_donotdelete_sig(LLVMContext &C) { - return FunctionType::get(getVoidTy(C), true); -} - static AttributeList get_func_attrs(LLVMContext &C) { return AttributeList::get(C, @@ -617,18 +620,6 @@ static AttributeList get_func_attrs(LLVMContext &C) Attributes(C, {Attribute::NoAlias, Attribute::ReadOnly, Attribute::NoCapture, Attribute::NoUndef})}); } -static AttributeList get_donotdelete_func_attrs(LLVMContext &C) -{ - AttrBuilder FnAttrs(C); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); - FnAttrs.addAttribute(Attribute::WillReturn); - FnAttrs.addAttribute(Attribute::NoUnwind); - return AttributeList::get(C, - AttributeSet::get(C, FnAttrs), - Attributes(C, {}), - None); -} - static AttributeList get_attrs_noreturn(LLVMContext &C) { return AttributeList::get(C, @@ -1440,11 +1431,21 @@ static const auto box_ssavalue_func = new JuliaFunction{ }, get_attrs_basic, }; -static const auto jlgetbuiltinfptr_func = new JuliaFunction<>{ - XSTR(jl_get_builtin_fptr), - [](LLVMContext &C) { return FunctionType::get(getPointerTy(C), - {JuliaType::get_prjlvalue_ty(C)}, false); }, - nullptr, +static const auto jldnd_func = new JuliaFunction<>{ + XSTR(jl_f_donotdelete), + [](LLVMContext &C) { + return FunctionType::get(getVoidTy(C), true); + }, + [](LLVMContext &C) { + AttrBuilder FnAttrs(C); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addAttribute(Attribute::WillReturn); + FnAttrs.addAttribute(Attribute::NoUnwind); + return AttributeList::get(C, + AttributeSet::get(C, FnAttrs), + Attributes(C, {}), + None); + }, }; // placeholder functions @@ -1558,56 +1559,23 @@ static const auto julia_call3 = new JuliaFunction<>{ static const auto jltuple_func = new JuliaFunction<>{XSTR(jl_f_tuple), get_func_sig, get_func_attrs}; static const auto jlintrinsic_func = new JuliaFunction<>{XSTR(jl_f_intrinsic_call), get_func3_sig, get_func_attrs}; +static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; + +static const auto mk_builtin_func_map() { + auto builtin_addrs = new DenseMap*>(); + for (int i = 0; i < jl_n_builtins; i++) { + jl_value_t *builtin = jl_builtin_instances[i]; + if (builtin) // a couple do not have instances (e.g. IntrinsicFunction) + (*builtin_addrs)[builtin] = new JuliaFunction<>{StringRef(jl_builtin_f_names[i]), get_func_sig, get_func_attrs}; + } + return builtin_addrs; +} static const auto &builtin_func_map() { - static auto builtins = new DenseMap*> { - { jl_f_is_addr, new JuliaFunction<>{XSTR(jl_f_is), get_func_sig, get_func_attrs} }, - { jl_f_typeof_addr, new JuliaFunction<>{XSTR(jl_f_typeof), get_func_sig, get_func_attrs} }, - { jl_f_sizeof_addr, new JuliaFunction<>{XSTR(jl_f_sizeof), get_func_sig, get_func_attrs} }, - { jl_f_issubtype_addr, new JuliaFunction<>{XSTR(jl_f_issubtype), get_func_sig, get_func_attrs} }, - { jl_f_isa_addr, new JuliaFunction<>{XSTR(jl_f_isa), get_func_sig, get_func_attrs} }, - { jl_f_typeassert_addr, new JuliaFunction<>{XSTR(jl_f_typeassert), get_func_sig, get_func_attrs} }, - { jl_f_ifelse_addr, new JuliaFunction<>{XSTR(jl_f_ifelse), get_func_sig, get_func_attrs} }, - { jl_f__apply_iterate_addr, new JuliaFunction<>{XSTR(jl_f__apply_iterate), get_func_sig, get_func_attrs} }, - { jl_f_invokelatest_addr, new JuliaFunction<>{XSTR(jl_f_invokelatest), get_func_sig, get_func_attrs} }, - { jl_f_invoke_in_world_addr, new JuliaFunction<>{XSTR(jl_f_invoke_in_world), get_func_sig, get_func_attrs} }, - { jl_f__call_in_world_total_addr, new JuliaFunction<>{XSTR(jl_f__call_in_world_total), get_func_sig, get_func_attrs} }, - { jl_f_throw_addr, new JuliaFunction<>{XSTR(jl_f_throw), get_func_sig, get_func_attrs} }, - { jl_f_throw_methoderror_addr, new JuliaFunction<>{XSTR(jl_f_throw_methoderror), get_func_sig, get_func_attrs} }, - { jl_f_tuple_addr, jltuple_func }, - { jl_f_svec_addr, new JuliaFunction<>{XSTR(jl_f_svec), get_func_sig, get_func_attrs} }, - { jl_f_applicable_addr, new JuliaFunction<>{XSTR(jl_f_applicable), get_func_sig, get_func_attrs} }, - { jl_f_invoke_addr, new JuliaFunction<>{XSTR(jl_f_invoke), get_func_sig, get_func_attrs} }, - { jl_f_isdefined_addr, new JuliaFunction<>{XSTR(jl_f_isdefined), get_func_sig, get_func_attrs} }, - { jl_f_getfield_addr, new JuliaFunction<>{XSTR(jl_f_getfield), get_func_sig, get_func_attrs} }, - { jl_f_setfield_addr, new JuliaFunction<>{XSTR(jl_f_setfield), get_func_sig, get_func_attrs} }, - { jl_f_swapfield_addr, new JuliaFunction<>{XSTR(jl_f_swapfield), get_func_sig, get_func_attrs} }, - { jl_f_modifyfield_addr, new JuliaFunction<>{XSTR(jl_f_modifyfield), get_func_sig, get_func_attrs} }, - { jl_f_fieldtype_addr, new JuliaFunction<>{XSTR(jl_f_fieldtype), get_func_sig, get_func_attrs} }, - { jl_f_nfields_addr, new JuliaFunction<>{XSTR(jl_f_nfields), get_func_sig, get_func_attrs} }, - { jl_f__expr_addr, new JuliaFunction<>{XSTR(jl_f__expr), get_func_sig, get_func_attrs} }, - { jl_f__typevar_addr, new JuliaFunction<>{XSTR(jl_f__typevar), get_func_sig, get_func_attrs} }, - { jl_f_memorynew_addr, new JuliaFunction<>{XSTR(jl_f_memorynew), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_addr, new JuliaFunction<>{XSTR(jl_f_memoryref), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefoffset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefoffset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefswap_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefswap), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefreplace_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefreplace), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefmodify_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefmodify), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefsetonce_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefsetonce), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, - { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, - { jl_f_donotdelete_addr, new JuliaFunction<>{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} }, - { jl_f_compilerbarrier_addr, new JuliaFunction<>{XSTR(jl_f_compilerbarrier), get_func_sig, get_func_attrs} }, - { jl_f_finalizer_addr, new JuliaFunction<>{XSTR(jl_f_finalizer), get_func_sig, get_func_attrs} }, - { jl_f__svec_ref_addr, new JuliaFunction<>{XSTR(jl_f__svec_ref), get_func_sig, get_func_attrs} }, - { jl_f_current_scope_addr, new JuliaFunction<>{XSTR(jl_f_current_scope), get_func_sig, get_func_attrs} }, - }; + static auto builtins = mk_builtin_func_map(); return *builtins; } -static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; - static _Atomic(uint64_t) globalUniqueGeneratedNames{1}; // --- code generation --- @@ -2901,7 +2869,7 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef arg return NULL; v[i] = args[i].constant; } - assert(v[0] == jl_builtin_apply_type); + assert(v[0] == BUILTIN(apply_type)); size_t last_age = jl_current_task->world_age; // call apply_type, but ignore errors. we know that will work in world 1. jl_current_task->world_age = 1; @@ -2971,7 +2939,7 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) if (e->head == jl_call_sym) { jl_value_t *f = static_eval(ctx, jl_exprarg(e, 0)); if (f) { - if (jl_array_dim0(e->args) == 3 && (f == jl_builtin_getfield || f == jl_builtin_getglobal)) { + if (jl_array_dim0(e->args) == 3 && (f == BUILTIN(getfield) || f == BUILTIN(getglobal))) { m = (jl_module_t*)static_eval(ctx, jl_exprarg(e, 1)); // Check the tag before evaluating `s` so that a value of random // type won't be corrupted. @@ -2990,10 +2958,11 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) } } } - else if (f==jl_builtin_tuple || f==jl_builtin_apply_type) { + else if (f==BUILTIN(tuple) || f==BUILTIN(apply_type)) { size_t i; size_t n = jl_array_dim0(e->args)-1; - if (n==0 && f==jl_builtin_tuple) return (jl_value_t*)jl_emptytuple; + if (n==0 && f==BUILTIN(tuple)) + return (jl_value_t*)jl_emptytuple; jl_value_t **v; JL_GC_PUSHARGS(v, n+1); v[0] = f; @@ -3625,11 +3594,11 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetglobal = f == jl_builtin_setglobal; - bool isreplaceglobal = f == jl_builtin_replaceglobal; - bool isswapglobal = f == jl_builtin_swapglobal; - bool ismodifyglobal = f == jl_builtin_modifyglobal; - bool issetglobalonce = f == jl_builtin_setglobalonce; + bool issetglobal = f == BUILTIN(setglobal); + bool isreplaceglobal = f == BUILTIN(replaceglobal); + bool isswapglobal = f == BUILTIN(swapglobal); + bool ismodifyglobal = f == BUILTIN(modifyglobal); + bool issetglobalonce = f == BUILTIN(setglobalonce); const jl_cgval_t undefval; const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; @@ -3698,11 +3667,11 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { ++EmittedOpfields; - bool issetfield = f == jl_builtin_setfield; - bool isreplacefield = f == jl_builtin_replacefield; - bool isswapfield = f == jl_builtin_swapfield; - bool ismodifyfield = f == jl_builtin_modifyfield; - bool issetfieldonce = f == jl_builtin_setfieldonce; + bool issetfield = f == BUILTIN(setfield); + bool isreplacefield = f == BUILTIN(replacefield); + bool isswapfield = f == BUILTIN(swapfield); + bool ismodifyfield = f == BUILTIN(modifyfield); + bool issetfieldonce = f == BUILTIN(setfieldonce); const jl_cgval_t undefval; const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; @@ -3845,11 +3814,11 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetmemory = f == jl_builtin_memoryrefset; - bool isreplacememory = f == jl_builtin_memoryrefreplace; - bool isswapmemory = f == jl_builtin_memoryrefswap; - bool ismodifymemory = f == jl_builtin_memoryrefmodify; - bool issetmemoryonce = f == jl_builtin_memoryrefsetonce; + bool issetmemory = f == BUILTIN(memoryrefset); + bool isreplacememory = f == BUILTIN(memoryrefreplace); + bool isswapmemory = f == BUILTIN(memoryrefswap); + bool ismodifymemory = f == BUILTIN(memoryrefmodify); + bool issetmemoryonce = f == BUILTIN(memoryrefsetonce); const jl_cgval_t undefval; const jl_cgval_t &ref = argv[1]; @@ -4035,14 +4004,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // returns true if the call has been handled { ++EmittedBuiltinCalls; - if (f == jl_builtin_is && nargs == 2) { + if (f == BUILTIN(is) && nargs == 2) { // emit comparison test Value *ans = emit_f_is(ctx, argv[1], argv[2]); *ret = mark_julia_type(ctx, ans, false, jl_bool_type); return true; } - else if (f == jl_builtin_typeof && nargs == 1) { + else if (f == BUILTIN(ifelse) && nargs == 3) { + *ret = emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); + return true; + } + + else if (f == BUILTIN(typeof) && nargs == 1) { const jl_cgval_t &p = argv[1]; if (p.constant) *ret = mark_julia_const(ctx, jl_typeof(p.constant)); @@ -4053,7 +4027,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_typeassert && nargs == 2) { + else if (f == BUILTIN(typeassert) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { @@ -4071,7 +4045,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isa && nargs == 2) { + else if (f == BUILTIN(isa) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { @@ -4082,7 +4056,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_issubtype && nargs == 2) { + else if (f == BUILTIN(issubtype) && nargs == 2) { const jl_cgval_t &ta = argv[1]; const jl_cgval_t &tb = argv[2]; if (jl_is_type_type(ta.typ) && !jl_has_free_typevars(ta.typ) && @@ -4093,7 +4067,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin__apply_iterate && nargs == 3) && ctx.vaSlot > 0) { + else if ((f == BUILTIN(_apply_iterate) && nargs == 3) && ctx.vaSlot > 0) { // turn Core._apply_iterate(iter, f, Tuple) ==> f(Tuple...) using the jlcall calling convention if Tuple is the va allocation if (LoadInst *load = dyn_cast_or_null(argv[3].V)) { if (load->getPointerOperand() == ctx.slots[ctx.vaSlot].boxroot && ctx.argArray) { @@ -4110,7 +4084,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_tuple) { + else if (f == BUILTIN(tuple)) { if (nargs == 0) { *ret = ghostValue(ctx, jl_emptytuple_type); return true; @@ -4121,14 +4095,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_throw && nargs == 1) { + else if (f == BUILTIN(throw) && nargs == 1) { Value *arg1 = boxed(ctx, argv[1]); raise_exception(ctx, arg1); *ret = jl_cgval_t(); return true; } - else if (f == jl_builtin_memorynew && (nargs == 2)) { + else if (f == BUILTIN(memorynew) && (nargs == 2)) { const jl_cgval_t &memty = argv[1]; if (!memty.constant) return false; @@ -4152,7 +4126,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_memoryref && nargs == 1) { + else if (f == BUILTIN(memoryrefnew) && nargs == 1) { const jl_cgval_t &mem = argv[1]; jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(mem.typ); if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { @@ -4163,7 +4137,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryref && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(memoryrefnew) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4171,13 +4145,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; if (nargs == 3) - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); return true; } } - else if (f == jl_builtin_memoryrefoffset && nargs == 1) { + else if (f == BUILTIN(memoryrefoffset) && nargs == 1) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4188,7 +4162,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryrefget && nargs == 3) { + else if (f == BUILTIN(memoryrefget) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4227,7 +4201,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } jl_value_t *boundscheck = argv[3].constant; - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefget"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); @@ -4304,16 +4278,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin_memoryrefset && nargs == 4) || - (f == jl_builtin_memoryrefswap && nargs == 4) || - (f == jl_builtin_memoryrefreplace && nargs == 6) || - (f == jl_builtin_memoryrefmodify && nargs == 5) || - (f == jl_builtin_memoryrefsetonce && nargs == 5)) { + else if ((f == BUILTIN(memoryrefset) && nargs == 4) || + (f == BUILTIN(memoryrefswap) && nargs == 4) || + (f == BUILTIN(memoryrefreplace) && nargs == 6) || + (f == BUILTIN(memoryrefmodify) && nargs == 5) || + (f == BUILTIN(memoryrefsetonce) && nargs == 5)) { return emit_f_opmemory(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { + else if (f == BUILTIN(memoryref_isassigned) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4384,7 +4358,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } if (!isboxed) elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); - // emit this using the same type as jl_builtin_memoryrefget + // emit this using the same type as BUILTIN(memoryrefget) // so that LLVM may be able to load-load forward them and fold the result auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -4413,7 +4387,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } - else if (f == jl_builtin_getfield && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(getfield) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4573,7 +4547,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if (f == jl_builtin_getglobal && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(getglobal) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4605,23 +4579,23 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if ((f == jl_builtin_setglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replaceglobal && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyglobal && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setglobalonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replaceglobal) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyglobal) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setglobalonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opglobal(ctx, ret, f, argv, nargs, nullptr); } - else if ((f == jl_builtin_setfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replacefield && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setfieldonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replacefield) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyfield) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setfieldonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opfield(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_nfields && nargs == 1) { + else if (f == BUILTIN(nfields) && nargs == 1) { const jl_cgval_t &obj = argv[1]; if (ctx.vaSlot > 0) { // optimize VA tuple @@ -4653,7 +4627,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_fieldtype && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(fieldtype) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &typ = argv[1]; const jl_cgval_t &fld = argv[2]; if ((jl_is_type_type(typ.typ) && jl_is_concrete_type(jl_tparam0(typ.typ))) || @@ -4678,7 +4652,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_sizeof && nargs == 1) { + else if (f == BUILTIN(sizeof) && nargs == 1) { const jl_cgval_t &obj = argv[1]; jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); assert(jl_string_type->name->mutabl); @@ -4723,7 +4697,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_apply_type && nargs > 0) { + else if (f == BUILTIN(apply_type) && nargs > 0) { if (jl_is_method(ctx.linfo->def.method)) { // don't bother codegen constant-folding for toplevel. jl_value_t *ty = static_apply_type(ctx, argv, nargs + 1); @@ -4737,7 +4711,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isdefinedglobal && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(isdefinedglobal) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; bool allow_import = true; @@ -4773,7 +4747,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_isdefined && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(isdefined) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; jl_datatype_t *stt = (jl_datatype_t*)obj.typ; @@ -4889,7 +4863,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_current_scope && (nargs == 0)) { + else if (f == BUILTIN(current_scope) && (nargs == 0)) { jl_aliasinfo_t scope_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); Instruction *v = scope_ai.decorateInst( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, get_scope_field(ctx), ctx.types().alignof_ptr)); @@ -4897,18 +4871,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_donotdelete) { + else if (f == BUILTIN(donotdelete)) { // For now we emit this as a vararg call to the builtin // (which doesn't look at the arguments). In the future, // this should be an LLVM builtin. - auto it = builtin_func_map().find(jl_f_donotdelete_addr); - if (it == builtin_func_map().end()) { - return false; - } - *ret = mark_julia_const(ctx, jl_nothing); FunctionType *Fty = FunctionType::get(getVoidTy(ctx.builder.getContext()), true); - Function *dnd = prepare_call(it->second); + Function *dnd = prepare_call(jldnd_func); SmallVector call_args; for (size_t i = 1; i <= nargs; ++i) { @@ -4927,7 +4896,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_compilerbarrier && (nargs == 2)) { + else if (f == BUILTIN(compilerbarrier) && (nargs == 2)) { emit_typecheck(ctx, argv[1], (jl_value_t*)jl_symbol_type, "compilerbarrier"); *ret = argv[2]; return true; @@ -5350,22 +5319,22 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ if (f.constant) { jl_cgval_t ret; auto it = builtin_func_map().end(); - if (f.constant == jl_builtin_modifyfield) { - if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv, nargs - 1, &lival)) + if (f.constant == BUILTIN(modifyfield)) { + if (emit_f_opfield(ctx, &ret, BUILTIN(modifyfield), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyfield_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_modifyglobal) { - if (emit_f_opglobal(ctx, &ret, jl_builtin_modifyglobal, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(modifyglobal)) { + if (emit_f_opglobal(ctx, &ret, BUILTIN(modifyglobal), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyglobal_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_memoryrefmodify) { - if (emit_f_opmemory(ctx, &ret, jl_builtin_memoryrefmodify, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(memoryrefmodify)) { + if (emit_f_opmemory(ctx, &ret, BUILTIN(memoryrefmodify), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_memoryrefmodify_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } else if (jl_typetagis(f.constant, jl_intrinsic_type)) { @@ -5425,15 +5394,15 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); } + // a couple intrinsics (really just llvmcall, though partly cglobal too) + // have non-standard (aka invalid) evaluation semantics, so we must handle these first if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); return emit_intrinsic(ctx, fi, args, nargs - 1); } size_t n_generic_args = nargs; - SmallVector argv(n_generic_args); - argv[0] = f; for (size_t i = 1; i < nargs; ++i) { argv[i] = emit_expr(ctx, args[i]); @@ -5441,37 +5410,23 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); // anything past here is unreachable } - if (jl_subtype(f.typ, (jl_value_t*)jl_builtin_type)) { - if (f.constant) { - if (f.constant == jl_builtin_ifelse && nargs == 4) - return emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); - jl_cgval_t result; - bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); - if (handled) - return result; - jl_fptr_args_t builtin_fptr = jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(f.constant)); - // special case for some known builtin not handled by emit_builtin_call - auto it = builtin_func_map().find(builtin_fptr); - if (it != builtin_func_map().end()) { - Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); - setName(ctx.emission_context, ret, it->second->name + "_ret"); - return mark_julia_type(ctx, ret, true, rt); - } - } - Value *fptr; - JuliaFunction<> *cc; - if (f.typ == (jl_value_t*)jl_intrinsic_type) { - fptr = prepare_call(jlintrinsic_func); - cc = julia_call3; - } - else { - fptr = ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)}); - cc = julia_call; - } - Value *ret = emit_jlcall(ctx, fptr, nullptr, argv, nargs, cc); + if (f.typ == (jl_value_t*)jl_intrinsic_type) { + Value *ret = emit_jlcall(ctx, prepare_call(jlintrinsic_func), nullptr, argv, nargs, julia_call3); setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); } + else if (f.constant && jl_isa(f.constant, (jl_value_t*)jl_builtin_type)) { + jl_cgval_t result; + bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); + if (handled) + return result; + auto it = builtin_func_map().find(f.constant); + if (it != builtin_func_map().end()) { + Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); + setName(ctx.emission_context, ret, it->second->name + "_ret"); + return mark_julia_type(ctx, ret, true, rt); + } + } // handle calling an OpaqueClosure if (jl_is_concrete_type(f.typ) && jl_subtype(f.typ, (jl_value_t*)jl_opaque_closure_type)) { @@ -9845,7 +9800,6 @@ static void init_jit_functions(void) { add_named_global("jl_fptr_args", jl_fptr_args_addr); add_named_global("jl_fptr_sparam", jl_fptr_sparam_addr); - add_named_global("jl_f_opaque_closure_call", &jl_f_opaque_closure_call); add_named_global(jl_small_typeof_var, &jl_small_typeof); add_named_global(jlstack_chk_guard_var, &__stack_chk_guard); add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle); @@ -9882,10 +9836,8 @@ static void init_jit_functions(void) add_named_global(jlcheckassign_func, &jl_checked_assignment); add_named_global(jlcheckbpwritable_func, &jl_check_binding_currently_writable); add_named_global(jlboundp_func, &jl_boundp); - for (auto it : builtin_func_map()) - add_named_global(it.second, it.first); - add_named_global(jlintrinsic_func, &jl_f_intrinsic_call); - add_named_global(jlgetbuiltinfptr_func, &jl_get_builtin_fptr); + for (int i = 0; i < jl_n_builtins; i++) + add_named_global(jl_builtin_f_names[i], jl_builtin_f_addrs[i]); add_named_global(jlapplygeneric_func, &jl_apply_generic); add_named_global(jlinvoke_func, &jl_invoke); add_named_global(jltopeval_func, &jl_toplevel_eval); diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index 2a6990bac52ff..e9c070ee8da6a 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -97,7 +97,7 @@ jl_symbol("pointerref"), jl_symbol("multidimensional.jl"), jl_symbol("Generator"), jl_symbol("leave"), -jl_symbol("memoryref"), +jl_symbol("memoryrefnew"), jl_symbol("show.jl"), jl_symbol("pointer_from_objref"), jl_symbol("memoryrefget"), diff --git a/src/gf.c b/src/gf.c index 2f4b838f04908..b130fbe05ca6d 100644 --- a/src/gf.c +++ b/src/gf.c @@ -288,16 +288,8 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ // ----- MethodInstance specialization instantiation ----- // -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED +void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED { - jl_sym_t *sname = jl_symbol(name); - if (dt == NULL) { - // Builtins are specially considered available from world 0 - jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); - jl_set_initial_const(jl_core_module, sname, f, 0); - dt = (jl_datatype_t*)jl_typeof(f); - } - jl_method_t *m = jl_new_method_uninit(jl_core_module); m->name = sname; m->module = jl_core_module; @@ -335,7 +327,6 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a mt->frozen = 1; JL_GC_POP(); - return dt; } // only relevant for bootstrapping. otherwise fairly broken. @@ -3110,6 +3101,7 @@ JL_DLLEXPORT const jl_callptr_t jl_fptr_const_return_addr = &jl_fptr_const_retur JL_DLLEXPORT const jl_callptr_t jl_fptr_sparam_addr = &jl_fptr_sparam; +JL_CALLABLE(jl_f_opaque_closure_call); JL_DLLEXPORT const jl_callptr_t jl_f_opaque_closure_call_addr = (jl_callptr_t)&jl_f_opaque_closure_call; JL_DLLEXPORT const jl_callptr_t jl_fptr_wait_for_compiled_addr = &jl_fptr_wait_for_compiled; diff --git a/src/init.c b/src/init.c index 6d5212cf8d370..cf432a6bd6047 100644 --- a/src/init.c +++ b/src/init.c @@ -23,9 +23,7 @@ #include "julia.h" #include "julia_internal.h" -#define DEFINE_BUILTIN_GLOBALS #include "builtin_proto.h" -#undef DEFINE_BUILTIN_GLOBALS #include "threading.h" #include "julia_assert.h" #include "processor.h" diff --git a/src/julia_internal.h b/src/julia_internal.h index 864097d8d22fe..5d418ceb2051c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -21,6 +21,9 @@ #include #include +#define STR(x) #x +#define XSTR(x) STR(x) + #if !defined(_WIN32) #include #else @@ -770,16 +773,11 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); #define JL_CALLABLE(name) \ JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs) -JL_CALLABLE(jl_f_svec); JL_CALLABLE(jl_f_tuple); -JL_CALLABLE(jl_f_intrinsic_call); -JL_CALLABLE(jl_f_opaque_closure_call); void jl_install_default_signal_handlers(void); void restore_signals(void); void jl_install_thread_signal_handler(jl_ptls_t ptls); -JL_DLLEXPORT jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt); - extern uv_loop_t *jl_io_loop; JL_DLLEXPORT void jl_uv_flush(uv_stream_t *stream); @@ -806,7 +804,7 @@ jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry); jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED; +void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index cfdb8eb5b1a99..1dece818fa998 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -13,9 +13,6 @@ #include "julia.h" -#define STR(csym) #csym -#define XSTR(csym) STR(csym) - static constexpr std::nullopt_t None = std::nullopt; enum AddressSpace { diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 30a6e0b37fe50..dbafc12aeff94 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -18,6 +18,9 @@ #include "julia_assert.h" #include "llvm-pass-helpers.h" +#define STR(csym) #csym +#define XSTR(csym) STR(csym) + using namespace llvm; JuliaPassContext::JuliaPassContext() diff --git a/src/method.c b/src/method.c index b3ed63e810f77..aa4f438ede080 100644 --- a/src/method.c +++ b/src/method.c @@ -10,13 +10,12 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "builtin_proto.h" #ifdef __cplusplus extern "C" { #endif -extern jl_value_t *jl_builtin_getfield; -extern jl_value_t *jl_builtin_tuple; jl_methtable_t *jl_kwcall_mt; jl_method_t *jl_opaque_closure_method; @@ -225,7 +224,7 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_sym_t *fe_sym = jl_globalref_name(fe); // look at some known called functions jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); - if (jl_get_binding_value_if_const(b) == jl_builtin_tuple) { + if (jl_get_binding_value_if_const(b) == BUILTIN(tuple)) { size_t j; for (j = 1; j < nargs; j++) { if (!jl_is_quotenode(jl_exprarg(e, j))) diff --git a/src/signals-mach.c b/src/signals-mach.c index 05f1111bab6ae..b7057416ad407 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -146,8 +146,6 @@ static void jl_mach_gc_wait(jl_ptls_t ptls2, mach_port_t thread, int16_t tid) static mach_port_t segv_port = 0; -#define STR(x) #x -#define XSTR(x) STR(x) #define HANDLE_MACH_ERROR(msg, retval) \ if (retval != KERN_SUCCESS) { mach_error(msg XSTR(: __FILE__:__LINE__:), (retval)); abort(); } diff --git a/src/staticdata.c b/src/staticdata.c index 64f66e87252aa..a8ebf1d9bc345 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -116,228 +116,177 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 197 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. -jl_value_t **const*const get_tags(void) { +static void get_tags(jl_value_t **tags[NUM_TAGS]) +{ // Make sure to keep an extra slot at the end to sentinel length - static void * _tags[NUM_TAGS] = {NULL}; - - // Lazyily-initialize this list - if (_tags[0] == NULL) { - unsigned int i = 0; -#define INSERT_TAG(sym) _tags[i++] = &(sym) - // builtin types - INSERT_TAG(jl_any_type); - INSERT_TAG(jl_symbol_type); - INSERT_TAG(jl_ssavalue_type); - INSERT_TAG(jl_datatype_type); - INSERT_TAG(jl_slotnumber_type); - INSERT_TAG(jl_simplevector_type); - INSERT_TAG(jl_array_type); - INSERT_TAG(jl_expr_type); - INSERT_TAG(jl_binding_type); - INSERT_TAG(jl_binding_partition_type); - INSERT_TAG(jl_globalref_type); - INSERT_TAG(jl_string_type); - INSERT_TAG(jl_module_type); - INSERT_TAG(jl_tvar_type); - INSERT_TAG(jl_method_instance_type); - INSERT_TAG(jl_method_type); - INSERT_TAG(jl_code_instance_type); - INSERT_TAG(jl_linenumbernode_type); - INSERT_TAG(jl_lineinfonode_type); - INSERT_TAG(jl_gotonode_type); - INSERT_TAG(jl_quotenode_type); - INSERT_TAG(jl_gotoifnot_type); - INSERT_TAG(jl_enternode_type); - INSERT_TAG(jl_argument_type); - INSERT_TAG(jl_returnnode_type); - INSERT_TAG(jl_const_type); - INSERT_TAG(jl_partial_struct_type); - INSERT_TAG(jl_partial_opaque_type); - INSERT_TAG(jl_interconditional_type); - INSERT_TAG(jl_method_match_type); - INSERT_TAG(jl_pinode_type); - INSERT_TAG(jl_phinode_type); - INSERT_TAG(jl_phicnode_type); - INSERT_TAG(jl_upsilonnode_type); - INSERT_TAG(jl_type_type); - INSERT_TAG(jl_bottom_type); - INSERT_TAG(jl_ref_type); - INSERT_TAG(jl_pointer_type); - INSERT_TAG(jl_llvmpointer_type); - INSERT_TAG(jl_vararg_type); - INSERT_TAG(jl_abstractarray_type); - INSERT_TAG(jl_densearray_type); - INSERT_TAG(jl_nothing_type); - INSERT_TAG(jl_function_type); - INSERT_TAG(jl_typeofbottom_type); - INSERT_TAG(jl_unionall_type); - INSERT_TAG(jl_typename_type); - INSERT_TAG(jl_builtin_type); - INSERT_TAG(jl_code_info_type); - INSERT_TAG(jl_opaque_closure_type); - INSERT_TAG(jl_task_type); - INSERT_TAG(jl_uniontype_type); - INSERT_TAG(jl_abstractstring_type); - INSERT_TAG(jl_array_any_type); - INSERT_TAG(jl_intrinsic_type); - INSERT_TAG(jl_methtable_type); - INSERT_TAG(jl_typemap_level_type); - INSERT_TAG(jl_typemap_entry_type); - INSERT_TAG(jl_voidpointer_type); - INSERT_TAG(jl_uint8pointer_type); - INSERT_TAG(jl_newvarnode_type); - INSERT_TAG(jl_anytuple_type_type); - INSERT_TAG(jl_anytuple_type); - INSERT_TAG(jl_namedtuple_type); - INSERT_TAG(jl_emptytuple_type); - INSERT_TAG(jl_array_symbol_type); - INSERT_TAG(jl_array_uint8_type); - INSERT_TAG(jl_array_uint32_type); - INSERT_TAG(jl_array_int32_type); - INSERT_TAG(jl_array_uint64_type); - INSERT_TAG(jl_int32_type); - INSERT_TAG(jl_int64_type); - INSERT_TAG(jl_bool_type); - INSERT_TAG(jl_uint8_type); - INSERT_TAG(jl_uint16_type); - INSERT_TAG(jl_uint32_type); - INSERT_TAG(jl_uint64_type); - INSERT_TAG(jl_char_type); - INSERT_TAG(jl_weakref_type); - INSERT_TAG(jl_int8_type); - INSERT_TAG(jl_int16_type); - INSERT_TAG(jl_float16_type); - INSERT_TAG(jl_float32_type); - INSERT_TAG(jl_float64_type); - INSERT_TAG(jl_bfloat16_type); - INSERT_TAG(jl_floatingpoint_type); - INSERT_TAG(jl_number_type); - INSERT_TAG(jl_signed_type); - INSERT_TAG(jl_pair_type); - INSERT_TAG(jl_genericmemory_type); - INSERT_TAG(jl_memory_any_type); - INSERT_TAG(jl_memory_uint8_type); - INSERT_TAG(jl_memory_uint16_type); - INSERT_TAG(jl_memory_uint32_type); - INSERT_TAG(jl_memory_uint64_type); - INSERT_TAG(jl_genericmemoryref_type); - INSERT_TAG(jl_memoryref_any_type); - INSERT_TAG(jl_memoryref_uint8_type); - INSERT_TAG(jl_addrspace_type); - INSERT_TAG(jl_addrspace_typename); - INSERT_TAG(jl_addrspacecore_type); - INSERT_TAG(jl_debuginfo_type); - INSERT_TAG(jl_abioverride_type); - - // special typenames - INSERT_TAG(jl_tuple_typename); - INSERT_TAG(jl_pointer_typename); - INSERT_TAG(jl_llvmpointer_typename); - INSERT_TAG(jl_array_typename); - INSERT_TAG(jl_type_typename); - INSERT_TAG(jl_namedtuple_typename); - INSERT_TAG(jl_vecelement_typename); - INSERT_TAG(jl_opaque_closure_typename); - INSERT_TAG(jl_genericmemory_typename); - INSERT_TAG(jl_genericmemoryref_typename); - - // special exceptions - INSERT_TAG(jl_errorexception_type); - INSERT_TAG(jl_argumenterror_type); - INSERT_TAG(jl_typeerror_type); - INSERT_TAG(jl_methoderror_type); - INSERT_TAG(jl_loaderror_type); - INSERT_TAG(jl_initerror_type); - INSERT_TAG(jl_undefvarerror_type); - INSERT_TAG(jl_fielderror_type); - INSERT_TAG(jl_stackovf_exception); - INSERT_TAG(jl_diverror_exception); - INSERT_TAG(jl_interrupt_exception); - INSERT_TAG(jl_boundserror_type); - INSERT_TAG(jl_memory_exception); - INSERT_TAG(jl_undefref_exception); - INSERT_TAG(jl_readonlymemory_exception); - INSERT_TAG(jl_atomicerror_type); - INSERT_TAG(jl_missingcodeerror_type); - INSERT_TAG(jl_precompilable_error); - INSERT_TAG(jl_trimfailure_type); - - // other special values - INSERT_TAG(jl_emptysvec); - INSERT_TAG(jl_emptytuple); - INSERT_TAG(jl_false); - INSERT_TAG(jl_true); - INSERT_TAG(jl_an_empty_string); - INSERT_TAG(jl_an_empty_vec_any); - INSERT_TAG(jl_an_empty_memory_any); - INSERT_TAG(jl_module_init_order); - INSERT_TAG(jl_core_module); - INSERT_TAG(jl_base_module); - INSERT_TAG(jl_main_module); - INSERT_TAG(jl_top_module); - INSERT_TAG(jl_typeinf_func); - INSERT_TAG(jl_type_type_mt); - INSERT_TAG(jl_nonfunction_mt); - INSERT_TAG(jl_kwcall_mt); - INSERT_TAG(jl_kwcall_func); - INSERT_TAG(jl_opaque_closure_method); - INSERT_TAG(jl_nulldebuginfo); - - // some Core.Builtin Functions that we want to be able to reference: - INSERT_TAG(jl_builtin_throw); - INSERT_TAG(jl_builtin_is); - INSERT_TAG(jl_builtin_typeof); - INSERT_TAG(jl_builtin_sizeof); - INSERT_TAG(jl_builtin_issubtype); - INSERT_TAG(jl_builtin_isa); - INSERT_TAG(jl_builtin_typeassert); - INSERT_TAG(jl_builtin__apply_iterate); - INSERT_TAG(jl_builtin_isdefined); - INSERT_TAG(jl_builtin_nfields); - INSERT_TAG(jl_builtin_tuple); - INSERT_TAG(jl_builtin_svec); - INSERT_TAG(jl_builtin_getfield); - INSERT_TAG(jl_builtin_setfield); - INSERT_TAG(jl_builtin_swapfield); - INSERT_TAG(jl_builtin_modifyfield); - INSERT_TAG(jl_builtin_replacefield); - INSERT_TAG(jl_builtin_setfieldonce); - INSERT_TAG(jl_builtin_fieldtype); - INSERT_TAG(jl_builtin_memorynew); - INSERT_TAG(jl_builtin_memoryref); - INSERT_TAG(jl_builtin_memoryrefoffset); - INSERT_TAG(jl_builtin_memoryrefget); - INSERT_TAG(jl_builtin_memoryrefset); - INSERT_TAG(jl_builtin_memoryref_isassigned); - INSERT_TAG(jl_builtin_memoryrefswap); - INSERT_TAG(jl_builtin_memoryrefmodify); - INSERT_TAG(jl_builtin_memoryrefreplace); - INSERT_TAG(jl_builtin_memoryrefsetonce); - INSERT_TAG(jl_builtin_apply_type); - INSERT_TAG(jl_builtin_applicable); - INSERT_TAG(jl_builtin_invoke); - INSERT_TAG(jl_builtin__expr); - INSERT_TAG(jl_builtin_ifelse); - INSERT_TAG(jl_builtin__typebody); - INSERT_TAG(jl_builtin_donotdelete); - INSERT_TAG(jl_builtin_compilerbarrier); - INSERT_TAG(jl_builtin_getglobal); - INSERT_TAG(jl_builtin_setglobal); - INSERT_TAG(jl_builtin_isdefinedglobal); - INSERT_TAG(jl_builtin_swapglobal); - INSERT_TAG(jl_builtin_modifyglobal); - INSERT_TAG(jl_builtin_replaceglobal); - INSERT_TAG(jl_builtin_setglobalonce); - INSERT_TAG(jl_builtin_current_scope); - // n.b. must update NUM_TAGS when you add something here + unsigned int i = 0; +#define INSERT_TAG(sym) tags[i++] = (jl_value_t**)&(sym) + // builtin types + INSERT_TAG(jl_any_type); + INSERT_TAG(jl_symbol_type); + INSERT_TAG(jl_ssavalue_type); + INSERT_TAG(jl_datatype_type); + INSERT_TAG(jl_slotnumber_type); + INSERT_TAG(jl_simplevector_type); + INSERT_TAG(jl_array_type); + INSERT_TAG(jl_expr_type); + INSERT_TAG(jl_binding_type); + INSERT_TAG(jl_binding_partition_type); + INSERT_TAG(jl_globalref_type); + INSERT_TAG(jl_string_type); + INSERT_TAG(jl_module_type); + INSERT_TAG(jl_tvar_type); + INSERT_TAG(jl_method_instance_type); + INSERT_TAG(jl_method_type); + INSERT_TAG(jl_code_instance_type); + INSERT_TAG(jl_linenumbernode_type); + INSERT_TAG(jl_lineinfonode_type); + INSERT_TAG(jl_gotonode_type); + INSERT_TAG(jl_quotenode_type); + INSERT_TAG(jl_gotoifnot_type); + INSERT_TAG(jl_enternode_type); + INSERT_TAG(jl_argument_type); + INSERT_TAG(jl_returnnode_type); + INSERT_TAG(jl_const_type); + INSERT_TAG(jl_partial_struct_type); + INSERT_TAG(jl_partial_opaque_type); + INSERT_TAG(jl_interconditional_type); + INSERT_TAG(jl_method_match_type); + INSERT_TAG(jl_pinode_type); + INSERT_TAG(jl_phinode_type); + INSERT_TAG(jl_phicnode_type); + INSERT_TAG(jl_upsilonnode_type); + INSERT_TAG(jl_type_type); + INSERT_TAG(jl_bottom_type); + INSERT_TAG(jl_ref_type); + INSERT_TAG(jl_pointer_type); + INSERT_TAG(jl_llvmpointer_type); + INSERT_TAG(jl_vararg_type); + INSERT_TAG(jl_abstractarray_type); + INSERT_TAG(jl_densearray_type); + INSERT_TAG(jl_nothing_type); + INSERT_TAG(jl_function_type); + INSERT_TAG(jl_typeofbottom_type); + INSERT_TAG(jl_unionall_type); + INSERT_TAG(jl_typename_type); + INSERT_TAG(jl_builtin_type); + INSERT_TAG(jl_code_info_type); + INSERT_TAG(jl_opaque_closure_type); + INSERT_TAG(jl_task_type); + INSERT_TAG(jl_uniontype_type); + INSERT_TAG(jl_abstractstring_type); + INSERT_TAG(jl_array_any_type); + INSERT_TAG(jl_intrinsic_type); + INSERT_TAG(jl_methtable_type); + INSERT_TAG(jl_typemap_level_type); + INSERT_TAG(jl_typemap_entry_type); + INSERT_TAG(jl_voidpointer_type); + INSERT_TAG(jl_uint8pointer_type); + INSERT_TAG(jl_newvarnode_type); + INSERT_TAG(jl_anytuple_type_type); + INSERT_TAG(jl_anytuple_type); + INSERT_TAG(jl_namedtuple_type); + INSERT_TAG(jl_emptytuple_type); + INSERT_TAG(jl_array_symbol_type); + INSERT_TAG(jl_array_uint8_type); + INSERT_TAG(jl_array_uint32_type); + INSERT_TAG(jl_array_int32_type); + INSERT_TAG(jl_array_uint64_type); + INSERT_TAG(jl_int32_type); + INSERT_TAG(jl_int64_type); + INSERT_TAG(jl_bool_type); + INSERT_TAG(jl_uint8_type); + INSERT_TAG(jl_uint16_type); + INSERT_TAG(jl_uint32_type); + INSERT_TAG(jl_uint64_type); + INSERT_TAG(jl_char_type); + INSERT_TAG(jl_weakref_type); + INSERT_TAG(jl_int8_type); + INSERT_TAG(jl_int16_type); + INSERT_TAG(jl_float16_type); + INSERT_TAG(jl_float32_type); + INSERT_TAG(jl_float64_type); + INSERT_TAG(jl_bfloat16_type); + INSERT_TAG(jl_floatingpoint_type); + INSERT_TAG(jl_number_type); + INSERT_TAG(jl_signed_type); + INSERT_TAG(jl_pair_type); + INSERT_TAG(jl_genericmemory_type); + INSERT_TAG(jl_memory_any_type); + INSERT_TAG(jl_memory_uint8_type); + INSERT_TAG(jl_memory_uint16_type); + INSERT_TAG(jl_memory_uint32_type); + INSERT_TAG(jl_memory_uint64_type); + INSERT_TAG(jl_genericmemoryref_type); + INSERT_TAG(jl_memoryref_any_type); + INSERT_TAG(jl_memoryref_uint8_type); + INSERT_TAG(jl_addrspace_type); + INSERT_TAG(jl_addrspace_typename); + INSERT_TAG(jl_addrspacecore_type); + INSERT_TAG(jl_debuginfo_type); + INSERT_TAG(jl_abioverride_type); + + // special typenames + INSERT_TAG(jl_tuple_typename); + INSERT_TAG(jl_pointer_typename); + INSERT_TAG(jl_llvmpointer_typename); + INSERT_TAG(jl_array_typename); + INSERT_TAG(jl_type_typename); + INSERT_TAG(jl_namedtuple_typename); + INSERT_TAG(jl_vecelement_typename); + INSERT_TAG(jl_opaque_closure_typename); + INSERT_TAG(jl_genericmemory_typename); + INSERT_TAG(jl_genericmemoryref_typename); + + // special exceptions + INSERT_TAG(jl_errorexception_type); + INSERT_TAG(jl_argumenterror_type); + INSERT_TAG(jl_typeerror_type); + INSERT_TAG(jl_methoderror_type); + INSERT_TAG(jl_loaderror_type); + INSERT_TAG(jl_initerror_type); + INSERT_TAG(jl_undefvarerror_type); + INSERT_TAG(jl_fielderror_type); + INSERT_TAG(jl_stackovf_exception); + INSERT_TAG(jl_diverror_exception); + INSERT_TAG(jl_interrupt_exception); + INSERT_TAG(jl_boundserror_type); + INSERT_TAG(jl_memory_exception); + INSERT_TAG(jl_undefref_exception); + INSERT_TAG(jl_readonlymemory_exception); + INSERT_TAG(jl_atomicerror_type); + INSERT_TAG(jl_missingcodeerror_type); + INSERT_TAG(jl_precompilable_error); + INSERT_TAG(jl_trimfailure_type); + + // other special values + INSERT_TAG(jl_emptysvec); + INSERT_TAG(jl_emptytuple); + INSERT_TAG(jl_false); + INSERT_TAG(jl_true); + INSERT_TAG(jl_an_empty_string); + INSERT_TAG(jl_an_empty_vec_any); + INSERT_TAG(jl_an_empty_memory_any); + INSERT_TAG(jl_module_init_order); + INSERT_TAG(jl_core_module); + INSERT_TAG(jl_base_module); + INSERT_TAG(jl_main_module); + INSERT_TAG(jl_top_module); + INSERT_TAG(jl_typeinf_func); + INSERT_TAG(jl_type_type_mt); + INSERT_TAG(jl_nonfunction_mt); + INSERT_TAG(jl_kwcall_mt); + INSERT_TAG(jl_kwcall_func); + INSERT_TAG(jl_opaque_closure_method); + INSERT_TAG(jl_nulldebuginfo); + // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG - assert(i == NUM_TAGS - 1); - } - return (jl_value_t**const*const) _tags; + assert(i == NUM_TAGS - 1); + tags[i] = NULL; } // hash of definitions for predefined tagged object @@ -373,12 +322,12 @@ static uintptr_t img_max; // HT_NOTFOUND is a valid integer ID, so we store the integer ids mangled. // This pair of functions mangles/demanges -static size_t from_seroder_entry(void *entry) +static size_t from_seroder_entry(void *entry) JL_NOTSAFEPOINT { return (size_t)((char*)entry - (char*)HT_NOTFOUND - 1); } -static void *to_seroder_entry(size_t idx) +static void *to_seroder_entry(size_t idx) JL_NOTSAFEPOINT { return (void*)((char*)HT_NOTFOUND + 1 + idx); } @@ -386,7 +335,7 @@ static void *to_seroder_entry(size_t idx) static htable_t new_methtables; static size_t precompilation_world; -static int ptr_cmp(const void *l, const void *r) +static int ptr_cmp(const void *l, const void *r) JL_NOTSAFEPOINT { uintptr_t left = *(const uintptr_t*)l; uintptr_t right = *(const uintptr_t*)r; @@ -394,7 +343,7 @@ static int ptr_cmp(const void *l, const void *r) } // Build an eytzinger tree from a sorted array -static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) +static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) JL_NOTSAFEPOINT { if (k <= n) { i = eytzinger(src, dest, i, 2 * k, n); @@ -434,7 +383,7 @@ static size_t eyt_obj_idx(jl_value_t *obj) JL_NOTSAFEPOINT } //used in staticdata.c after we add an image -void rebuild_image_blob_tree(void) +void rebuild_image_blob_tree(void) JL_NOTSAFEPOINT { size_t inc = 1 + jl_linkage_blobs.len - eytzinger_image_tree.len; assert(eytzinger_idxs.len == eytzinger_image_tree.len); @@ -510,33 +459,15 @@ JL_DLLEXPORT jl_value_t *jl_object_top_module(jl_value_t* v) JL_NOTSAFEPOINT } // hash of definitions for predefined function pointers +// (reverse is jl_builtin_f_addrs) static htable_t fptr_to_id; + void *native_functions; // opaque jl_native_code_desc_t blob used for fetching data from LLVM // table of struct field addresses to rewrite during saving static htable_t field_replace; static htable_t bits_replace; -// array of definitions for the predefined function pointers -// (reverse of fptr_to_id) -// This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C. -static const jl_fptr_args_t id_to_fptrs[] = { - &jl_f_throw, &jl_f_throw_methoderror, &jl_f_is, &jl_f_typeof, &jl_f_issubtype, &jl_f_isa, - &jl_f_typeassert, &jl_f__apply_iterate, - &jl_f_invokelatest, &jl_f_invoke_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_isdefinedglobal, - &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, - &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_setfieldonce, - &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, &jl_f_memorynew, - &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryref_isassigned, - &jl_f_memoryrefset, &jl_f_memoryrefswap, &jl_f_memoryrefmodify, &jl_f_memoryrefreplace, &jl_f_memoryrefsetonce, - &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, - &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, - &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f__defaultctors, - &jl_f_opaque_closure_call, &jl_f_donotdelete, &jl_f_compilerbarrier, &jl_f_get_binding_type, - &jl_f_getglobal, &jl_f_setglobal, &jl_f_swapglobal, &jl_f_modifyglobal, &jl_f_replaceglobal, &jl_f_setglobalonce, - &jl_f_finalizer, &jl_f__compute_sparams, &jl_f__svec_ref, - &jl_f_current_scope, - NULL }; typedef struct { ios_t *s; // the main stream @@ -1242,7 +1173,8 @@ static void write_pointer(ios_t *s) JL_NOTSAFEPOINT } // Records the buildid holding `v` and returns the tagged offset within the corresponding image -static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) { +static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED +{ size_t i = external_blob_index(v); if (i < n_linkage_blobs()) { // We found the sysimg/pkg that this item links against @@ -1271,7 +1203,7 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // but symbols, small integers, and a couple of special items (`nothing` and the root Task) // have special handling. #define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), link_ids) -static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_NOTSAFEPOINT +static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED { assert(v != NULL && "cannot get backref to NULL object"); if (jl_is_symbol(v)) { @@ -1484,7 +1416,7 @@ static void record_memoryrefs_inside(jl_serializer_state *s, jl_datatype_t *t, s } } -static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT +static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_GC_DISABLED { for (size_t i = 0; i < globals->len; i++) jl_queue_for_serialization(s, globals->items[i]); @@ -2126,7 +2058,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id"); + assert(offset < jl_n_builtins && "unknown function pointer id"); } else { assert(offset < JL_API_MAX && "unknown function pointer id"); @@ -2181,8 +2113,8 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID"); - return (uintptr_t)id_to_fptrs[offset]; + assert(offset < jl_n_builtins && "unknown function pointer ID"); + return (uintptr_t)jl_builtin_f_addrs[offset]; } switch ((jl_callingconv_t)offset) { case JL_API_BOXED: @@ -2395,7 +2327,7 @@ void gc_sweep_sysimg(void) // the image proper. For example, new methods added to external callables require // insertion into the appropriate method table. #define jl_write_value(s, v) _jl_write_value((s), (jl_value_t*)(v)) -static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) +static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) JL_GC_DISABLED { if (v == NULL) { write_reloc_t(s->s, 0); @@ -2497,7 +2429,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) jl_register_fptrs(image->base, &fvars, linfos, img_fvars_max); } -static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_NOTSAFEPOINT +static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_GC_DISABLED { size_t len = globals->len + external_fns->len; ios_ensureroom(s->gvar_record, len * sizeof(reloc_t)); @@ -3160,10 +3092,10 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, nsym_tag = 0; htable_new(&symbol_table, 0); - htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs)); + htable_new(&fptr_to_id, jl_n_builtins); uintptr_t i; - for (i = 0; id_to_fptrs[i] != NULL; i++) { - ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2)); + for (i = 0; i < jl_n_builtins; i++) { + ptrhash_put(&fptr_to_id, (void*)(uintptr_t)jl_builtin_f_addrs[i], (void*)(i + 2)); } htable_new(&serialization_order, 25000); htable_new(&nullptrs, 0); @@ -3202,11 +3134,15 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, 0); s.method_roots_list = NULL; htable_new(&s.method_roots_index, 0); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; if (worklist) { s.method_roots_list = jl_alloc_vec_any(0); s.worklist_key = jl_worklist_key(worklist); } - jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; + else { + get_tags(_tags); + } if (worklist == NULL) { // empty!(Core.ARGS) @@ -3232,6 +3168,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_queue_for_serialization(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_queue_for_serialization(&s, jl_builtin_instances[i]); jl_queue_for_serialization(&s, s.ptls->root_task->tls); } else { @@ -3428,6 +3366,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_write_value(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_write_value(&s, jl_builtin_instances[i]); jl_write_value(&s, global_roots_list); jl_write_value(&s, global_roots_keyset); jl_write_value(&s, s.ptls->root_task->tls); @@ -3866,7 +3806,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; s.ptls = ct->ptls; - jl_value_t **const*const tags = get_tags(); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; + if (!s.incremental) + get_tags(_tags); + htable_t new_dt_objs; htable_new(&new_dt_objs, 0); arraylist_new(&deser_sym, 0); @@ -3926,6 +3870,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_value_t **tag = tags[i]; *tag = jl_read_value(&s); } + for (i = 0; i < jl_n_builtins; i++) + jl_builtin_instances[i] = jl_read_value(&s); #define XX(name) \ ijl_small_typeof[(jl_##name##_tag << 4) / sizeof(*ijl_small_typeof)] = jl_##name##_type; JL_SMALL_TYPEOF(XX) diff --git a/src/toplevel.c b/src/toplevel.c index 868c4e28a2849..64a559ac774e6 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -426,7 +426,7 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int if (jl_is_intrinsic(called) && jl_unbox_int32(called) == (int)llvmcall) { *has_ccall = 1; } - if (called == jl_builtin__typebody) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) + if (called == BUILTIN(_typebody)) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) *has_defs = 1; } } @@ -705,7 +705,7 @@ static void jl_eval_throw(jl_module_t *m, jl_value_t *exc, const char *filename, { jl_value_t *throw_ex = (jl_value_t*)jl_exprn(jl_call_sym, 2); JL_GC_PUSH1(&throw_ex); - jl_exprargset(throw_ex, 0, jl_builtin_throw); + jl_exprargset(throw_ex, 0, BUILTIN(throw)); jl_exprargset(throw_ex, 1, exc); jl_toplevel_eval_flex(m, throw_ex, 0, 0, &filename, &lineno); JL_GC_POP(); From bdab032dbd80b6469353cd7fb8389c09512b2a9f Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Thu, 24 Apr 2025 07:39:38 -0700 Subject: [PATCH 127/662] Narrow `@inbounds` annotations in `accumulate.jl` to only indexing calls (#58200) `op` should not be `inbounds`-ed here the test case I added might technically be an illegal (or at least bad) use of `@propagate_inbounds` in case anyone has a suggestion for a more natural test case, but nonetheless it demonstrates getting a `BoundsError` instead of a segfault (worse) or incorrect / junk data (more worse), both of which happen on master --- base/accumulate.jl | 55 +++++++++++++++++++++++----------------- test/boundscheck_exec.jl | 7 +++++ 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/base/accumulate.jl b/base/accumulate.jl index 2748a4da481fa..c155ecfb4f75f 100644 --- a/base/accumulate.jl +++ b/base/accumulate.jl @@ -5,12 +5,14 @@ # it does double the number of operations compared to accumulate, # though for cheap operations like + this does not have much impact (20%) function _accumulate_pairwise!(op::Op, c::AbstractVector{T}, v::AbstractVector, s, i1, n)::T where {T,Op} - @inbounds if n < 128 - s_ = v[i1] - c[i1] = op(s, s_) + if n < 128 + @inbounds s_ = v[i1] + ci1 = op(s, s_) + @inbounds c[i1] = ci1 for i = i1+1:i1+n-1 - s_ = op(s_, v[i]) - c[i] = op(s, s_) + s_ = op(s_, @inbounds(v[i])) + ci = op(s, s_) + @inbounds c[i] = ci end else n2 = n >> 1 @@ -26,7 +28,8 @@ function accumulate_pairwise!(op::Op, result::AbstractVector, v::AbstractVector) n = length(li) n == 0 && return result i1 = first(li) - @inbounds result[i1] = v1 = reduce_first(op,v[i1]) + v1 = reduce_first(op, @inbounds(v[i1])) + @inbounds result[i1] = v1 n == 1 && return result _accumulate_pairwise!(op, result, v, v1, i1+1, n-1) return result @@ -378,16 +381,16 @@ function _accumulate!(op, B, A, dims::Integer, init::Union{Nothing, Some}) # We can accumulate to a temporary variable, which allows # register usage and will be slightly faster ind1 = inds_t[1] - @inbounds for I in CartesianIndices(tail(inds_t)) + for I in CartesianIndices(tail(inds_t)) if init === nothing - tmp = reduce_first(op, A[first(ind1), I]) + tmp = reduce_first(op, @inbounds(A[first(ind1), I])) else - tmp = op(something(init), A[first(ind1), I]) + tmp = op(something(init), @inbounds(A[first(ind1), I])) end - B[first(ind1), I] = tmp + @inbounds B[first(ind1), I] = tmp for i_1 = first(ind1)+1:last(ind1) - tmp = op(tmp, A[i_1, I]) - B[i_1, I] = tmp + tmp = op(tmp, @inbounds(A[i_1, I])) + @inbounds B[i_1, I] = tmp end end else @@ -401,12 +404,15 @@ end @noinline function _accumulaten!(op, B, A, R1, ind, R2, init::Nothing) # Copy the initial element in each 1d vector along dimension `dim` ii = first(ind) - @inbounds for J in R2, I in R1 - B[I, ii, J] = reduce_first(op, A[I, ii, J]) + for J in R2, I in R1 + tmp = reduce_first(op, @inbounds(A[I, ii, J])) + @inbounds B[I, ii, J] = tmp end # Accumulate - @inbounds for J in R2, i in first(ind)+1:last(ind), I in R1 - B[I, i, J] = op(B[I, i-1, J], A[I, i, J]) + for J in R2, i in first(ind)+1:last(ind), I in R1 + @inbounds Bv, Av = B[I, i-1, J], A[I, i, J] + tmp = op(Bv, Av) + @inbounds B[I, i, J] = tmp end B end @@ -414,12 +420,15 @@ end @noinline function _accumulaten!(op, B, A, R1, ind, R2, init::Some) # Copy the initial element in each 1d vector along dimension `dim` ii = first(ind) - @inbounds for J in R2, I in R1 - B[I, ii, J] = op(something(init), A[I, ii, J]) + for J in R2, I in R1 + tmp = op(something(init), @inbounds(A[I, ii, J])) + @inbounds B[I, ii, J] = tmp end # Accumulate - @inbounds for J in R2, i in first(ind)+1:last(ind), I in R1 - B[I, i, J] = op(B[I, i-1, J], A[I, i, J]) + for J in R2, i in first(ind)+1:last(ind), I in R1 + @inbounds Bv, Av = B[I, i-1, J], A[I, i, J] + tmp = op(Bv, Av) + @inbounds B[I, i, J] = tmp end B end @@ -433,10 +442,10 @@ function _accumulate1!(op, B, v1, A::AbstractVector, dim::Integer) cur_val = v1 B[i1] = cur_val next = iterate(inds, state) - @inbounds while next !== nothing + while next !== nothing (i, state) = next - cur_val = op(cur_val, A[i]) - B[i] = cur_val + cur_val = op(cur_val, @inbounds(A[i])) + @inbounds B[i] = cur_val next = iterate(inds, state) end return B diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index af57c8ee6c156..e4334619f19aa 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -239,6 +239,13 @@ if bc_opt != bc_off @test_throws BoundsError BadVector20469([1,2,3])[:] end +# Accumulate: do not set inbounds context for user-supplied functions +if bc_opt != bc_off + Base.@propagate_inbounds op58200(a, b) = (1, 2)[a] + (1, 2)[b] + @test_throws BoundsError accumulate(op58200, 1:10) + @test_throws BoundsError Base.accumulate_pairwise(op58200, 1:10) +end + # Ensure iteration over arrays is vectorizable function g27079(X) r = 0 From a47b58f2668436fa78563d08ec5157d523eae392 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 24 Apr 2025 11:22:05 -0400 Subject: [PATCH 128/662] "improve" allocations macro with more functions (#58057) Change the implementation of the allocations to always add a function call (closure) wrapper so that the inner code can be optimized, even if the outer scope is `@latestworld`. The implementation creates a closure if necessary (the code is complex) or just makes a call if the expression is simple (just a call on symbols). Many packages in the wild are observed to already be doing this themselves. --- base/timing.jl | 52 +++++++++++++++++++++++++++++----------- test/boundscheck_exec.jl | 2 +- test/math.jl | 15 ++++++------ 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/base/timing.jl b/base/timing.jl index 2dabe964f08c6..9e3a4cf128413 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -472,6 +472,35 @@ function gc_bytes() b[] end +function allocated(f, args::Vararg{Any,N}) where {N} + b0 = Ref{Int64}(0) + b1 = Ref{Int64}(0) + Base.gc_bytes(b0) + f(args...) + Base.gc_bytes(b1) + return b1[] - b0[] +end +only(methods(allocated)).called = 0xff + +function allocations(f, args::Vararg{Any,N}) where {N} + stats = Base.gc_num() + f(args...) + diff = Base.GC_Diff(Base.gc_num(), stats) + return Base.gc_alloc_count(diff) +end +only(methods(allocations)).called = 0xff + +function is_simply_call(@nospecialize ex) + Meta.isexpr(ex, :call) || return false + for a in ex.args + a isa QuoteNode && continue + a isa Symbol && continue + Base.is_self_quoting(a) && continue + return false + end + return true +end + """ @allocated @@ -487,15 +516,11 @@ julia> @allocated rand(10^6) ``` """ macro allocated(ex) - quote - Experimental.@force_compile - local b0 = Ref{Int64}(0) - local b1 = Ref{Int64}(0) - gc_bytes(b0) - $(esc(ex)) - gc_bytes(b1) - b1[] - b0[] + if !is_simply_call(ex) + ex = :((() -> $ex)()) end + pushfirst!(ex.args, GlobalRef(Base, :allocated)) + return esc(ex) end """ @@ -516,15 +541,14 @@ julia> @allocations rand(10^6) This macro was added in Julia 1.9. """ macro allocations(ex) - quote - Experimental.@force_compile - local stats = Base.gc_num() - $(esc(ex)) - local diff = Base.GC_Diff(Base.gc_num(), stats) - Base.gc_alloc_count(diff) + if !is_simply_call(ex) + ex = :((() -> $ex)()) end + pushfirst!(ex.args, GlobalRef(Base, :allocations)) + return esc(ex) end + """ @lock_conflicts diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index e4334619f19aa..e3d5e77c05384 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -350,7 +350,7 @@ if bc_opt == bc_default m1 === m2 end no_alias_prove(1) - @test_broken (@allocated no_alias_prove(5)) == 0 + @test (@allocated no_alias_prove(5)) == 0 end end diff --git a/test/math.jl b/test/math.jl index a32d66edc30f8..fa8dd172b0c2d 100644 --- a/test/math.jl +++ b/test/math.jl @@ -46,8 +46,7 @@ has_fma = Dict( @test clamp(100, Int8) === Int8(100) @test clamp(200, Int8) === typemax(Int8) - begin - x = [0.0, 1.0, 2.0, 3.0, 4.0] + let x = [0.0, 1.0, 2.0, 3.0, 4.0] clamp!(x, 1, 3) @test x == [1.0, 1.0, 2.0, 3.0, 3.0] end @@ -59,12 +58,14 @@ has_fma = Dict( @test clamp(typemax(UInt16), Int16) === Int16(32767) # clamp should not allocate a BigInt for typemax(Int16) - x = big(2) ^ 100 - @test (@allocated clamp(x, Int16)) == 0 + let x = big(2) ^ 100 + @test (@allocated clamp(x, Int16)) == 0 + end - x = clamp(2.0, BigInt) - @test x isa BigInt - @test x == big(2) + let x = clamp(2.0, BigInt) + @test x isa BigInt + @test x == big(2) + end end end From 485575c0f0a8432882da69dd0cac20159232bbb8 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Thu, 24 Apr 2025 12:24:33 -0700 Subject: [PATCH 129/662] Tracy: annotate timings in `JIT_Total` zone with function names Co-authored-by: Cody Tapscott --- src/jitlayers.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 081f3f59acedd..044a7ae651f6d 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2027,6 +2027,13 @@ void JuliaOJIT::addModule(orc::ThreadSafeModule TSM) TSM = (*JITPointers)(std::move(TSM)); auto Lock = TSM.getContext().getLock(); Module &M = *TSM.getModuleUnlocked(); + + for (auto &f : M) { + if (!f.isDeclaration()){ + jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, f.getName().str().c_str()); + } + } + // Treat this as if one of the passes might contain a safepoint // even though that shouldn't be the case and might be unwise Expected> Obj = CompileLayer.getCompiler()(M); From 8a8e3d1f988c602641d051552b9ed84e4d3f6764 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 24 Apr 2025 22:08:45 +0200 Subject: [PATCH 130/662] transition @zone in Core.Compiler to directly use jl_timing (#58213) This builds on the prior work from Kristoffer and Prem to implement a Julia-side `@zone` macro that interacts with the JL_TIMING system to provide profiling events to Tracy, VTune, and/or the TIMING_COUNTS back-end. This implementation includes some extra tricks to make sure that the timing block is stack-allocated, which can be important for performance if we start to use `@zone` for high-frequency events. Co-authored-by: Cody Tapscott Co-authored-by: Kristoffer Carlsson Co-authored-by: Prem Chintalapudi --- Compiler/src/Compiler.jl | 2 +- Compiler/src/profiling.jl | 33 ----------------- Compiler/src/profiling/ittapi.jl | 18 --------- Compiler/src/profiling/tracy.jl | 63 -------------------------------- Compiler/src/timing.jl | 38 +++++++++++++++++++ src/timing.c | 19 +--------- src/timing.h | 5 +-- 7 files changed, 43 insertions(+), 135 deletions(-) delete mode 100644 Compiler/src/profiling.jl delete mode 100644 Compiler/src/profiling/ittapi.jl delete mode 100644 Compiler/src/profiling/tracy.jl create mode 100644 Compiler/src/timing.jl diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index a046c71beff8f..a5a5f1b09a1b5 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -120,7 +120,7 @@ function is_return_type(Core.@nospecialize(f)) return false end -include("profiling.jl") +include("timing.jl") include("sort.jl") # We don't include some.jl, but this definition is still useful. diff --git a/Compiler/src/profiling.jl b/Compiler/src/profiling.jl deleted file mode 100644 index c29644cc902fc..0000000000000 --- a/Compiler/src/profiling.jl +++ /dev/null @@ -1,33 +0,0 @@ -const WITH_ITTAPI = ccall(:jl_ittapi_enabled, Cint, ()) != 0 -const WITH_TRACY = ccall(:jl_tracy_enabled, Cint, ()) != 0 - -include("profiling/tracy.jl") -include("profiling/ittapi.jl") - -if WITH_TRACY || WITH_ITTAPI - macro zone(name, ex::Expr) - srcloc = WITH_TRACY && Tracy.tracy_zone_create(name, ex, __source__) - tracy_begin_expr = WITH_TRACY ? :(ctx_tracy = Tracy.tracy_zone_begin($srcloc, true)) : :() - tracy_end_expr = WITH_TRACY ? :(Tracy.tracy_zone_end(ctx_tracy)) : :() - - event = WITH_ITTAPI && ITTAPI.ittapi_zone_create(name, ex, __source__) - ittapi_begin_expr = WITH_ITTAPI ? :(ctx_ittapi = ITTAPI.ittapi_zone_begin($event, true)) : :() - ittapi_end_expr = WITH_ITTAPI ? :(ITTAPI.ittapi_zone_end(ctx_ittapi)) : :() - - return quote - $tracy_begin_expr - $ittapi_begin_expr - $(Expr(:tryfinally, - :($(esc(ex))), - quote - $tracy_end_expr - $ittapi_end_expr - end - )) - end - end -else - macro zone(name::String, ex::Expr) - esc(ex) - end -end diff --git a/Compiler/src/profiling/ittapi.jl b/Compiler/src/profiling/ittapi.jl deleted file mode 100644 index 84140ef3a6002..0000000000000 --- a/Compiler/src/profiling/ittapi.jl +++ /dev/null @@ -1,18 +0,0 @@ -# Stubs -module ITTAPI - -import ..String, ..Expr, ..LineNumberNode, ..nothing - -function ittapi_zone_create(name::String, ex::Expr, linfo::LineNumberNode) - return nothing -end - -function ittapi_zone_begin(loc, active) - return nothing -end - -function ittapi_zone_end(ctx) - return nothing -end - -end diff --git a/Compiler/src/profiling/tracy.jl b/Compiler/src/profiling/tracy.jl deleted file mode 100644 index 5b1b7195b0f50..0000000000000 --- a/Compiler/src/profiling/tracy.jl +++ /dev/null @@ -1,63 +0,0 @@ -module Tracy - -import ..@noinline, ..@atomic, ..Cint, ..Vector, ..push!, ..unsafe_convert, ..esc, - ..pointer_from_objref, ..String, ..Ptr, ..UInt8, ..Cvoid, - ..Expr, ..LineNumberNode, ..Symbol, ..UInt32, ..C_NULL, ..(===) - -_strpointer(s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) - -mutable struct TracySrcLoc - @atomic zone_name::Ptr{UInt8} - @atomic function_name::Ptr{UInt8} - @atomic file::Ptr{UInt8} - const line::UInt32 - const color::UInt32 - # Roots - const zone_name_str::String - const function_name_sym::Symbol - const file_sym::Symbol -end -TracySrcLoc(zone_name::String, function_name::Symbol, file::Symbol, line::UInt32, color::UInt32) = - TracySrcLoc(C_NULL, C_NULL, C_NULL, line, color, zone_name, function_name, file) - -@noinline function reinit!(srcloc::TracySrcLoc) - @atomic :monotonic srcloc.file = unsafe_convert(Ptr{UInt8}, srcloc.file_sym) - @atomic :monotonic srcloc.function_name = unsafe_convert(Ptr{UInt8}, srcloc.function_name_sym) - @atomic :release srcloc.zone_name = _strpointer(srcloc.zone_name_str) -end - -struct TracyZoneCtx - id::UInt32 - active::Cint -end - -const srclocs = Vector{TracySrcLoc}() - -function tracy_zone_create(name::String, ex::Expr, linfo::LineNumberNode) - # Intern strings - for loc in srclocs - if loc.zone_name_str === name - name = loc.zone_name_str - break - end - end - loc = TracySrcLoc(name, Symbol("unknown"), linfo.file, UInt32(linfo.line), UInt32(0)) - # Also roots `loc` in `srclocs` - push!(srclocs, loc) - return loc -end - -function tracy_zone_begin(loc, active) - if (@atomic :acquire loc.zone_name) === Ptr{UInt8}(0) - reinit!(loc) - end - # `loc` is rooted in the global `srclocs` - ptr = Ptr{TracySrcLoc}(pointer_from_objref(loc)) - return ccall((:___tracy_emit_zone_begin, "libTracyClient"), TracyZoneCtx, (Ptr{TracySrcLoc}, Cint), ptr, active) -end - -function tracy_zone_end(ctx) - ccall((:___tracy_emit_zone_end, "libTracyClient"), Cvoid, (TracyZoneCtx,), ctx) -end - -end diff --git a/Compiler/src/timing.jl b/Compiler/src/timing.jl new file mode 100644 index 0000000000000..8b8240b025a66 --- /dev/null +++ b/Compiler/src/timing.jl @@ -0,0 +1,38 @@ +if ccall(:jl_timing_enabled, Cint, ()) != 0 + function getzonedexpr(name::Union{Symbol, String}, ex::Expr, func::Symbol, file::Symbol, line::Integer, color::Integer) + event = RefValue{Ptr{Cvoid}}(C_NULL) + name = QuoteNode(Symbol(name)) + func = QuoteNode(func) + file = QuoteNode(file) + + # XXX: This buffer must be large enough to store any jl_timing_block_t (runtime-checked) + buffer = (0, 0, 0, 0, 0, 0, 0) + buffer_size = Core.sizeof(buffer) + return quote + if $event[] === C_NULL + $event[] = ccall(:_jl_timing_event_create, Ptr{Cvoid}, + (Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Cint, Cint), + :CORE_COMPILER, $name, $func, $file, $line, $color) + end + timing_block = RefValue($buffer) + block_ptr = pointer_from_objref(timing_block) + $(Expr(:gc_preserve, quote + ccall(:_jl_timing_block_init, Cvoid, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid}), block_ptr, $buffer_size, $event[]) + ccall(:_jl_timing_block_start, Cvoid, (Ptr{Cvoid},), block_ptr) + $(Expr(:tryfinally, + :($(Expr(:escape, ex))), + quote + ccall(:_jl_timing_block_end, Cvoid, (Ptr{Cvoid},), block_ptr) + end + )) + end, :timing_block)) + end + end + macro zone(name, ex::Expr) + return getzonedexpr(name, ex, :unknown_julia_function, __source__.file, __source__.line, 0) + end +else + macro zone(name, ex::Expr) + return esc(ex) + end +end diff --git a/src/timing.c b/src/timing.c index ce9f98882710b..d84701475dd43 100644 --- a/src/timing.c +++ b/src/timing.c @@ -16,29 +16,14 @@ jl_module_t *jl_module_root(jl_module_t *m); extern "C" { #endif -JL_DLLEXPORT int jl_tracy_enabled(void) { -#ifdef USE_TRACY - return 1; -#else - return 0; -#endif -} - -JL_DLLEXPORT int jl_ittapi_enabled(void) { -#ifdef USE_ITTAPI +JL_DLLEXPORT int jl_timing_enabled(void) { +#ifdef ENABLE_TIMINGS return 1; #else return 0; #endif } -JL_DLLEXPORT int jl_nvtx_enabled(void) { -#ifdef USE_NVTX - return 1; -#else - return 0; -#endif -} #ifdef ENABLE_TIMINGS diff --git a/src/timing.h b/src/timing.h index ca6aebce445b8..833f5f68d34f5 100644 --- a/src/timing.h +++ b/src/timing.h @@ -55,9 +55,7 @@ JL_DLLEXPORT jl_timing_event_t *_jl_timing_event_create(const char *subsystem, c JL_DLLEXPORT void _jl_timing_block_init(char *buf, size_t size, jl_timing_event_t *event); JL_DLLEXPORT void _jl_timing_block_start(jl_timing_block_t *cur_block); JL_DLLEXPORT void _jl_timing_block_end(jl_timing_block_t *cur_block); -JL_DLLEXPORT int jl_tracy_enabled(void); -JL_DLLEXPORT int jl_ittapi_enabled(void); -JL_DLLEXPORT int jl_nvtx_enabled(void); +JL_DLLEXPORT int jl_timing_enabled(void); #ifdef __cplusplus @@ -191,6 +189,7 @@ JL_DLLEXPORT void jl_timing_puts(jl_timing_block_t *cur_block, const char *str); X(STACKWALK) \ X(DL_OPEN) \ X(JULIA_INIT) \ + X(CORE_COMPILER) \ #define JL_TIMING_COUNTERS \ From ead23dbe1032d96476722a3d6d62c38a7f17f18e Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 25 Apr 2025 09:34:45 +0900 Subject: [PATCH 131/662] improve type stability of `stat` (#58212) Make sure that its return type is inferred to be `String`. Allows JET to not report false positive errors from `@report_call Downloads.download(::String)` --- base/filesystem.jl | 2 +- base/libuv.jl | 3 ++- base/stat.jl | 3 ++- stdlib/FileWatching/src/FileWatching.jl | 5 ++--- test/file.jl | 2 ++ 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 87b3552c80a2f..2934fc15e392f 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -139,7 +139,7 @@ export File, import .Base: IOError, _UVError, _sizeof_uv_fs, check_open, close, closewrite, eof, eventloop, fd, isopen, bytesavailable, position, read, read!, readbytes!, readavailable, seek, seekend, show, - skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, + skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, _uv_error, setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, isexecutable, isreadable, iswritable, MutableDenseArrayType, truncate diff --git a/base/libuv.jl b/base/libuv.jl index 306854e9f4436..35b1a9097293e 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -103,7 +103,8 @@ struverror(err::Int32) = unsafe_string(ccall(:uv_strerror, Cstring, (Int32,), er uverrorname(err::Int32) = unsafe_string(ccall(:uv_err_name, Cstring, (Int32,), err)) uv_error(prefix::Symbol, c::Integer) = uv_error(string(prefix), c) -uv_error(prefix::AbstractString, c::Integer) = c < 0 ? throw(_UVError(prefix, c)) : nothing +uv_error(prefix::AbstractString, c::Integer) = c < 0 ? _uv_error(prefix, c) : nothing +_uv_error(prefix::AbstractString, c::Integer) = throw(_UVError(prefix, c)) ## event loop ## diff --git a/base/stat.jl b/base/stat.jl index 36496db4df415..8753c18f3ea38 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -183,7 +183,8 @@ show(io::IO, ::MIME"text/plain", st::StatStruct) = show_statstruct(io, st, false # stat & lstat functions -checkstat(s::StatStruct) = Int(s.ioerrno) in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL) ? s : uv_error(string("stat(", repr(s.desc), ")"), s.ioerrno) +checkstat(s::StatStruct) = Int(s.ioerrno) in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL) ? s : + _uv_error(string("stat(", repr(s.desc), ")"), s.ioerrno) macro stat_call(sym, arg1type, arg) return quote diff --git a/stdlib/FileWatching/src/FileWatching.jl b/stdlib/FileWatching/src/FileWatching.jl index 7c743ce634193..ebfdd9c8fea6b 100644 --- a/stdlib/FileWatching/src/FileWatching.jl +++ b/stdlib/FileWatching/src/FileWatching.jl @@ -488,12 +488,11 @@ end function getproperty(fdw::FDWatcher, s::Symbol) # support deprecated field names - s === :readable && return fdw.mask.readable - s === :writable && return fdw.mask.writable + s === :readable && return getfield(fdw, :mask).readable + s === :writable && return getfield(fdw, :mask).writable return getfield(fdw, s) end - close(t::_FDWatcher, mask::FDEvent) = close(t, mask.readable, mask.writable) function close(t::_FDWatcher, readable::Bool, writable::Bool) iolock_begin() diff --git a/test/file.jl b/test/file.jl index 9134073511cb2..a163bc07034ab 100644 --- a/test/file.jl +++ b/test/file.jl @@ -2165,3 +2165,5 @@ end @test dstat.total < 32PB @test dstat.used + dstat.available == dstat.total end + +@test Base.infer_return_type(stat, (String,)) == Base.Filesystem.StatStruct From 4441dc68fc926119b2adeb256a84164622a7f96a Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:25:52 +0200 Subject: [PATCH 132/662] Export `fieldindex` (#58119) At present, we export `Base.fieldname`, which maps a field's index to its symbol. Although we have the inverse function, `Base.fieldindex`, which maps a field's symbol to its index, defined and documented, we currently do not export it. This PR changes this and exports it to enhance the consistency and completeness of the field introspection functions. Additionally, it is unlikely that the function will undergo fundamental changes in the future, so exporting it should not pose a significant maintenance burden. The interface is likely fully defined by the following two properties. ```julia @test fieldname(Some{Int}, Base.fieldindex(Some{Int}, :value)) === :value @test Base.fieldindex(Some{Int}, fieldname(Some{Int}, 1)) === 1 ``` Besides doing the `export`, this PR adapts the documentation and adds test cases. Fixes #58092 Thanks to @nsajko for the support in covering all artifacts. --- NEWS.md | 2 ++ base/exports.jl | 1 + base/runtime_internals.jl | 12 +++++++++--- doc/src/base/base.md | 1 + test/reflection.jl | 4 ++++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3f6ab863b05e8..d7fa1e789a18f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -24,6 +24,8 @@ Build system changes New library functions --------------------- +* Exporting function `fieldindex` to get the index of a struct's field ([#58119]). + New library features -------------------- diff --git a/base/exports.jl b/base/exports.jl index 2e0bb3ccfe4cf..bf65ad8d1f07f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -807,6 +807,7 @@ export fieldoffset, fieldname, fieldnames, + fieldindex, fieldcount, fieldtypes, hasfield, diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 0599cc0b6ec36..696b3d1ab0a59 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1057,7 +1057,7 @@ String fieldtype """ - Base.fieldindex(T, name::Symbol, err:Bool=true) + fieldindex(T, name::Symbol, err:Bool=true) Get the index of a named field, throwing an error if the field does not exist (when err==true) or returning 0 (when err==false). @@ -1069,14 +1069,20 @@ julia> struct Foo y::String end -julia> Base.fieldindex(Foo, :z) +julia> fieldindex(Foo, :y) +2 + +julia> fieldindex(Foo, :z) ERROR: FieldError: type Foo has no field `z`, available fields: `x`, `y` Stacktrace: [...] -julia> Base.fieldindex(Foo, :z, false) +julia> fieldindex(Foo, :z, false) 0 ``` + +!!! compat "Julia 1.13" + This function is exported as of Julia 1.13. """ function fieldindex(T::DataType, name::Symbol, err::Bool=true) return err ? _fieldindex_maythrow(T, name) : _fieldindex_nothrow(T, name) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index b1f70c6230a18..695f6db46e848 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -207,6 +207,7 @@ Base.isstructtype Base.nameof(::DataType) Base.fieldnames Base.fieldname +Base.fieldindex Core.fieldtype Base.fieldtypes Base.fieldcount diff --git a/test/reflection.jl b/test/reflection.jl index b9d1eaa1c86f9..df271ef426f03 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -361,6 +361,10 @@ tlayout = TLayout(5,7,11) @test fieldtype(Union{Tuple{Char},Tuple{Char,Char}},2) === Char @test_throws BoundsError fieldtype(Union{Tuple{Char},Tuple{Char,Char}},3) +@test [fieldindex(TLayout, i) for i = (:x, :y, :z)] == [1, 2, 3] +@test fieldname(TLayout, fieldindex(TLayout, :z)) === :z +@test fieldindex(TLayout, fieldname(TLayout, 3)) === 3 + @test fieldnames(NTuple{3, Int}) == ntuple(i -> fieldname(NTuple{3, Int}, i), 3) == (1, 2, 3) @test_throws ArgumentError fieldnames(Union{}) @test_throws BoundsError fieldname(NTuple{3, Int}, 0) From 2fd8343b102bf72a4c6fc4d71501d3dfe5218b13 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Fri, 25 Apr 2025 15:38:54 +0200 Subject: [PATCH 133/662] TaskLocalRNG: delete confusing warning in docstring (#58208) --- stdlib/Random/src/Xoshiro.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 94c7e1ab24e1d..40e42d9fd7743 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -195,9 +195,6 @@ task creation, simulation results are also independent of the number of availabl threads / CPUs. The random stream should not depend on hardware specifics, up to endianness and possibly word size. -Using or seeding the RNG of any other task than the one returned by `current_task()` -is undefined behavior: it will work most of the time, and may sometimes fail silently. - When seeding `TaskLocalRNG()` with [`seed!`](@ref), the passed seed, if any, may be any integer. From 7487b8f1937ef09ee3d88654e045de1bf6083d33 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 25 Apr 2025 18:24:38 +0200 Subject: [PATCH 134/662] move writing to REPL history file to behind a PID lock (#45450) Should fix https://github.com/JuliaLang/julia/issues/37015 --------- Co-authored-by: KristofferC Co-authored-by: Ian Butterworth Co-authored-by: Matt Bauman --- stdlib/REPL/Project.toml | 1 + stdlib/REPL/src/REPL.jl | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 5ee3849f507f2..968786d492bcc 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -3,6 +3,7 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" [deps] +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JuliaSyntaxHighlighting = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index d349e75d128d4..f6314ed92ee53 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -41,6 +41,7 @@ end using Base.Meta, Sockets, StyledStrings using JuliaSyntaxHighlighting import InteractiveUtils +import FileWatching export AbstractREPL, @@ -927,7 +928,6 @@ function add_history(hist::REPLHistoryProvider, s::PromptState) # mode: $mode $(replace(str, r"^"ms => "\t")) """ - # TODO: write-lock history file try seekend(hist.history_file) catch err @@ -936,8 +936,15 @@ function add_history(hist::REPLHistoryProvider, s::PromptState) # If this doesn't fix it (e.g. when file is deleted), we'll end up rethrowing anyway hist_open_file(hist) end - print(hist.history_file, entry) - flush(hist.history_file) + if isfile(hist.file_path) + FileWatching.mkpidlock(hist.file_path * ".pid", stale_age=3) do + print(hist.history_file, entry) + flush(hist.history_file) + end + else # handle eg devnull + print(hist.history_file, entry) + flush(hist.history_file) + end nothing end From d9925544e0ca3023f9f35196ab2b4694dfa8dd04 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 25 Apr 2025 12:34:15 -0400 Subject: [PATCH 135/662] optimize Type method table queries and insertions (#58216) Ensure a total split of constructors and non-constructors, so they do not end up seaching the opposing table. Instead cache all kind lookups as keyed by typename(Type). Not really needed on its own, but it avoids a minor performance regression in https://github.com/JuliaLang/julia/pull/58131. --- base/staticdata.jl | 3 ++- src/typemap.c | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index 741937369b73b..cb0888e9d56bb 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -174,7 +174,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi edge = get_ci_mi(edge) end if edge isa MethodInstance - sig = typeintersect((edge.def::Method).sig, edge.specTypes) # TODO?? + sig = edge.specTypes min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world) j += 1 elseif edge isa Int @@ -346,6 +346,7 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn matched = nothing if invokesig === expected.sig # the invoke match is `expected` for `expected->sig`, unless `expected` is invalid + # TODO: this is broken since PR #53415 minworld = expected.primary_world maxworld = expected.deleted_world @assert minworld ≤ world diff --git a/src/typemap.c b/src/typemap.c index b8b699e101fe5..6ee8f9c599f3a 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -23,29 +23,29 @@ static int jl_is_any(jl_value_t *t1) return t1 == (jl_value_t*)jl_any_type; } -static jl_value_t *jl_type_extract_name(jl_value_t *t1 JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +static jl_value_t *jl_type_extract_name(jl_value_t *t1 JL_PROPAGATES_ROOT, int invariant) JL_NOTSAFEPOINT { if (jl_is_unionall(t1)) t1 = jl_unwrap_unionall(t1); if (jl_is_vararg(t1)) { - return jl_type_extract_name(jl_unwrap_vararg(t1)); + return jl_type_extract_name(jl_unwrap_vararg(t1), invariant); } else if (jl_is_typevar(t1)) { - return jl_type_extract_name(((jl_tvar_t*)t1)->ub); + return jl_type_extract_name(((jl_tvar_t*)t1)->ub, invariant); } else if (t1 == jl_bottom_type || t1 == (jl_value_t*)jl_typeofbottom_type || t1 == (jl_value_t*)jl_typeofbottom_type->super) { return (jl_value_t*)jl_typeofbottom_type->name; // put Union{} and typeof(Union{}) and Type{Union{}} together for convenience } else if (jl_is_datatype(t1)) { jl_datatype_t *dt = (jl_datatype_t*)t1; - if (!jl_is_kind(t1)) - return (jl_value_t*)dt->name; - return NULL; + if (jl_is_kind(t1) && !invariant) + return (jl_value_t*)jl_type_typename; + return (jl_value_t*)dt->name; } else if (jl_is_uniontype(t1)) { jl_uniontype_t *u1 = (jl_uniontype_t*)t1; - jl_value_t *tn1 = jl_type_extract_name(u1->a); - jl_value_t *tn2 = jl_type_extract_name(u1->b); + jl_value_t *tn1 = jl_type_extract_name(u1->a, invariant); + jl_value_t *tn2 = jl_type_extract_name(u1->b, invariant); if (tn1 == tn2) return tn1; // TODO: if invariant is false, instead find the nearest common ancestor @@ -71,7 +71,7 @@ static int jl_type_extract_name_precise(jl_value_t *t1, int invariant) } else if (jl_is_datatype(t1)) { jl_datatype_t *dt = (jl_datatype_t*)t1; - if ((invariant || !dt->name->abstract) && !jl_is_kind(t1)) + if (invariant || !dt->name->abstract || dt->name == jl_type_typename) return 1; return 0; } @@ -81,8 +81,8 @@ static int jl_type_extract_name_precise(jl_value_t *t1, int invariant) return 0; if (!jl_type_extract_name_precise(u1->b, invariant)) return 0; - jl_value_t *tn1 = jl_type_extract_name(u1->a); - jl_value_t *tn2 = jl_type_extract_name(u1->b); + jl_value_t *tn1 = jl_type_extract_name(u1->a, invariant); + jl_value_t *tn2 = jl_type_extract_name(u1->b, invariant); if (tn1 == tn2) return 1; return 0; @@ -469,7 +469,7 @@ static int jl_typemap_intersection_memory_visitor(jl_genericmemory_t *a, jl_valu tydt = (jl_datatype_t*)ttype; } else if (ttype) { - ttype = jl_type_extract_name(ttype); + ttype = jl_type_extract_name(ttype, tparam & 1); tydt = ttype ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)ttype)->wrapper) : NULL; } if (tydt == jl_any_type) @@ -641,7 +641,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, if (maybe_type && !maybe_kind) { typetype = jl_unwrap_unionall(ty); typetype = jl_is_type_type(typetype) ? jl_tparam0(typetype) : NULL; - name = typetype ? jl_type_extract_name(typetype) : NULL; + name = typetype ? jl_type_extract_name(typetype, 1) : NULL; if (!typetype) exclude_typeofbottom = !jl_subtype((jl_value_t*)jl_typeofbottom_type, ty); else if (jl_is_typevar(typetype)) @@ -717,7 +717,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } else { - jl_value_t *name = jl_type_extract_name(ty); + jl_value_t *name = jl_type_extract_name(ty, 0); if (name && jl_type_extract_name_precise(ty, 0)) { // direct lookup of leaf types jl_value_t *ml = mtcache_hash_lookup(cachearg1, name); @@ -782,7 +782,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *name = jl_type_extract_name(ty); + jl_value_t *name = jl_type_extract_name(ty, 0); if (name && jl_type_extract_name_precise(ty, 0)) { jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); // direct lookup of concrete types @@ -1003,7 +1003,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( // now look at the optimized TypeName caches jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); if (tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty)) : NULL; + jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty), 1) : NULL; if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); while (1) { @@ -1042,7 +1042,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { if (ty) { - jl_value_t *a0 = jl_type_extract_name(ty); + jl_value_t *a0 = jl_type_extract_name(ty, 0); if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); while (1) { @@ -1200,7 +1200,7 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v } jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); if (jl_is_kind(ty) && tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *name = jl_type_extract_name(a1); + jl_value_t *name = jl_type_extract_name(a1, 1); if (name) { if (ty != (jl_value_t*)jl_datatype_type) a1 = jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); @@ -1447,12 +1447,12 @@ static void jl_typemap_level_insert_( jl_value_t *a0; t1 = jl_unwrap_unionall(t1); if (jl_is_type_type(t1)) { - a0 = jl_type_extract_name(jl_tparam0(t1)); + a0 = jl_type_extract_name(jl_tparam0(t1), 1); jl_datatype_t *super = a0 ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper) : jl_any_type; jl_typemap_memory_insert_(map, &cache->tname, (jl_value_t*)super->name, newrec, (jl_value_t*)cache, 1, offs, NULL); return; } - a0 = jl_type_extract_name(t1); + a0 = jl_type_extract_name(t1, 0); if (a0 && a0 != (jl_value_t*)jl_any_type->name) { jl_typemap_memory_insert_(map, &cache->name1, a0, newrec, (jl_value_t*)cache, 0, offs, NULL); return; From c87456e4850de97bf06bd55ce438b8ff6cdb5b22 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 23 Apr 2025 08:40:44 -0700 Subject: [PATCH 136/662] Add `Core._lower` for swapping out lowering implementation Without using JuliaLowering, this should just call `fl_lower` in flfrontend.jl and behave as usual. This structure mirrors the parsing API. --- base/Base_compiler.jl | 3 +- base/boot.jl | 15 ++++++--- base/expr.jl | 3 +- base/{flparse.jl => flfrontend.jl} | 7 ++++ base/loading.jl | 6 +++- base/meta.jl | 2 +- src/ast.c | 46 +++++++++++++++++++++------ src/toplevel.c | 15 +++------ stdlib/InteractiveUtils/src/macros.jl | 6 ++-- stdlib/REPL/src/REPL.jl | 10 +++--- test/meta.jl | 2 +- 11 files changed, 79 insertions(+), 36 deletions(-) rename base/{flparse.jl => flfrontend.jl} (71%) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index e22a7d980f06c..5652b9738936c 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -375,8 +375,9 @@ const _return_type = Compiler.return_type # Enable compiler Compiler.bootstrap!() -include("flparse.jl") +include("flfrontend.jl") Core._setparser!(fl_parse) +Core._setlowerer!(fl_lower) # Further definition of Base will happen in Base.jl if loaded. diff --git a/base/boot.jl b/base/boot.jl index f9208c3874229..e97d3dea7927d 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -998,8 +998,9 @@ function struct_name_shim(@nospecialize(x), name::Symbol, mod::Module, @nospecia return x === mod ? t : getfield(x, name) end -# Binding for the julia parser, called as -# +# Bindings for the julia frontend. The internal jl_parse and jl_lower will call +# Core._parse and Core._lower respectively (if they are not `nothing`.) + # Core._parse(text, filename, lineno, offset, options) # # Parse Julia code from the buffer `text`, starting at `offset` and attributing @@ -1009,11 +1010,17 @@ end # # `_parse` must return an `svec` containing an `Expr` and the new offset as an # `Int`. -# -# The internal jl_parse will call into Core._parse if not `nothing`. _parse = nothing +# Core._lower(code, module, filename="none", linenum=0, world=0xfff..., warn=false) +# +# Lower `code` (usually Expr), returning `svec(e::Any xs::Any...)` where `e` is +# the lowered code, and `xs` is possible additional information from +# JuliaLowering (TBD). +_lower = nothing + _setparser!(parser) = setglobal!(Core, :_parse, parser) +_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer) # support for deprecated uses of builtin functions _apply(x...) = _apply_iterate(Main.Base.iterate, x...) diff --git a/base/expr.jl b/base/expr.jl index dd357736bc529..9b46795fc7ff9 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1669,7 +1669,8 @@ end # Implementation of generated functions function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) - ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod) + ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, defmod, "none", 0, typemax(Csize_t), 0)[1] if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error msg = ci.args[1] diff --git a/base/flparse.jl b/base/flfrontend.jl similarity index 71% rename from base/flparse.jl rename to base/flfrontend.jl index 8b474cf148fb2..86b291cf7328b 100644 --- a/base/flparse.jl +++ b/base/flfrontend.jl @@ -17,3 +17,10 @@ end function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) fl_parse(String(text), String(filename), lineno, offset, options) end + +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, lineno, world, warn) +end diff --git a/base/loading.jl b/base/loading.jl index b26a063247169..31faa86027b90 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2808,7 +2808,11 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString, loc = LineNumberNode(1, Symbol(filename)) try ast = Meta.parseall(code, filename=filename) - @assert Meta.isexpr(ast, :toplevel) + if !Meta.isexpr(ast, :toplevel) + @assert Core._lower != fl_lower + # Only reached when JuliaLowering and alternate parse functions are activated + return Core.eval(mod, ast) + end result = nothing line_and_ex = Expr(:toplevel, loc, nothing) for ex in ast.args diff --git a/base/meta.jl b/base/meta.jl index 32c78ce0639f8..5d880a7442b3e 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form for executing in module `m`. See also [`code_lowered`](@ref). """ -lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m) +lower(m::Module, @nospecialize(x)) = Core._lower(x, m, "none", 0, typemax(Csize_t), false)[1] """ @lower [m] x diff --git a/src/ast.c b/src/ast.c index b7edaac43f134..e8d60b03749bf 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1275,13 +1275,12 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule return expr; } -// Main entry point to flisp lowering. Most arguments are optional; see `jl_lower_expr_mod`. // warn: Print any lowering warnings returned; otherwise ignore JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn) + const char *filename, int line, size_t world, bool_t warn) { JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); + jl_timing_show_location(filename, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_array_t *kwargs = NULL; JL_GC_PUSH3(&expr, &kwargs, &inmodule); expr = jl_copy_ast(expr); @@ -1290,7 +1289,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, fl_context_t *fl_ctx = &ctx->fl; value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, - symbol(fl_ctx, file), fixnum(line)); + symbol(fl_ctx, filename), fixnum(line)); value_t lwr = car_(e); value_t warnings = car_(cdr_(e)); expr = scm_to_julia(fl_ctx, lwr, inmodule); @@ -1306,6 +1305,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, jl_error("julia-logmsg: bad argument list - expected " ":warn level (symbol) group (symbol) id file line msg . kwargs"); } + JL_GC_PUSH1(&warning); jl_value_t *level = jl_exprarg(warning, 0); jl_value_t *group = jl_exprarg(warning, 1); jl_value_t *id = jl_exprarg(warning, 2); @@ -1318,17 +1318,45 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, } JL_TYPECHK(logmsg, long, level); jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + JL_GC_POP(); } + jl_value_t *result = (jl_value_t *)jl_svec1(expr); JL_GC_POP(); - return expr; + return result; } -// Lower an expression tree into Julia's intermediate-representation. +// Main C entry point to lowering. Calls jl_fl_lower during bootstrap, and +// Core._lower otherwise (this is also jl_fl_lower unless we have JuliaLowering) JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn) + const char *filename, int line, size_t world, bool_t warn) { - // TODO: Allow change of lowerer - return jl_fl_lower(expr, inmodule, file, line, world, warn); + jl_value_t *core_lower = NULL; + if (jl_core_module) { + core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); + } + if (!core_lower || core_lower == jl_nothing) { + return jl_fl_lower(expr, inmodule, filename, line, world, warn); + } + jl_value_t **args; + JL_GC_PUSHARGS(args, 7); + args[0] = core_lower; + args[1] = expr; + args[2] = (jl_value_t*)inmodule; + args[3] = jl_cstr_to_string(filename); + args[4] = jl_box_ulong(line); + args[5] = jl_box_ulong(world); + args[6] = warn ? jl_true : jl_false; + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *result = jl_apply(args, 7); + ct->world_age = last_age; + args[0] = result; // root during error check below + JL_TYPECHK(parse, simplevector, result); + if (jl_svec_len(result) < 1) + jl_error("Result from lowering should be `svec(a::Any, x::Any...)`"); + JL_GC_POP(); + return result; } JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule) diff --git a/src/toplevel.c b/src/toplevel.c index 64a559ac774e6..7d2d209059f1c 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -195,8 +195,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0); + form = jl_svecref(jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0), 0); ct->world_age = jl_atomic_load_acquire(&jl_world_counter); (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno); } @@ -806,10 +805,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_GC_PUSH4(&mfunc, &thk, &ex, &root); size_t last_age = ct->world_age; - if (!expanded && jl_needs_lowering(e)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); - ct->world_age = last_age; + if (!expanded && (jl_needs_lowering(e))) { + ex = (jl_expr_t*)jl_svecref(jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL; @@ -971,8 +968,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val for (i = 0; i < jl_array_nrows(ex->args); i++) { root = jl_array_ptr_ref(ex->args, i); if (jl_needs_lowering(root)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); + root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } ct->world_age = jl_atomic_load_acquire(&jl_world_counter); res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno); @@ -1150,8 +1146,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, jl_lineno = lineno; continue; } - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); - expression = jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1); + expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0); ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 9c59dea132664..aa4092014361e 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -558,8 +558,8 @@ in the current environment. When using `@activate`, additional options for a component may be specified in square brackets `@activate Compiler[:option1, :option]` -Currently `@activate Compiler` is the only available component that may be -activatived. +Currently `Compiler` and `JuliaLowering` are the only available components that +may be activatived. For `@activate Compiler`, the following options are available: 1. `:reflection` - Activate the compiler for reflection purposes only. @@ -593,7 +593,7 @@ macro activate(what) if !isa(Component, Symbol) error("Usage Error: Component $Component is not a symbol") end - allowed_components = (:Compiler,) + allowed_components = (:Compiler, :JuliaLowering) if !(Component in allowed_components) error("Usage Error: Component $Component is not recognized. Expected one of $allowed_components") end diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f6314ed92ee53..637ac1ef6de7d 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -298,12 +298,12 @@ const install_packages_hooks = Any[] # N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL. # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro # code to run (and error). -__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0) -__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line) +__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + Core._lower(ast, mod, toplevel_file[], toplevel_line[])[1] +__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Csize_t}), mod, ast, 1, 1, toplevel_file, toplevel_line) -function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Cint}(1)) +function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Csize_t}(1)) if !isexpr(ast, :toplevel) ast = invokelatest(__repl_entry_lower_with_loc, mod, ast, toplevel_file, toplevel_line) check_for_missing_packages_and_run_hooks(ast) diff --git a/test/meta.jl b/test/meta.jl index f8707bc36791c..3d5fc08ee24e9 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar) @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar end -_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint), ex, m, "none", 0, world, 0) +_lower(m::Module, ex, world::UInt) = Base.fl_lower(ex, m, "none", 0, world, false)[1] module TestExpandInWorldModule macro m() 1 end From 39885378743d0cb6cbb9224ef112b1acf04bfa49 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Wed, 23 Apr 2025 08:43:05 -0700 Subject: [PATCH 137/662] Remove `jl_lower_expr_mod` --- doc/src/devdocs/eval.md | 2 +- src/ast.c | 5 ----- src/jl_exported_funcs.inc | 1 - src/julia.h | 1 - 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/src/devdocs/eval.md b/doc/src/devdocs/eval.md index e2c4adb741fcc..82bedc80116fe 100644 --- a/doc/src/devdocs/eval.md +++ b/doc/src/devdocs/eval.md @@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what else - Julia) via `fl_invoke_julia_macro()`, and back. -Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`, +Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`Core._lower()`, although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`. ## [Type Inference](@id dev-type-inference) diff --git a/src/ast.c b/src/ast.c index e8d60b03749bf..0e941f1d1fae8 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1359,11 +1359,6 @@ JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, return result; } -JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0); -} - jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) { JL_TIMING(LOWERING, LOWERING); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6cb9ab630a4b1..9f57de6420c52 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -285,7 +285,6 @@ XX(jl_load_file_string) \ XX(jl_lookup_code_address) \ XX(jl_lower) \ - XX(jl_lower_expr_mod) \ XX(jl_lseek) \ XX(jl_lstat) \ XX(jl_macroexpand) \ diff --git a/src/julia.h b/src/julia.h index 195c23f20662f..b1a4c583329f6 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2260,7 +2260,6 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, bool_t warn); -JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, const char *filename, size_t filename_len); From d9fafab2e0c5e15eed0c20082755a1c685d4f5b3 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Sat, 26 Apr 2025 21:21:09 +0530 Subject: [PATCH 138/662] Specialize `one` for the `SizedArray` test helper (#58209) Since the size of the array is encoded in the type, we may define `one` on the type. This is useful in certain linear algebra contexts. --- test/abstractarray.jl | 16 ++++++++++++++++ test/testhelpers/SizedArrays.jl | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index b882778e4b152..01e13f17460b5 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2191,6 +2191,22 @@ end @test one(Mat([1 2; 3 4])) == Mat([1 0; 0 1]) @test one(Mat([1 2; 3 4])) isa Mat + + @testset "SizedArray" begin + S = [1 2; 3 4] + A = SizedArrays.SizedArray{(2,2)}(S) + @test one(A) == one(typeof(A)) + @test oneunit(A) == oneunit(typeof(A)) + M = fill(A, 2, 2) + O = one(M) + for I in CartesianIndices(M) + if I[1] == I[2] + @test O[I] == one(S) + else + @test O[I] == zero(S) + end + end + end end @testset "copyto! with non-AbstractArray src" begin diff --git a/test/testhelpers/SizedArrays.jl b/test/testhelpers/SizedArrays.jl index 961784b89ab68..bd0272d78987d 100644 --- a/test/testhelpers/SizedArrays.jl +++ b/test/testhelpers/SizedArrays.jl @@ -54,6 +54,11 @@ Base.axes(a::SizedArray) = map(SOneTo, size(a)) Base.getindex(A::SizedArray, i...) = getindex(A.data, i...) Base.setindex!(A::SizedArray, v, i...) = setindex!(A.data, v, i...) Base.zero(::Type{T}) where T <: SizedArray = SizedArray{size(T)}(zeros(eltype(T), size(T))) +function Base.one(::Type{SizedMatrix{SZ,T,A}}) where {SZ,T,A} + allequal(SZ) || throw(DimensionMismatch("multiplicative identity defined only for square matrices")) + D = diagm(fill(one(T), SZ[1])) + SizedArray{SZ}(convert(A, D)) +end Base.parent(S::SizedArray) = S.data +(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = SizedArray{SZ}(S1.data + S2.data) ==(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = S1.data == S2.data From b4c12f4b820046f79918639a9c4c43702a63a30a Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sat, 26 Apr 2025 17:57:00 +0200 Subject: [PATCH 139/662] Random.DSFMT: thread-safe access to globals (#58120) --- stdlib/Random/src/DSFMT.jl | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/stdlib/Random/src/DSFMT.jl b/stdlib/Random/src/DSFMT.jl index 25155b4e8575d..9d4b2575a9556 100644 --- a/stdlib/Random/src/DSFMT.jl +++ b/stdlib/Random/src/DSFMT.jl @@ -126,15 +126,17 @@ function mulxmod!(f::GF2X, m::GF2X, deg=degree(m))::GF2X end # cache for X^(2i) mod m -const _squares = Dict{GF2X, Vector{GF2X}}() +const _squares = Base.Lockable(Dict{GF2X, Vector{GF2X}}()) # compute f^2 mod m function sqrmod!(f::GF2X, m::GF2X)::GF2X d = degree(m)-1 0 <= degree(f) <= d || throw(DomainError("f must satisfy 0 <= degree(f) <= degree(m)-1")) - sqrs = get!(_squares, m) do + sqrs = @lock _squares get(_squares[], m, nothing) + if sqrs === nothing x2i = GF2X(1) - GF2X[copy(mulxmod!(mulxmod!(x2i, m, d+1), m, d+1)) for i=1:d] + sqrs = GF2X[copy(mulxmod!(mulxmod!(x2i, m, d+1), m, d+1)) for i=1:d] + @lock _squares get!(_squares[], m, sqrs) end foldl(filter(i->coeff(f, i), 0:degree(f)); init=GF2X(0)) do g, i i <= d÷2 ? # optimization for "simple" squares @@ -154,16 +156,10 @@ function powxmod(e::BigInt, m::GF2X)::GF2X end "Cached jump polynomials for `MersenneTwister`." -const JumpPolys = Dict{BigInt,GF2X}() +const JumpPolys = Base.Lockable(Dict{BigInt,GF2X}()) -const CharPoly_ref = Ref{GF2X}() -# Ref because it can not be initialized at load time -function CharPoly() - if !isassigned(CharPoly_ref) - CharPoly_ref[] = GF2X(Poly19937) - end - return CharPoly_ref[] -end +# OncePerProcess because it can not be initialized at load time +const CharPoly = OncePerProcess{GF2X}(() -> GF2X(Poly19937)) """ calc_jump(steps::Integer) @@ -175,12 +171,17 @@ less than the period (e.g. ``steps ≪ 2^19937-1``). function calc_jump(steps::Integer, charpoly::GF2X=CharPoly())::GF2X steps < 0 && throw(DomainError("jump steps must be >= 0 (got $steps)")) - if isempty(JumpPolys) - JumpPolys[big(10)^20] = GF2X(JPOLY1e20) + poly = @lock JumpPolys begin + if isempty(JumpPolys[]) + JumpPolys[][big(10)^20] = GF2X(JPOLY1e20) + end + get(JumpPolys[], steps, nothing) end - get!(JumpPolys, steps) do - powxmod(big(steps), charpoly) + if poly === nothing + poly = powxmod(big(steps), charpoly) + @lock JumpPolys get!(JumpPolys[], steps, poly) end + poly end From 142f41901af14ad899918d089dc522ccd885c96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Sat, 26 Apr 2025 21:33:30 +0200 Subject: [PATCH 140/662] InteractiveUtils: support syntax found in stacktraces for introspection macros (#58222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continuing the work done at #57909, this PR adds support for the following syntax: ```julia @code_typed f(some_undef_var::Int) # some_undef_var is ignored, only the annotation is used @code_typed f(; x::Int) # same here, the name is used but not the value ``` This should allow us to copy and paste signatures found in stacktraces, such as ```julia julia> f(x; y = 3) = error() f (generic function with 1 method) julia> f(1) ERROR: Stacktrace: [1] error() @ Base ./error.jl:45 [2] f(x::Int64; y::Int64) @ Main ./REPL[40]:1 [3] top-level scope @ REPL[41]:1 julia> asin(-2) ERROR: DomainError with -2.0: asin(x) is not defined for |x| > 1. Stacktrace: [1] asin_domain_error(x::Float64) @ Base.Math ./special/trig.jl:429 [2] asin(x::Float64) @ Base.Math ./special/trig.jl:443 ``` where any function call may be copied and pasted into `@code_typed`, `@edit` etc as is, provided that the function and argument types are defined in the active module (i.e. `Main`). Thanks @topolarity for the idea. --------- Co-authored-by: Cédric Belmant --- NEWS.md | 2 +- stdlib/InteractiveUtils/src/macros.jl | 4 ++++ stdlib/InteractiveUtils/test/runtests.jl | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index d7fa1e789a18f..69f174d1b90b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,7 +49,7 @@ Standard library changes #### InteractiveUtils -* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}` ([#57909]). +* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}`. Type-annotated variables as in `f(val::Int; kw::Float64)` are not evaluated if the type annotation provides the necessary information, making this syntax compatible with signatures found in stacktraces ([#57909], [#58222]). External dependencies --------------------- diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 9c59dea132664..f00d0829c5a95 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -23,6 +23,7 @@ end function get_typeof(@nospecialize ex) isexpr(ex, :(::), 1) && return esc(ex.args[1]) + isexpr(ex, :(::), 2) && return esc(ex.args[2]) if isexpr(ex, :..., 1) splatted = ex.args[1] isexpr(splatted, :(::), 1) && return Expr(:curly, :Vararg, esc(splatted.args[1])) @@ -93,6 +94,7 @@ function are_kwargs_valid(kwargs::Vector{Any}) for kwarg in kwargs isexpr(kwarg, :..., 1) && continue isexpr(kwarg, :kw, 2) && isa(kwarg.args[1], Symbol) && continue + isexpr(kwarg, :(::), 2) && continue isa(kwarg, Symbol) && continue return false end @@ -113,6 +115,8 @@ function generate_merged_namedtuple_type(kwargs::Vector{Any}) push!(nts, Expr(:call, typeof_nt, esc(ex.args[1]))) elseif isexpr(ex, :kw, 2) push!(ntargs, ex.args[1]::Symbol => get_typeof(ex.args[2])) + elseif isexpr(ex, :(::), 2) + push!(ntargs, ex.args[1]::Symbol => get_typeof(ex)) else ex::Symbol push!(ntargs, ex => get_typeof(ex)) diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index d01e614abea84..4002e792fcf81 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -337,6 +337,7 @@ end @test (@which (::Base.RefValue{Int}).x = ::Int).name === :setproperty! @test (@which (::Float64)^2).name === :literal_pow @test (@which [::Int]).name === :vect + @test (@which [undef_var::Int]).name === :vect @test (@which [::Int 2]).name === :hcat @test (@which [::Int; 2]).name === :vcat @test (@which Int[::Int 2]).name === :typed_hcat @@ -345,6 +346,7 @@ end @test (@which Int[::Int 2;3 (::Int)]).name === :typed_hvcat @test (@which (::Vector{Float64})').name === :adjoint @test (@which "$(::Symbol) is a symbol").sig === Tuple{typeof(string), Vararg{Union{Char, String, Symbol}}} + @test (@which +(some_x::Int, some_y::Float64)).name === :+ @test (@which +(::Any, ::Any, ::Any, ::Any...)).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} @test (@which +(::Any, ::Any, ::Any, ::Vararg{Any})).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} n = length(@code_typed +(::Float64, ::Vararg{Float64})) @@ -358,6 +360,7 @@ end @test (@which +(::T, ::T) where {T<:Number}).sig === Tuple{typeof(+), T, T} where {T<:Number} @test (@which round(::Float64; digits=3)).name === :round @test (@which round(1.2; digits = ::Int)).name === :round + @test (@which round(1.2; digits::Int)).name === :round @test (@code_typed round(::T; digits = ::T) where {T<:Float64})[2] === Union{} @test (@code_typed round(::T; digits = ::T) where {T<:Int})[2] === Float64 base = 10 From c9ad04dcc73256bc24eb079f9f6506299b64b8ec Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Sun, 27 Apr 2025 12:53:58 +0800 Subject: [PATCH 141/662] Subtype: enable more Tuple related fast path. (#58196) Fix the subtyping hang found in https://github.com/JuliaLang/julia/issues/58115#issuecomment-2821388658 --- src/subtype.c | 38 +++++++++++++++++++++++++++++--------- test/subtype.jl | 4 ++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index ed6a8818dc3f2..3d50dc492e190 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1069,7 +1069,8 @@ static int forall_exists_equal(jl_value_t *x, jl_value_t *y, jl_stenv_t *e); static int subtype_tuple_varargs( jl_vararg_t *vtx, jl_vararg_t *vty, - size_t vx, size_t vy, + jl_value_t *lastx, jl_value_t *lasty, + size_t vx, size_t vy, size_t x_reps, jl_stenv_t *e, int param) { jl_value_t *xp0 = jl_unwrap_vararg(vtx); jl_value_t *xp1 = jl_unwrap_vararg_num(vtx); @@ -1111,12 +1112,30 @@ static int subtype_tuple_varargs( } } } - - // in Vararg{T1} <: Vararg{T2}, need to check subtype twice to - // simulate the possibility of multiple arguments, which is needed - // to implement the diagonal rule correctly. - if (!subtype(xp0, yp0, e, param)) return 0; - if (!subtype(xp0, yp0, e, 1)) return 0; + int x_same = vx > 1 || (lastx && obviously_egal(xp0, lastx)); + int y_same = vy > 1 || (lasty && obviously_egal(yp0, lasty)); + // keep track of number of consecutive identical subtyping + x_reps = y_same && x_same ? x_reps + 1 : 1; + if (x_reps > 2) { + // an identical type on the left doesn't need to be compared to the same + // element type on the right more than twice. + } + else if (x_same && e->Runions.depth == 0 && y_same && + !jl_has_free_typevars(xp0) && !jl_has_free_typevars(yp0)) { + // fast path for repeated elements + } + else if ((e->Runions.depth == 0 ? !jl_has_free_typevars(xp0) : jl_is_concrete_type(xp0)) && !jl_has_free_typevars(yp0)) { + // fast path for separable sub-formulas + if (!jl_subtype(xp0, yp0)) + return 0; + } + else { + // in Vararg{T1} <: Vararg{T2}, need to check subtype twice to + // simulate the possibility of multiple arguments, which is needed + // to implement the diagonal rule correctly. + if (!subtype(xp0, yp0, e, param)) return 0; + if (x_reps < 2 && !subtype(xp0, yp0, e, 1)) return 0; + } constrain_length: if (!yp1) { @@ -1246,7 +1265,8 @@ static int subtype_tuple_tail(jl_datatype_t *xd, jl_datatype_t *yd, int8_t R, jl return subtype_tuple_varargs( (jl_vararg_t*)xi, (jl_vararg_t*)yi, - vx, vy, e, param); + lastx, lasty, + vx, vy, x_reps, e, param); } if (j >= ly) @@ -1267,7 +1287,7 @@ static int subtype_tuple_tail(jl_datatype_t *xd, jl_datatype_t *yd, int8_t R, jl (yi == lastx && !vx && vy && jl_is_concrete_type(xi)))) { // fast path for repeated elements } - else if (e->Runions.depth == 0 && !jl_has_free_typevars(xi) && !jl_has_free_typevars(yi)) { + else if ((e->Runions.depth == 0 ? !jl_has_free_typevars(xi) : jl_is_concrete_type(xi)) && !jl_has_free_typevars(yi)) { // fast path for separable sub-formulas if (!jl_subtype(xi, yi)) return 0; diff --git a/test/subtype.jl b/test/subtype.jl index da69022b1466e..dbd761c7f5867 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2782,3 +2782,7 @@ let Tvar1 = TypeVar(:Tvar1), Tvar2 = TypeVar(:Tvar2) V2 = UnionAll(Tvar2, Union{(@eval($(Symbol(:T58129, k)){$Tvar2}) for k in 1:100)...}) @test Set{<:V2} <: AbstractSet{<:V1} end + +#issue 58115 +@test Tuple{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{ Union{Tuple{}, Tuple{Tuple{}}}}}}}}}}}}} , Tuple{}} <: + Tuple{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Union{Tuple{}, Tuple{Tuple{}}}}}}}}}}}}}}}, Tuple{}} From 0cdbbf7cb590f15f86ffd0d806cdb004eb188809 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 27 Apr 2025 10:45:03 +0200 Subject: [PATCH 142/662] add backedge insertion to the timings (#58237) RIght now we have big uninsturmented "holes" between packages getting loaded. These mostly cover up those holes. --- base/loading.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b26a063247169..9de9dd07e5428 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1286,8 +1286,9 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No ext_edges = sv[4]::Union{Nothing,Vector{Any}} extext_methods = sv[5]::Vector{Any} internal_methods = sv[6]::Vector{Any} - StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods) - + Compiler.@zone "CC: INSERT_BACKEDGES" begin + StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods) + end restored = register_restored_modules(sv, pkg, path) for M in restored From db75908f97355337efbb7fe046cef0707449ac78 Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Sun, 27 Apr 2025 13:59:10 +0200 Subject: [PATCH 143/662] Support field name in `fieldoffset` (#58100) `fieldtype` accepts both the field's name or the field's index. It's inconsistent that `fieldoffset` only accepts the field's index, but not its name. This commit adds support to `fieldoffset` for the field's name, documents and tests it. --------- Co-authored-by: Alex Arslan Co-authored-by: Lilith Orion Hafner --- NEWS.md | 1 + base/runtime_internals.jl | 25 ++++++++++++++++++++++--- test/reflection.jl | 1 + 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 69f174d1b90b5..c6db2bbfbbe28 100644 --- a/NEWS.md +++ b/NEWS.md @@ -29,6 +29,7 @@ New library functions New library features -------------------- +* `fieldoffset` now also accepts the field name as a symbol as `fieldtype` already did ([#58100]). * `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). * `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 696b3d1ab0a59..d8c90b41842c8 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1007,10 +1007,25 @@ morespecific(@nospecialize(a), @nospecialize(b)) = (@_total_meta; ccall(:jl_type morespecific(a::Method, b::Method) = ccall(:jl_method_morespecific, Cint, (Any, Any), a, b) != 0 """ - fieldoffset(type, i) + fieldoffset(type, name::Symbol | i::Integer) -The byte offset of field `i` of a type relative to the data start. For example, we could -use it in the following manner to summarize information about a struct: +The byte offset of a field (specified by name or index) of a type relative to its start. + +# Examples +```jldoctest +julia> struct Foo + x::Int64 + y::String + end + +julia> fieldoffset(Foo, 2) +0x0000000000000008 + +julia> fieldoffset(Foo, :x) +0x0000000000000000 +``` + +We can use it to summarize information about a struct: ```jldoctest julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:fieldcount(T)]; @@ -1032,8 +1047,12 @@ julia> structinfo(Base.Filesystem.StatStruct) (0x0000000000000060, :ctime, Float64) (0x0000000000000068, :ioerrno, Int32) ``` + +!!! compat "Julia 1.13" + Specifying the field by name rather than index requires Julia 1.13 or later. """ fieldoffset(x::DataType, idx::Integer) = (@_foldable_meta; ccall(:jl_get_field_offset, Csize_t, (Any, Cint), x, idx)) +fieldoffset(x::DataType, name::Symbol) = fieldoffset(x, fieldindex(x, name)) """ fieldtype(T, name::Symbol | index::Int) diff --git a/test/reflection.jl b/test/reflection.jl index df271ef426f03..25b61945aefac 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -346,6 +346,7 @@ tlayout = TLayout(5,7,11) @test !hasproperty(tlayout, :p) @test [(fieldoffset(TLayout,i), fieldname(TLayout,i), fieldtype(TLayout,i)) for i = 1:fieldcount(TLayout)] == [(0, :x, Int8), (2, :y, Int16), (4, :z, Int32)] +@test [fieldoffset(TLayout, s) for s = (:x, :y, :z)] == [0, 2, 4] @test fieldnames(Complex) === (:re, :im) @test_throws BoundsError fieldtype(TLayout, 0) @test_throws ArgumentError fieldname(TLayout, 0) From 8d05eb6084e7f5fef5738324ac1ccd7999a5cd1f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 28 Apr 2025 07:54:11 +0200 Subject: [PATCH 144/662] add `TYPE_CACHE_INSERT` to the disabled frequent event list (#58235) --- src/timing.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/timing.c b/src/timing.c index d84701475dd43..f0abd19a1e347 100644 --- a/src/timing.c +++ b/src/timing.c @@ -194,6 +194,7 @@ void jl_init_timing(void) error |= jl_timing_set_enable("METHOD_LOOKUP_FAST", 0); error |= jl_timing_set_enable("AST_COMPRESS", 0); error |= jl_timing_set_enable("AST_UNCOMPRESS", 0); + error |= jl_timing_set_enable("TYPE_CACHE_INSERT", 0); if (error) jl_error("invalid timing subsystem encountered in jl_init_timing"); #endif From 175ef3eb02c922439ef91ef832f1ac4620fb1f61 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 28 Apr 2025 09:13:27 -0400 Subject: [PATCH 145/662] Test: Move the visual space in result prints to help grouping (#58198) --- base/errorshow.jl | 33 ++++++++++++++++++++------------- stdlib/Test/docs/src/index.md | 2 -- stdlib/Test/src/Test.jl | 12 +++++------- stdlib/Test/test/runtests.jl | 4 ++-- test/buildkitetestjson.jl | 3 +++ 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 23d6b771e954c..95ed2a51b57a4 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -638,14 +638,16 @@ const update_stackframes_callback = Ref{Function}(identity) const STACKTRACE_MODULECOLORS = Iterators.Stateful(Iterators.cycle([:magenta, :cyan, :green, :yellow])) const STACKTRACE_FIXEDCOLORS = IdDict(Base => :light_black, Core => :light_black) -function show_full_backtrace(io::IO, trace::Vector; print_linebreaks::Bool) +function show_full_backtrace(io::IO, trace::Vector; print_linebreaks::Bool, prefix=nothing) num_frames = length(trace) ndigits_max = ndigits(num_frames) - println(io, "\nStacktrace:") + println(io) + prefix === nothing || print(io, prefix) + println(io, "Stacktrace:") for (i, (frame, n)) in enumerate(trace) - print_stackframe(io, i, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS) + print_stackframe(io, i, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) if i < num_frames println(io) print_linebreaks && println(io) @@ -655,7 +657,7 @@ end const BIG_STACKTRACE_SIZE = 50 # Arbitrary constant chosen here -function show_reduced_backtrace(io::IO, t::Vector) +function show_reduced_backtrace(io::IO, t::Vector; prefix=nothing) recorded_positions = IdDict{UInt, Vector{Int}}() #= For each frame of hash h, recorded_positions[h] is the list of indices i such that hash(t[i-1]) == h, ie the list of positions in which the @@ -701,7 +703,9 @@ function show_reduced_backtrace(io::IO, t::Vector) try invokelatest(update_stackframes_callback[], displayed_stackframes) catch end - println(io, "\nStacktrace:") + println(io) + prefix === nothing || print(io, prefix) + println(io, "Stacktrace:") ndigits_max = ndigits(length(t)) @@ -709,8 +713,8 @@ function show_reduced_backtrace(io::IO, t::Vector) frame_counter = 1 for i in eachindex(displayed_stackframes) (frame, n) = displayed_stackframes[i] - - print_stackframe(io, frame_counter, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS) + prefix === nothing || print(io, prefix) + print_stackframe(io, frame_counter, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) if i < length(displayed_stackframes) println(io) @@ -721,6 +725,7 @@ function show_reduced_backtrace(io::IO, t::Vector) cycle_length = repeated_cycle[1][2] repetitions = repeated_cycle[1][3] popfirst!(repeated_cycle) + prefix === nothing || print(io, prefix) printstyled(io, "--- the above ", cycle_length, " lines are repeated ", repetitions, " more time", repetitions>1 ? "s" : "", " ---", color = :light_black) @@ -738,7 +743,7 @@ end # Print a stack frame where the module color is determined by looking up the parent module in # `modulecolordict`. If the module does not have a color, yet, a new one can be drawn # from `modulecolorcycler`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolordict, modulecolorcycler) +function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolordict, modulecolorcycler; prefix=nothing) m = Base.parentmodule(frame) modulecolor = if m !== nothing m = parentmodule_before_main(m) @@ -746,7 +751,7 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec else :default end - print_stackframe(io, i, frame, n, ndigits_max, modulecolor) + print_stackframe(io, i, frame, n, ndigits_max, modulecolor; prefix) end # Gets the topmost parent module that isn't Main @@ -761,7 +766,7 @@ end parentmodule_before_main(x) = parentmodule_before_main(parentmodule(x)) # Print a stack frame where the module color is set manually with `modulecolor`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolor) +function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolor; prefix=nothing) file, line = string(frame.file), frame.line # Used by the REPL to make it possible to open @@ -776,6 +781,7 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec digit_align_width = ndigits_max + 2 # frame number + prefix === nothing || print(io, prefix) print(io, " ", lpad("[" * string(i) * "]", digit_align_width)) print(io, " ") @@ -785,6 +791,7 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec end println(io) + prefix === nothing || print(io, prefix) # @ Module path / file : line print_module_path_file(io, modul, file, line; modulecolor, digit_align_width) @@ -813,7 +820,7 @@ function print_module_path_file(io, modul, file, line; modulecolor = :light_blac printstyled(io, basename(file), ":", line; color = :light_black, underline = true) end -function show_backtrace(io::IO, t::Vector) +function show_backtrace(io::IO, t::Vector; prefix=nothing) if haskey(io, :last_shown_line_infos) empty!(io[:last_shown_line_infos]) end @@ -835,12 +842,12 @@ function show_backtrace(io::IO, t::Vector) end if length(filtered) > BIG_STACKTRACE_SIZE - show_reduced_backtrace(IOContext(io, :backtrace => true), filtered) + show_reduced_backtrace(IOContext(io, :backtrace => true), filtered; prefix) return else try invokelatest(update_stackframes_callback[], filtered) catch end # process_backtrace returns a Vector{Tuple{Frame, Int}} - show_full_backtrace(io, filtered; print_linebreaks = stacktrace_linebreaks()) + show_full_backtrace(io, filtered; print_linebreaks = stacktrace_linebreaks(), prefix) end nothing end diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index a2922deebbd31..ab06531152554 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -59,7 +59,6 @@ julia> @test foo("f") == 20 Test Failed at none:1 Expression: foo("f") == 20 Evaluated: 1 == 20 - ERROR: There was an error during testing ``` @@ -230,7 +229,6 @@ Test Passed julia> @test 1 ≈ 0.999999 Test Failed at none:1 Expression: 1 ≈ 0.999999 - ERROR: There was an error during testing ``` You can specify relative and absolute tolerances by setting the `rtol` and `atol` keyword arguments of `isapprox`, respectively, diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index da102e37e7b80..738da88197c1c 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -182,7 +182,7 @@ function Base.show(io::IO, t::Fail) print(io, "\n") if t.backtrace !== nothing # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") end end elseif t.test_type === :test_throws_nothing @@ -199,7 +199,6 @@ function Base.show(io::IO, t::Fail) print(io, "\n Context: ", t.context) end end - println(io) # add some visual space to separate sequential failures end """ @@ -269,7 +268,7 @@ function Base.show(io::IO, t::Error) println(io, " Test threw exception") println(io, " Expression: ", t.orig_expr) # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") elseif t.test_type === :test_unbroken # A test that was expected to fail did not println(io, " Unexpected Pass") @@ -279,7 +278,7 @@ function Base.show(io::IO, t::Error) # we had an error outside of a @test println(io, " Got exception outside of a @test") # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") end end @@ -1169,12 +1168,13 @@ end # but do not terminate. Print a backtrace. function record(ts::DefaultTestSet, t::Union{Fail, Error}; print_result::Bool=TESTSET_PRINT_ENABLE[]) if print_result + println() # add some visual space to separate sequential failures print(ts.description, ": ") # don't print for interrupted tests if !(t isa Error) || t.test_type !== :test_interrupted print(t) if !isa(t, Error) # if not gets printed in the show method - Base.show_backtrace(stdout, scrub_backtrace(backtrace(), ts.file, extract_file(t.source))) + Base.show_backtrace(stdout, scrub_backtrace(backtrace(), ts.file, extract_file(t.source)); prefix=" ") end println() end @@ -1687,7 +1687,6 @@ Test Failed at none:3 Expression: !(iszero(real(logi))) Evaluated: !(iszero(0.0)) Context: logi = 0.0 + 1.5707963267948966im - ERROR: There was an error during testing julia> @testset let logi = log(im), op = !iszero @@ -1699,7 +1698,6 @@ Test Failed at none:3 Evaluated: op(0.0) Context: logi = 0.0 + 1.5707963267948966im op = !iszero - ERROR: There was an error during testing ``` """ diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index e5b9bd4bdd089..cffaa778cf265 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -816,9 +816,9 @@ end """) msg = read(pipeline(ignorestatus(`$(Base.julia_cmd()) --startup-file=no --color=no $runtests`), stderr=devnull), String) msg = win2unix(msg) - regex = r"((?:Tests|Other tests|Testset without source): Test Failed (?:.|\n)*?)\n\nStacktrace:(?:.|\n)*?(?=\n(?:Tests|Other tests))" + regex = r"((?:Tests|Other tests|Testset without source): Test Failed (?:.|\n)*?)\n Stacktrace:(?:.|\n)*?(?=\n(?:Tests|Other tests))" failures = map(eachmatch(regex, msg)) do m - m = match(r"(Tests|Other tests|Testset without source): .*? at (.*?)\n Expression: (.*)(?:.|\n)*\n+Stacktrace:\n((?:.|\n)*)", m.match) + m = match(r"(Tests|Other tests|Testset without source): .*? at (.*?)\n Expression: (.*)(?:.|\n)*\n Stacktrace:\n((?:.|\n)*)", m.match) (; testset = m[1], source = m[2], ex = m[3], stacktrace = m[4]) end @test length(failures) == 8 # 8 failed tests diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index a0a220d18a1fb..ac23fa8a932b2 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -145,6 +145,9 @@ const TEST_TYPE_MAP = Dict( :test_throws_nothing => "@test_throws" ) function get_test_call_str(result) + if result.test_type === :nontest_error + return "Got exception outside of a @test" + end prefix = get(TEST_TYPE_MAP, result.test_type, nothing) prefix === nothing && return error("Unknown test type $(repr(result.test_type))") return prefix == "@test_throws" ? "@test_throws $(result.data) $(result.orig_expr)" : "$prefix $(result.orig_expr)" From 8780aa93ac84198bc7ebcbdd784f26041c99dea5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 28 Apr 2025 11:03:41 -0400 Subject: [PATCH 146/662] [REPLCompletions] use better version of listing module imports for completions (#58218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test for this crashes on my machine, leading me to notice that the implementation of this function was nonsense. Fortunately, we already had a more correct implementation of this function in the REPL docview code that we could borrow from to fix this. While here, also filter out macros, since those are rather strange looking to see appearing in the results. We could alternatively use `Base.is_valid_identifier`, since the main point here is to print functions that are accessible in the module by identifier. ``` julia> @eval Base.MainInclude export broken julia> broken = 2 2 julia> ?(┌ Error: Error in the keymap │ exception = │ UndefVarError: `broken` not defined in `Base.MainInclude` ``` --- stdlib/REPL/src/REPLCompletions.jl | 51 +++++++++++------------------ stdlib/REPL/test/replcompletions.jl | 10 ++++++ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index c56f8b9653cbf..67145cdbd1714 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -729,24 +729,12 @@ function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool end MAX_ANY_METHOD_COMPLETIONS::Int = 10 -function recursive_explore_names!(seen::IdSet, callee_module::Module, initial_module::Module, exploredmodules::IdSet{Module}=IdSet{Module}()) - push!(exploredmodules, callee_module) - for name in names(callee_module; all=true, imported=true) - if !Base.isdeprecated(callee_module, name) && !startswith(string(name), '#') && isdefined(initial_module, name) - func = getfield(callee_module, name) - if !isa(func, Module) - funct = Core.Typeof(func) - push!(seen, funct) - elseif isa(func, Module) && func ∉ exploredmodules - recursive_explore_names!(seen, func, initial_module, exploredmodules) - end - end - end -end -function recursive_explore_names(callee_module::Module, initial_module::Module) - seen = IdSet{Any}() - recursive_explore_names!(seen, callee_module, initial_module) - seen + +function accessible(mod::Module, private::Bool) + bindings = IdSet{Any}(Core.Typeof(getglobal(mod, s)) for s in names(mod; all=private, imported=private, usings=private) + if !Base.isdeprecated(mod, s) && !startswith(string(s), '#') && !startswith(string(s), '@') && isdefined(mod, s)) + delete!(bindings, Module) + return collect(bindings) end function complete_any_methods(ex_org::Expr, callee_module::Module, context_module::Module, moreargs::Bool, shift::Bool) @@ -764,7 +752,7 @@ function complete_any_methods(ex_org::Expr, callee_module::Module, context_modul # semicolon for the ".?(" syntax moreargs && push!(args_ex, Vararg{Any}) - for seen_name in recursive_explore_names(callee_module, callee_module) + for seen_name in accessible(callee_module, callee_module === context_module) complete_methods!(out, seen_name, args_ex, kwargs_ex, MAX_ANY_METHOD_COMPLETIONS, false) end @@ -1273,20 +1261,20 @@ function dict_eval(@nospecialize(e), context_module::Module=Main) end function method_search(partial::AbstractString, context_module::Module, shift::Bool) - rexm = match(r"(\w+\.|)\?\((.*)$", partial) + rexm = match(r"([\w.]+.)?\?\((.*)$", partial) if rexm !== nothing # Get the module scope - if isempty(rexm.captures[1]) - callee_module = context_module - else - modname = Symbol(rexm.captures[1][1:end-1]) - if isdefined(context_module, modname) - callee_module = getfield(context_module, modname) - if !isa(callee_module, Module) - callee_module = context_module + callee_module = context_module + if !isnothing(rexm.captures[1]) + modnames = map(Symbol, split(something(rexm.captures[1]), '.')) + for m in modnames + if isdefined(callee_module, m) + callee_module = getfield(callee_module, m) + if !isa(callee_module, Module) + callee_module = context_module + break + end end - else - callee_module = context_module end end moreargs = !endswith(rexm.captures[2], ')') @@ -1296,7 +1284,8 @@ function method_search(partial::AbstractString, context_module::Module, shift::B end ex_org = Meta.parse(callstr, raise=false, depwarn=false) if isa(ex_org, Expr) - return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:length(rexm.captures[1])+1) .+ rexm.offset, false + pos_q = isnothing(rexm.captures[1]) ? 1 : sizeof(something(rexm.captures[1]))+1 # position after ? + return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:pos_q) .+ rexm.offset, false end end end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 442dff90cc039..5569b93640bd8 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -17,6 +17,16 @@ let ex = module CompletionFoo using Random import Test + # make everything public, so that nothing gets hidden unintentionally from completions + public Test_y, Text_x, type_test, unicode_αΒγ, CompletionFoo2, bar, + foo, @foobar, @barfoo, @error_expanding, + @error_lowering_conditional, @error_throwing, NonStruct, x, + CustomDict, NoLengthDict, test, test1, test2, test3, test4, test5, + test6, test7, test8, test9, test10, test11, a, test!12, kwtest, + kwtest2, kwtest3, kwtest4, kwtest5, named, fmsoebelkv, array, + varfloat, tuple, test_y_array, test_dict, test_customdict, + @teststr_str, @tϵsτstρ_str, @testcmd_cmd, @tϵsτcmδ_cmd, + var"complicated symbol with spaces", WeirdNames, @ignoremacro mutable struct Test_y yy From 0b40de7b642a5114a7fe0a1ac50465545d3a1513 Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:38:40 -0700 Subject: [PATCH 147/662] follow up to: "relax some doctests to better match non-requirement of ordering of Dict and Set" (#58053) I missed a few of these in the initial PR here https://github.com/JuliaLang/julia/pull/57693 hopefully there aren't any more... --- base/abstractdict.jl | 8 ++++---- base/dict.jl | 4 ++-- base/set.jl | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/abstractdict.jl b/base/abstractdict.jl index 786720bb6359f..31c00c5b4f8d5 100644 --- a/base/abstractdict.jl +++ b/base/abstractdict.jl @@ -92,7 +92,7 @@ But `keys(a)`, `values(a)` and `pairs(a)` all iterate `a` and return the elements in the same order. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+'\\S'.*\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -118,7 +118,7 @@ But `keys(a)`, `values(a)` and `pairs(a)` all iterate `a` and return the elements in the same order. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+(\\s+=>\\s+\\d)?\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -332,7 +332,7 @@ value for that key will be the value it has in the last collection listed. See also [`mergewith`](@ref) for custom handling of values with the same key. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\S+\$"m julia> a = Dict("foo" => 0.0, "bar" => 42.0) Dict{String, Float64} with 2 entries: "bar" => 42.0 @@ -377,7 +377,7 @@ Method `merge(combine::Union{Function,Type}, args...)` as an alias of `mergewith` requires Julia 1.5 or later. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\S+\$"m julia> a = Dict("foo" => 0.0, "bar" => 42.0) Dict{String, Float64} with 2 entries: "bar" => 42.0 diff --git a/base/dict.jl b/base/dict.jl index 1272229c714da..e059fde102bab 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -532,7 +532,7 @@ end Determine whether a collection has a mapping for a given `key`. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\d\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -554,7 +554,7 @@ in(key, v::KeySet{<:Any, <:Dict}) = (ht_keyindex(v.dict, key) >= 0) Return the key matching argument `key` if one exists in `collection`, otherwise return `default`. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\d\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 diff --git a/base/set.jl b/base/set.jl index 0427550c6da54..38287efe28bee 100644 --- a/base/set.jl +++ b/base/set.jl @@ -717,7 +717,7 @@ If `count` is specified, then replace at most `count` occurrences in total. See also [`replace`](@ref replace(A, old_new::Pair...)). # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\d\$"m julia> replace!([1, 2, 1, 3], 1=>0, 2=>4, count=2) 4-element Vector{Int64}: 0 From 59e9ada533850a0c15850644c1936d253aa2d8e7 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 28 Apr 2025 14:19:53 -0700 Subject: [PATCH 148/662] Add builtin functions Core._import, Core._using; implement import/using logic in Julia (#57965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, `import` and `using` statements are compiled into calls to `jl_toplevel_eval` with the Expr as an argument. It can be useful in an interactive environment to do import/using programmatically, for which `eval(Expr(:import, ...))` or a macro is the only option at the moment (see for example `InteractiveUtils.@activate`). This PR adds two functions, `_module_import ` and `_module_using`, with these signatures, and removes `Expr(:import/:using, ...)` from the lowered IR: ``` _module_import(explicit::Bool, to::Module, ctx::Union{Expr, Nothing}, names::Expr...) _module_using(to::Module, from::Expr{Symbol}) ``` ## Lowering example `import` statements and `using X:` statements are lowered to `_module_import` like so: ``` import A => _module_import(true, Main, nothing, Expr(:., :A)) import A.b => _module_import(true, Main, nothing, Expr(:., :A, :b)) import A.b as c => _module_import(true, Main, nothing, Expr(:as, Expr(:., :A, :b), :c)) import A.B: C.d, e => _module_import(true, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) import A.B: C.d as e => _module_import(true, Main, Expr(:., :A, :B), Expr(:as, Expr(:., :C, :d), :e)) using A.B: C.d, e => _module_import(false, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) ``` Plain `using` statements become `_module_using`: ``` using A.B => _module_using(Main, Expr(:., :A, :B)) ``` Multiple comma-separated `using` or `import` paths are lowered to multiple calls to the appropriate builtin: ``` julia> Meta.@lower using A.B, C :($(Expr(:thunk, CodeInfo( 1 ─ builtin Core._module_using(Main, $(QuoteNode(:($(Expr(:., :A, :B)))))) │ builtin Core._module_using(Main, $(QuoteNode(:($(Expr(:., :C)))))) │ $(Expr(:latestworld)) └── return nothing )))) ``` --- Compiler/src/Compiler.jl | 2 +- base/Base_compiler.jl | 4 + base/boot.jl | 48 +++++++ base/c.jl | 2 +- base/checked.jl | 5 +- base/docs/basedocs.jl | 19 +++ base/iterators.jl | 12 +- base/loading.jl | 2 +- base/module.jl | 143 ++++++++++++++++++++ base/ordering.jl | 2 +- src/ast.c | 4 - src/builtin_proto.h | 2 + src/builtins.c | 39 ++++++ src/jl_exported_funcs.inc | 1 - src/julia-syntax.scm | 58 ++++++++- src/julia.h | 6 +- src/julia_internal.h | 2 - src/module.c | 39 +++--- src/toplevel.c | 268 +------------------------------------- stdlib/REPL/src/REPL.jl | 33 +++-- stdlib/REPL/test/repl.jl | 54 ++++---- sysimage.mk | 1 + test/loading.jl | 6 + test/worlds.jl | 9 ++ 24 files changed, 409 insertions(+), 352 deletions(-) create mode 100644 base/module.jl diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index a5a5f1b09a1b5..9c3581677132e 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -65,7 +65,7 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout, uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal, _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized, - partialstruct_init_undefs, fieldcount_noerror + partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using using Base.Order import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!, diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index e22a7d980f06c..28c07379c54d9 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -2,6 +2,9 @@ module Base +Core._import(Base, Core, :_eval_import, :_eval_import, true) +Core._import(Base, Core, :_eval_using, :_eval_using, true) + using .Core.Intrinsics, .Core.IR # to start, we're going to use a very simple definition of `include` @@ -340,6 +343,7 @@ include("ordering.jl") using .Order include("coreir.jl") +include("module.jl") include("invalidation.jl") BUILDROOT::String = "" diff --git a/base/boot.jl b/base/boot.jl index f9208c3874229..566839d430e93 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -695,6 +695,54 @@ function Symbol(a::Array{UInt8, 1}) end Symbol(s::Symbol) = s +# Minimal implementations of using/import for bootstrapping (supports only +# `import .M: a, b, c, ...`, little error checking) +let + fail() = throw(ArgumentError("unsupported import/using while bootstrapping")) + length(a::Array{T, 1}) where {T} = getfield(getfield(a, :size), 1) + function getindex(A::Array, i::Int) + Intrinsics.ult_int(Intrinsics.bitcast(UInt, Intrinsics.sub_int(i, 1)), Intrinsics.bitcast(UInt, length(A))) || fail() + memoryrefget(memoryrefnew(getfield(A, :ref), i, false), :not_atomic, false) + end + x == y = Intrinsics.eq_int(x, y) + x + y = Intrinsics.add_int(x, y) + x <= y = Intrinsics.sle_int(x, y) + + global function _eval_import(explicit::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + from isa Expr || fail() + if length(from.args) == 2 && getindex(from.args, 1) === :. + from = getglobal(to, getindex(from.args, 2)) + elseif length(from.args) == 1 && getindex(from.args, 1) === :Core + from = Core + elseif length(from.args) == 1 && getindex(from.args, 1) === :Base + from = Main.Base + else + fail() + end + from isa Module || fail() + i = 1 + while i <= nfields(paths) + a = getfield(paths, i).args + length(a) == 1 || fail() + s = getindex(a, 1) + Core._import(to, from, s, s, explicit) + i += 1 + end + end + + global function _eval_using(to::Module, path::Expr) + getindex(path.args, 1) === :. || fail() + from = getglobal(to, getindex(path.args, 2)) + i = 3 + while i <= length(path.args) + from = getfield(from, getindex(path.args, i)) + i += 1 + end + from isa Module || fail() + Core._using(to, from) + end +end + # module providing the IR object model module IR diff --git a/base/c.jl b/base/c.jl index 78c48f267ca71..69ea3adf24404 100644 --- a/base/c.jl +++ b/base/c.jl @@ -2,7 +2,7 @@ # definitions related to C interface -import Core.Intrinsics: cglobal +import .Intrinsics: cglobal """ cglobal((symbol, library) [, type=Cvoid]) diff --git a/base/checked.jl b/base/checked.jl index b374d34830280..39d487cba6e37 100644 --- a/base/checked.jl +++ b/base/checked.jl @@ -16,12 +16,13 @@ export checked_neg, checked_abs, checked_add, checked_sub, checked_mul, checked_div, checked_rem, checked_fld, checked_mod, checked_cld, checked_pow, checked_length, add_with_overflow, sub_with_overflow, mul_with_overflow -import Core.Intrinsics: +import Core: Intrinsics +import .Intrinsics: checked_sadd_int, checked_ssub_int, checked_smul_int, checked_sdiv_int, checked_srem_int, checked_uadd_int, checked_usub_int, checked_umul_int, checked_udiv_int, checked_urem_int -import ..no_op_err, ..@inline, ..@noinline, ..checked_length +import Base: no_op_err, @inline, @noinline, checked_length # define promotion behavior for checked operations checked_add(x::Integer, y::Integer) = checked_add(promote(x,y)...) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index cf8a087b76489..60770c42bae7d 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -2753,6 +2753,25 @@ See also [`setpropertyonce!`](@ref Base.setpropertyonce!) and [`setglobal!`](@re """ setglobalonce! +""" + _import(to::Module, from::Module, asname::Symbol, [sym::Symbol, imported::Bool]) + +With all five arguments, imports `sym` from module `from` into `to` with name +`asname`. `imported` is true for bindings created with `import` (set it to +false for `using A: ...`). + +With only the first three arguments, creates a binding for the module `from` +with name `asname` in `to`. +""" +Core._import + +""" + _using(to::Module, from::Module) + +Add `from` to the usings list of `to`. +""" +Core._using + """ typeof(x) diff --git a/base/iterators.jl b/base/iterators.jl index c7450781c4928..4b956073f0c04 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -6,7 +6,7 @@ Methods for working with Iterators. baremodule Iterators # small dance to make this work from Base or Intrinsics -import ..@__MODULE__, ..parentmodule +import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) using .Base: @inline, Pair, Pairs, AbstractDict, IndexLinear, IndexStyle, AbstractVector, Vector, @@ -17,14 +17,14 @@ using .Base: any, _counttuple, eachindex, ntuple, zero, prod, reduce, in, firstindex, lastindex, tail, fieldtypes, min, max, minimum, zero, oneunit, promote, promote_shape, LazyString, afoldl -using Core +using .Core using Core: @doc -using .Base: - cld, fld, resize!, IndexCartesian -using .Base.Checked: checked_mul +using Base: + cld, fld, resize!, IndexCartesian, Checked +using .Checked: checked_mul -import .Base: +import Base: first, last, isempty, length, size, axes, ndims, eltype, IteratorSize, IteratorEltype, promote_typejoin, diff --git a/base/loading.jl b/base/loading.jl index 9de9dd07e5428..f6b501621b895 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2332,7 +2332,7 @@ function require(into::Module, mod::Symbol) if world == typemax(UInt) world = get_world_counter() end - return invoke_in_world(world, __require, into, mod) + return Compiler.@zone "LOAD_Require" invoke_in_world(world, __require, into, mod) end function check_for_hint(into, mod) diff --git a/base/module.jl b/base/module.jl new file mode 100644 index 0000000000000..3e95d9882339c --- /dev/null +++ b/base/module.jl @@ -0,0 +1,143 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Full-featured versions of _eval_import and _eval_using + +for m in methods(_eval_import) + delete_method(m) +end +for m in methods(_eval_using) + delete_method(m) +end + +function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, keyword::String) + isempty(path.args) && error("malformed import statement") + + i::Int = 1 + function next!() + i <= length(path.args) || error("invalid module path") + v = path.args[i] + i += 1 + v isa Symbol || throw(TypeError(Symbol(keyword), "", Symbol, v)) + v + end + v = next!() + m = nothing + + if from !== nothing + m = from + elseif v !== :. + # `A.B`: call the loader to obtain the root A in the current environment. + if v === :Core + m = Core + elseif v === :Base + m = Base + else + m = require(at, v) + m isa Module || error("failed to load module $v") + end + i > lastindex(path.args) && return m, nothing + v = next!() + else + # `.A.B.C`: strip off leading dots by following parent links + m = at + while (v = next!()) === :. + m = parentmodule(m) + end + end + + while true + v === :. && error("invalid $keyword path: \".\" in identifier path") + i > lastindex(path.args) && break + m = getglobal(m, v) + m isa Module || error("invalid $keyword path: \"$v\" does not name a module") + v = next!() + end + m, v +end + +function eval_import_path_all(at::Module, path::Expr, keyword::String) + m, v = eval_import_path(at, nothing, path, keyword) + if v !== nothing + m = getglobal(m, v) + m isa Module || error("invalid $keyword path: \"$v\" does not name a module") + end + m +end + +function check_macro_rename(from::Symbol, to::Symbol, keyword::String) + c1(sym) = bitcast(Char, UInt32(unsafe_load(unsafe_convert(Ptr{UInt8}, sym))) << 24) + from_c, to_c = c1(from), c1(to) + if from_c == '@' && to_c != '@' + error("cannot rename macro \"$from\" to non-macro \"$to\" in \"$keyword\"") + end + if from_c != '@' && to_c == '@' + error("cannot rename non-macro \"$from\" to macro \"$to\" in \"$keyword\"") + end +end + +""" + _eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + +Evaluate the import paths, calling `Core._import` for each name to be imported. +`imported` imports are created with `import`, `using A: x` sets this to false. +The `from` is the part of the import path before the `:`. This is the lowered +form of `import`, `import ...:`, and `using ...:`. + +``` +import A => _eval_import(true, Main, nothing, Expr(:., :A)) +import A.b => _eval_import(true, Main, nothing, Expr(:., :A, :b)) +import A.b as c => _eval_import(true, Main, nothing, Expr(:as, Expr(:., :A, :b), :c)) +import A.B: C.d, e => _eval_import(true, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) +import A.B: C.d as e => _eval_import(true, Main, Expr(:., :A, :B), Expr(:as, Expr(:., :C, :d), :e)) +using A.B: C.d, e => _eval_import(false, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) + +See also [`_import`](@ref Core._import). +``` +""" +function _eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + keyword = imported ? "import" : "using" + fail() = error("malformed \"$keyword\" statement") + from = from !== nothing ? eval_import_path_all(to, from, keyword) : nothing + + for path in paths + path isa Expr || fail() + asname = nothing + if path.head === :as && length(path.args) == 2 + path, asname = path.args + elseif path.head !== :. + fail() + end + old_from = from + from, name = eval_import_path(to, from, path, keyword) + + if name !== nothing + asname = asname === nothing ? name : asname + check_macro_rename(name, asname, keyword) + Core._import(to, from, asname, name, imported) + else + Core._import(to, from, asname === nothing ? nameof(from) : asname) + end + end +end + +""" + _eval_using(to::Module, path::Expr) + +Evaluate the import path to a module and call [`Core._using`](@ref) on it, +making its exports available to the `to` module; this is the lowered form of +`using A`. + +``` +using A.B => _module_using(Main, Expr(:., :A, :B)) +``` + +See also [`_using`](@ref Core._using). +""" +function _eval_using(to::Module, path::Expr) + from = eval_import_path_all(to, path, "using") + Core._using(to, from) + is_package = length(path.args) == 1 && path.args[1] !== :. + if to == Main && is_package + Core._import(to, from, nameof(from)) + end +end diff --git a/base/ordering.jl b/base/ordering.jl index 19e8a1cf18109..f2ddd20ab09f0 100644 --- a/base/ordering.jl +++ b/base/ordering.jl @@ -3,7 +3,7 @@ module Order -import ..@__MODULE__, ..parentmodule +import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) import .Base: AbstractVector, @propagate_inbounds, isless, identity, getindex, reverse, diff --git a/src/ast.c b/src/ast.c index b7edaac43f134..37621d6778b2f 100644 --- a/src/ast.c +++ b/src/ast.c @@ -30,7 +30,6 @@ JL_DLLEXPORT jl_sym_t *jl_module_sym; JL_DLLEXPORT jl_sym_t *jl_slot_sym; JL_DLLEXPORT jl_sym_t *jl_export_sym; JL_DLLEXPORT jl_sym_t *jl_public_sym; -JL_DLLEXPORT jl_sym_t *jl_import_sym; JL_DLLEXPORT jl_sym_t *jl_toplevel_sym; JL_DLLEXPORT jl_sym_t *jl_quote_sym; JL_DLLEXPORT jl_sym_t *jl_line_sym; @@ -51,7 +50,6 @@ JL_DLLEXPORT jl_sym_t *jl_pop_exception_sym; JL_DLLEXPORT jl_sym_t *jl_exc_sym; JL_DLLEXPORT jl_sym_t *jl_error_sym; JL_DLLEXPORT jl_sym_t *jl_new_sym; -JL_DLLEXPORT jl_sym_t *jl_using_sym; JL_DLLEXPORT jl_sym_t *jl_splatnew_sym; JL_DLLEXPORT jl_sym_t *jl_block_sym; JL_DLLEXPORT jl_sym_t *jl_new_opaque_closure_sym; @@ -349,8 +347,6 @@ void jl_init_common_symbols(void) jl_module_sym = jl_symbol("module"); jl_export_sym = jl_symbol("export"); jl_public_sym = jl_symbol("public"); - jl_import_sym = jl_symbol("import"); - jl_using_sym = jl_symbol("using"); jl_assign_sym = jl_symbol("="); jl_method_sym = jl_symbol("method"); jl_exc_sym = jl_symbol("the_exception"); diff --git a/src/builtin_proto.h b/src/builtin_proto.h index 586d948f722c1..607106f35bac0 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -16,12 +16,14 @@ extern "C" { XX(_defaultctors,"_defaultctors") \ XX(_equiv_typedef,"_equiv_typedef") \ XX(_expr,"_expr") \ + XX(_import, "_import") \ XX(_primitivetype,"_primitivetype") \ XX(_setsuper,"_setsuper!") \ XX(_structtype,"_structtype") \ XX(_svec_ref,"_svec_ref") \ XX(_typebody,"_typebody!") \ XX(_typevar,"_typevar") \ + XX(_using, "_using") \ XX(applicable,"applicable") \ XX(apply_type,"apply_type") \ XX(compilerbarrier,"compilerbarrier") \ diff --git a/src/builtins.c b/src/builtins.c index e3a0380182e15..36b5d79ec0851 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -3,6 +3,8 @@ /* implementations of built-in functions */ +#include "dtypes.h" +#include "julia_atomics.h" #include "platform.h" #include @@ -1504,7 +1506,44 @@ JL_CALLABLE(jl_f_setglobalonce) return old == NULL ? jl_true : jl_false; } +// import, using -------------------------------------------------------------- +// Import binding `from.sym` as `asname` into `to`: +// _import(to::Module, from::Module, asname::Symbol, sym::Symbol, imported::Bool) +// +// Create const binding to `mod` in `to` with name `asname`: +// _import(to::Module, mod::Module, asname::Symbol) +JL_CALLABLE(jl_f__import) +{ + JL_NARGS(_import, 3, 5); + JL_TYPECHK(_import, module, args[0]); + JL_TYPECHK(_import, module, args[1]); + JL_TYPECHK(_import, symbol, args[2]); + if (nargs == 3) { + jl_import_module(jl_current_task, (jl_module_t *)args[0], (jl_module_t *)args[1], + (jl_sym_t *)args[2]); + } + else if (nargs == 4) { + jl_too_few_args("_import", 5); + } + else if (nargs == 5) { + JL_TYPECHK(_import, symbol, args[3]); + JL_TYPECHK(_import, bool, args[4]); + jl_module_import(jl_current_task, (jl_module_t *)args[0], (jl_module_t *)args[1], + (jl_sym_t *)args[2], (jl_sym_t *)args[3], args[4] == jl_true); + } + return jl_nothing; +} + +// _using(to::Module, from::Module) +JL_CALLABLE(jl_f__using) +{ + JL_NARGS(_using, 2, 2); + JL_TYPECHK(_using, module, args[0]); + JL_TYPECHK(_using, module, args[1]); + jl_module_using((jl_module_t *)args[0], (jl_module_t *)args[1]); + return jl_nothing; +} // apply_type ----------------------------------------------------------------- diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6cb9ab630a4b1..64dcb2278d240 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -311,7 +311,6 @@ XX(jl_module_parent) \ XX(jl_module_getloc) \ XX(jl_module_public_p) \ - XX(jl_module_use) \ XX(jl_module_using) \ XX(jl_module_usings) \ XX(jl_module_uuid) \ diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 4fa6dc2e14a70..3b05f19e5456e 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2521,7 +2521,7 @@ `(= ,lhs ,rhs))) (define (expand-forms e) - (if (or (atom? e) (memq (car e) '(quote inert top core globalref module toplevel ssavalue null true false meta using import export public thismodule toplevel-only))) + (if (or (atom? e) (memq (car e) '(quote inert top core globalref module toplevel ssavalue null true false meta export public thismodule toplevel-only))) e (let ((ex (get expand-table (car e) #f))) (if ex @@ -2541,6 +2541,20 @@ (define (something e) (find (lambda (x) (not (equal? x '(null)))) e)) +(define (check-import-paths what e) + (define (check-dot-path e) + (and (list? e) (eq? (car e) '|.|) (every symbol? (cdr e)))) + (define (check-path e) + (and (pair? e) + (or (check-dot-path e) + (and (eq? (car e) 'as) + (check-dot-path (cadr e)) (symbol? (caddr e)))))) + (unless (and (list? e) + (or (every check-path e) + (and (list? (car e)) (eq? (caar e) ':) + (every check-path (cdar e))))) + (error (string "malformed \"" what "\" statement")))) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -2939,6 +2953,38 @@ (lambda (e) (set! *current-desugar-loc* e) e) + + ;; We insert (latestworld) after every call to _eval_import or _eval_using + ;; to avoid having to do it in eval_import_path (#57316) + 'import + (lambda (e) + (check-import-paths "import" (cdr e)) + `(block + (toplevel-only import) + ,.(if (eq? (caadr e) ':) + `((call (top _eval_import) (true) (thismodule) + ,.(map (lambda (x) `(inert ,x)) (cdadr e))) + (latestworld)) + (map (lambda (x) + `(block + (call (top _eval_import) (true) (thismodule) (null) (inert ,x)) + (latestworld))) + (cdr e))))) + + 'using + (lambda (e) + (check-import-paths "using" (cdr e)) + `(block + (toplevel-only using) + ,.(if (eq? (caadr e) ':) + `((call (top _eval_import) (false) (thismodule) + ,.(map (lambda (x) `(inert ,x)) (cdadr e))) + (latestworld)) + (map (lambda (x) + `(block + (call (top _eval_using) (thismodule) (inert ,x)) + (latestworld))) + (cdr e))))) )) (define (has-return? e) @@ -3183,7 +3229,7 @@ (check-valid-name (cadr e)) ;; remove local decls '(null)) - ((memq (car e) '(using import export public)) + ((memq (car e) '(export public)) ;; no scope resolution - identifiers remain raw symbols e) ((eq? (car e) 'require-existing-local) @@ -3815,7 +3861,7 @@ f(x) = yt(x) thunk with-static-parameters toplevel-only global globalref global-if-global assign-const-if-global isglobal thismodule const atomic null true false ssavalue isdefined toplevel module lambda - error gc_preserve_begin gc_preserve_end import using export public inline noinline purity))) + error gc_preserve_begin gc_preserve_end export public inline noinline purity))) (define (local-in? s lam (tab #f)) (or (and tab (has? tab s)) @@ -4047,7 +4093,7 @@ f(x) = yt(x) ((atom? e) e) (else (case (car e) - ((quote top core global globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e) + ((quote top core global globalref thismodule lineinfo line break inert module toplevel null true false meta) e) ((toplevel-only) ;; hack to avoid generating a (method x) expr for struct types (if (eq? (cadr e) 'struct) @@ -5045,7 +5091,7 @@ f(x) = yt(x) '(null)) ;; other top level expressions - ((import using export public latestworld) + ((export public latestworld) (check-top-level e) (if (not (eq? (car e) 'latestworld)) (emit e)) @@ -5288,7 +5334,7 @@ f(x) = yt(x) ((nospecialize-meta? e) ;; convert nospecialize vars to slot numbers `(meta ,(cadr e) ,@(map renumber-stuff (cddr e)))) - ((or (atom? e) (quoted? e) (memq (car e) '(using import export public global toplevel))) + ((or (atom? e) (quoted? e) (memq (car e) '(export public global toplevel))) e) ((ssavalue? e) (let ((idx (get ssavalue-table (cadr e) #f))) diff --git a/src/julia.h b/src/julia.h index 195c23f20662f..faad02c0aa50c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2111,11 +2111,9 @@ JL_DLLEXPORT jl_value_t *jl_checked_modify(jl_binding_t *b, jl_module_t *mod, jl JL_DLLEXPORT jl_value_t *jl_checked_assignonce(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind); +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici); +JL_DLLEXPORT void jl_import_module(jl_task_t *ct, jl_module_t *m, jl_module_t *import, jl_sym_t *asname); JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from); -JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); -JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world); JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var); diff --git a/src/julia_internal.h b/src/julia_internal.h index 5d418ceb2051c..b588f84d05319 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1869,7 +1869,6 @@ extern JL_DLLEXPORT jl_sym_t *jl_module_sym; extern JL_DLLEXPORT jl_sym_t *jl_slot_sym; extern JL_DLLEXPORT jl_sym_t *jl_export_sym; extern JL_DLLEXPORT jl_sym_t *jl_public_sym; -extern JL_DLLEXPORT jl_sym_t *jl_import_sym; extern JL_DLLEXPORT jl_sym_t *jl_toplevel_sym; extern JL_DLLEXPORT jl_sym_t *jl_quote_sym; extern JL_DLLEXPORT jl_sym_t *jl_line_sym; @@ -1891,7 +1890,6 @@ extern JL_DLLEXPORT jl_sym_t *jl_pop_exception_sym; extern JL_DLLEXPORT jl_sym_t *jl_exc_sym; extern JL_DLLEXPORT jl_sym_t *jl_error_sym; extern JL_DLLEXPORT jl_sym_t *jl_new_sym; -extern JL_DLLEXPORT jl_sym_t *jl_using_sym; extern JL_DLLEXPORT jl_sym_t *jl_splatnew_sym; extern JL_DLLEXPORT jl_sym_t *jl_block_sym; extern JL_DLLEXPORT jl_sym_t *jl_new_opaque_closure_sym; diff --git a/src/module.c b/src/module.c index 4c685ca574523..108f0f3c17275 100644 --- a/src/module.c +++ b/src/module.c @@ -1192,7 +1192,7 @@ static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_ } // NOTE: we use explici since explicit is a C++ keyword -static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) { check_safe_import_from(from); jl_binding_t *b = jl_get_binding(from, s); @@ -1269,24 +1269,27 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl JL_UNLOCK(&world_counter_lock); } -JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) +JL_DLLEXPORT void jl_import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) { - module_import_(ct, to, from, s, s, 1); -} - -JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) -{ - module_import_(ct, to, from, asname, s, 1); -} - -JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) -{ - module_import_(ct, to, from, s, s, 0); -} - -JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) -{ - module_import_(ct, to, from, asname, s, 0); + assert(m); + jl_sym_t *name = asname ? asname : import->name; + // TODO: this is a bit race-y with what error message we might print + jl_binding_t *b = jl_get_module_binding(m, name, 1); + size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (!jl_bkind_is_some_implicit(kind) && kind != PARTITION_KIND_DECLARED) { + // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. + jl_walk_binding_inplace(&b, &bpart, world); + if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { + // Already declared (e.g. on another thread) or imported. + if (bpart->restriction == (jl_value_t*)import) + return; + } + jl_errorf("importing %s into %s conflicts with an existing global", + jl_symbol_name(name), jl_symbol_name(m->name)); + } + jl_declare_constant_val2(b, m, name, (jl_value_t*)import, PARTITION_KIND_CONST_IMPORT); } void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) diff --git a/src/toplevel.c b/src/toplevel.c index 64a559ac774e6..2e0370aee8a5d 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -478,110 +478,10 @@ static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int *forced_compile = jl_has_meta(body, jl_force_compile_sym); } -extern size_t jl_require_world; -static jl_module_t *call_require(jl_task_t *ct, jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_ROOTED -{ - JL_TIMING(LOAD_IMAGE, LOAD_Require); - jl_timing_printf(JL_TIMING_DEFAULT_BLOCK, "%s", jl_symbol_name(var)); - - int build_mode = jl_options.incremental && jl_generating_output(); - jl_module_t *m = NULL; - static jl_value_t *require_func = NULL; - if (require_func == NULL && jl_base_module != NULL) { - require_func = jl_get_global(jl_base_module, jl_symbol("require")); - } - if (require_func != NULL) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - if (build_mode && jl_require_world < ct->world_age) - ct->world_age = jl_require_world; - jl_value_t *reqargs[3]; - reqargs[0] = require_func; - reqargs[1] = (jl_value_t*)mod; - reqargs[2] = (jl_value_t*)var; - m = (jl_module_t*)jl_apply(reqargs, 3); - } - if (m == NULL || !jl_is_module(m)) { - jl_errorf("failed to load module %s", jl_symbol_name(var)); - } - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - return m; -} - -// either: -// - sets *name and returns the module to import *name from -// - sets *name to NULL and returns a module to import -// also updates world_age -static jl_module_t *eval_import_path(jl_task_t *ct, jl_module_t *where, jl_module_t *from JL_PROPAGATES_ROOT, - jl_array_t *args, jl_sym_t **name, const char *keyword) JL_GLOBALLY_ROOTED -{ - if (jl_array_nrows(args) == 0) - jl_errorf("malformed \"%s\" statement", keyword); - jl_sym_t *var = (jl_sym_t*)jl_array_ptr_ref(args, 0); - size_t i = 1; - jl_module_t *m = NULL; - *name = NULL; - if (!jl_is_symbol(var)) - jl_type_error(keyword, (jl_value_t*)jl_symbol_type, (jl_value_t*)var); - - if (from != NULL) { - m = from; - i = 0; - } - else if (var != jl_dot_sym) { - // `A.B`: call the loader to obtain the root A in the current environment. - if (jl_core_module && var == jl_core_module->name) { - m = jl_core_module; - } - else if (jl_base_module && var == jl_base_module->name) { - m = jl_base_module; - } - else { - m = call_require(ct, where, var); - } - if (i == jl_array_nrows(args)) - return m; - } - else { - // `.A.B.C`: strip off leading dots by following parent links - m = where; - while (1) { - if (i >= jl_array_nrows(args)) - jl_error("invalid module path"); - var = (jl_sym_t*)jl_array_ptr_ref(args, i); - if (var != jl_dot_sym) - break; - i++; - assert(m); - m = m->parent; - } - } - - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - - while (1) { - var = (jl_sym_t*)jl_array_ptr_ref(args, i); - if (!jl_is_symbol(var)) - jl_type_error(keyword, (jl_value_t*)jl_symbol_type, (jl_value_t*)var); - if (var == jl_dot_sym) - jl_errorf("invalid %s path: \".\" in identifier path", keyword); - if (i == jl_array_nrows(args)-1) - break; - m = (jl_module_t*)jl_eval_global_var(m, var); - JL_GC_PROMISE_ROOTED(m); - if (!jl_is_module(m)) - jl_errorf("invalid %s path: \"%s\" does not name a module", keyword, jl_symbol_name(var)); - i++; - } - *name = var; - return m; -} - int jl_is_toplevel_only_expr(jl_value_t *e) JL_NOTSAFEPOINT { return jl_is_expr(e) && (((jl_expr_t*)e)->head == jl_module_sym || - ((jl_expr_t*)e)->head == jl_import_sym || - ((jl_expr_t*)e)->head == jl_using_sym || ((jl_expr_t*)e)->head == jl_export_sym || ((jl_expr_t*)e)->head == jl_public_sym || ((jl_expr_t*)e)->head == jl_thunk_sym || @@ -599,10 +499,9 @@ int jl_needs_lowering(jl_value_t *e) JL_NOTSAFEPOINT return 0; jl_expr_t *ex = (jl_expr_t*)e; jl_sym_t *head = ex->head; - if (head == jl_module_sym || head == jl_import_sym || head == jl_using_sym || - head == jl_export_sym || head == jl_public_sym || head == jl_thunk_sym || - head == jl_toplevel_sym || head == jl_error_sym || head == jl_incomplete_sym || - head == jl_method_sym) { + if (head == jl_module_sym || head == jl_export_sym || head == jl_public_sym || + head == jl_thunk_sym || head == jl_toplevel_sym || head == jl_error_sym || + head == jl_incomplete_sym || head == jl_method_sym) { return 0; } if (head == jl_global_sym || head == jl_const_sym) { @@ -642,62 +541,6 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_instance_for_thunk(jl_code_info_t * return mi; } -static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) -{ - assert(m); - jl_sym_t *name = asname ? asname : import->name; - // TODO: this is a bit race-y with what error message we might print - jl_binding_t *b = jl_get_module_binding(m, name, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (!jl_bkind_is_some_implicit(kind) && kind != PARTITION_KIND_DECLARED) { - // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. - jl_walk_binding_inplace(&b, &bpart, ct->world_age); - if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { - // Already declared (e.g. on another thread) or imported. - if (bpart->restriction == (jl_value_t*)import) - return; - } - jl_errorf("importing %s into %s conflicts with an existing global", - jl_symbol_name(name), jl_symbol_name(m->name)); - } - jl_declare_constant_val2(b, m, name, (jl_value_t*)import, PARTITION_KIND_CONST_IMPORT); -} - -// in `import A.B: x, y, ...`, evaluate the `A.B` part if it exists -static jl_module_t *eval_import_from(jl_task_t *ct, jl_module_t *m JL_PROPAGATES_ROOT, jl_expr_t *ex, const char *keyword) -{ - if (jl_expr_nargs(ex) == 1 && jl_is_expr(jl_exprarg(ex, 0))) { - jl_expr_t *fr = (jl_expr_t*)jl_exprarg(ex, 0); - if (fr->head == jl_colon_sym) { - if (jl_expr_nargs(fr) > 0 && jl_is_expr(jl_exprarg(fr, 0))) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(fr, 0); - if (((jl_expr_t*)path)->head == jl_dot_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_path(ct, m, NULL, path->args, &name, keyword); - if (name != NULL) { - from = (jl_module_t*)jl_eval_global_var(from, name); - if (!from || !jl_is_module(from)) - jl_errorf("invalid %s path: \"%s\" does not name a module", keyword, jl_symbol_name(name)); - } - return from; - } - } - jl_errorf("malformed \"%s:\" statement", keyword); - } - } - return NULL; -} - -static void check_macro_rename(jl_sym_t *from, jl_sym_t *to, const char *keyword) -{ - char *n1 = jl_symbol_name(from), *n2 = jl_symbol_name(to); - if (n1[0] == '@' && n2[0] != '@') - jl_errorf("cannot rename macro \"%s\" to non-macro \"%s\" in \"%s\"", n1, n2, keyword); - if (n1[0] != '@' && n2[0] == '@') - jl_errorf("cannot rename non-macro \"%s\" to macro \"%s\" in \"%s\"", n1, n2, keyword); -} - // Eval `throw(ErrorException(msg)))` in module `m`. // Used in `jl_toplevel_eval_flex` instead of `jl_throw` so that the error // location in julia code gets into the backtrace. @@ -818,111 +661,6 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_GC_POP(); return val; } - else if (head == jl_using_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(ct, m, ex, "using"); - size_t i = 0; - if (from) { - i = 1; - ex = (jl_expr_t*)jl_exprarg(ex, 0); - } - for (; i < jl_expr_nargs(ex); i++) { - jl_value_t *a = jl_exprarg(ex, i); - if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "using"); - if (from) { - // `using A: B` and `using A: B.c` syntax - jl_module_use(ct, m, import, name); - } - else { - jl_module_t *u = import; - if (name != NULL) - u = (jl_module_t*)jl_eval_global_var(import, name); - if (!jl_is_module(u)) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "invalid using path: \"%s\" does not name a module", - jl_symbol_name(name)); - // `using A` and `using A.B` syntax - jl_module_using(m, u); - if (m == jl_main_module && name == NULL) { - // TODO: for now, `using A` in Main also creates an explicit binding for `A` - // This will possibly be extended to all modules. - import_module(ct, m, u, NULL); - } - } - continue; - } - else if (from && jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_as_sym && jl_expr_nargs(a) == 2 && - jl_is_expr(jl_exprarg(a, 0)) && ((jl_expr_t*)jl_exprarg(a, 0))->head == jl_dot_sym) { - jl_sym_t *asname = (jl_sym_t*)jl_exprarg(a, 1); - if (jl_is_symbol(asname)) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "using"); - assert(name); - check_macro_rename(name, asname, "using"); - // `using A: B as C` syntax - jl_module_use_as(ct, m, import, name, asname); - continue; - } - } - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "syntax: malformed \"using\" statement"); - } - JL_GC_POP(); - ct->world_age = last_age; - return jl_nothing; - } - else if (head == jl_import_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(ct, m, ex, "import"); - size_t i = 0; - if (from) { - i = 1; - ex = (jl_expr_t*)jl_exprarg(ex, 0); - } - for (; i < jl_expr_nargs(ex); i++) { - jl_value_t *a = jl_exprarg(ex, i); - if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "import"); - if (name == NULL) { - // `import A` syntax - import_module(ct, m, import, NULL); - } - else { - // `import A.B` or `import A: B` syntax - jl_module_import(ct, m, import, name); - } - continue; - } - else if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_as_sym && jl_expr_nargs(a) == 2 && - jl_is_expr(jl_exprarg(a, 0)) && ((jl_expr_t*)jl_exprarg(a, 0))->head == jl_dot_sym) { - jl_sym_t *asname = (jl_sym_t*)jl_exprarg(a, 1); - if (jl_is_symbol(asname)) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "import"); - if (name == NULL) { - // `import A as B` syntax - import_module(ct, m, import, asname); - } - else { - check_macro_rename(name, asname, "import"); - // `import A.B as C` syntax - jl_module_import_as(ct, m, import, name, asname); - } - continue; - } - } - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "syntax: malformed \"import\" statement"); - } - JL_GC_POP(); - ct->world_age = last_age; - return jl_nothing; - } else if (head == jl_export_sym || head == jl_public_sym) { int exp = (head == jl_export_sym); volatile int any_new = 0; diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f6314ed92ee53..8e4bf4fd9db85 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -360,26 +360,31 @@ function check_for_missing_packages_and_run_hooks(ast) end function _modules_to_be_loaded!(ast::Expr, mods::Vector{Symbol}) + function add!(ctx) + if ctx.head == :as + ctx = ctx.args[1] + end + if ctx.args[1] != :. # don't include local import `import .Foo` + push!(mods, ctx.args[1]) + end + end ast.head === :quote && return mods # don't search if it's not going to be run during this eval - if ast.head === :using || ast.head === :import - for arg in ast.args - arg = arg::Expr - arg1 = first(arg.args) - if arg1 isa Symbol # i.e. `Foo` - if arg1 != :. # don't include local import `import .Foo` - push!(mods, arg1) - end - else # i.e. `Foo: bar` - sym = first((arg1::Expr).args)::Symbol - if sym != :. # don't include local import `import .Foo: a` - push!(mods, sym) - end + if ast.head == :call + if length(ast.args) == 5 && ast.args[1] === GlobalRef(Base, :_eval_import) + ctx = ast.args[4] + if ctx isa QuoteNode # i.e. `Foo: bar` + ctx = ctx.value + else + ctx = ast.args[5].value end + add!(ctx) + elseif length(ast.args) == 3 && ast.args[1] == GlobalRef(Base, :_eval_using) + add!(ast.args[3].value) end end if ast.head !== :thunk for arg in ast.args - if isexpr(arg, (:block, :if, :using, :import)) + if isexpr(arg, (:block, :if)) _modules_to_be_loaded!(arg, mods) end end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index ee787e55d78c8..b2172096a2e6b 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1550,59 +1550,61 @@ end @testset "Install missing packages via hooks" begin @testset "Parse AST for packages" begin - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo")) + test_find_packages(e) = + REPL.modules_to_be_loaded(Meta.lower(@__MODULE__, e)) + test_find_packages(s::String) = + REPL.modules_to_be_loaded(Meta.lower(@__MODULE__, Meta.parse(s))) + + mods = test_find_packages("using Foo") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo")) + mods = test_find_packages("import Foo") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo, Bar")) + mods = test_find_packages("using Foo, Bar") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo, Bar")) + mods = test_find_packages("import Foo, Bar") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo.bar, Foo.baz")) + mods = test_find_packages("using Foo.bar, Foo.baz") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo end")) + mods = test_find_packages("if false using Foo end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false if false using Foo end end")) + mods = test_find_packages("if false if false using Foo end end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo, Bar end")) + mods = test_find_packages("if false using Foo, Bar end") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo: bar end")) + mods = test_find_packages("if false using Foo: bar end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo.bar as baz")) + mods = test_find_packages("import Foo.bar as baz") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using .Foo")) + mods = test_find_packages("using .Foo") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Base")) + mods = test_find_packages("using Base") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Base: nope")) + mods = test_find_packages("using Base: nope") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Main")) + mods = test_find_packages("using Main") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Core")) + mods = test_find_packages("using Core") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line(":(using Foo)")) - @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("ex = :(using Foo)")) + mods = test_find_packages(":(using Foo)") @test isempty(mods) - - mods = REPL.modules_to_be_loaded(Base.parse_input_line("Foo")) + mods = test_find_packages("ex = :(using Foo)") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("@eval using Foo")) + mods = test_find_packages("@eval using Foo") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("begin using Foo; @eval using Bar end")) + mods = test_find_packages("begin using Foo; @eval using Bar end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("Core.eval(Main,\"using Foo\")")) + mods = test_find_packages("Core.eval(Main,\"using Foo\")") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("begin using Foo; Core.eval(Main,\"using Foo\") end")) + mods = test_find_packages("begin using Foo; Core.eval(Main,\"using Foo\") end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(:(import .Foo: a)) + mods = test_find_packages(:(import .Foo: a)) @test isempty(mods) - mods = REPL.modules_to_be_loaded(:(using .Foo: a)) + mods = test_find_packages(:(using .Foo: a)) @test isempty(mods) end end diff --git a/sysimage.mk b/sysimage.mk index 550373bfba588..987ca4035870e 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -45,6 +45,7 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ base/indices.jl \ base/iterators.jl \ base/invalidation.jl \ + base/module.jl \ base/namedtuple.jl \ base/number.jl \ base/operators.jl \ diff --git a/test/loading.jl b/test/loading.jl index d12cd2769ef1d..2dcf129a741a9 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1731,3 +1731,9 @@ end rm(depot_path, force=true, recursive=true) end end + +# Test `import Package as M` +module M57965 + import Random as R +end +@test M57965.R === Base.require(M57965, :Random) diff --git a/test/worlds.jl b/test/worlds.jl index dd98170721b1a..42cc74a010670 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -558,3 +558,12 @@ module C57316; import ..X57316.Y57316 as Z, .Z.Y57316 as W; end @test !isdefined(B57316, :X57316) @test !isdefined(C57316, :X57316) @test !isdefined(C57316, :Y57316) + +# jl_module_import should always manipulate the latest world +module M57965 +function f() + @eval Random = 1 + Core._eval_import(true, @__MODULE__, nothing, Expr(:., :Random)) +end +end +@test_throws ErrorException("importing Random into M57965 conflicts with an existing global") M57965.f()s From 4616aa270c81427413765d065db960875b6afd59 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 28 Apr 2025 17:58:50 -0400 Subject: [PATCH 149/662] Update MozillaCACerts_jll to 2025-02-25 (#58248) --- deps/checksums/cacert-2024-12-31.pem/md5 | 1 - deps/checksums/cacert-2024-12-31.pem/sha512 | 1 - deps/checksums/cacert-2025-02-25.pem/md5 | 1 + deps/checksums/cacert-2025-02-25.pem/sha512 | 1 + deps/libgit2.version | 2 +- stdlib/MozillaCACerts_jll/Project.toml | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/cacert-2024-12-31.pem/md5 delete mode 100644 deps/checksums/cacert-2024-12-31.pem/sha512 create mode 100644 deps/checksums/cacert-2025-02-25.pem/md5 create mode 100644 deps/checksums/cacert-2025-02-25.pem/sha512 diff --git a/deps/checksums/cacert-2024-12-31.pem/md5 b/deps/checksums/cacert-2024-12-31.pem/md5 deleted file mode 100644 index b01bf68ddc247..0000000000000 --- a/deps/checksums/cacert-2024-12-31.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -d9178b626f8b87f51b47987418d012bf diff --git a/deps/checksums/cacert-2024-12-31.pem/sha512 b/deps/checksums/cacert-2024-12-31.pem/sha512 deleted file mode 100644 index c12b8215a7855..0000000000000 --- a/deps/checksums/cacert-2024-12-31.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -bf578937d7826106bae1ebe74a70bfbc439387445a1f41ef57430de9d9aea6fcfa1884381bf0ef14632f6b89e9543642c9b774fcca93837efffdc557c4958dbd diff --git a/deps/checksums/cacert-2025-02-25.pem/md5 b/deps/checksums/cacert-2025-02-25.pem/md5 new file mode 100644 index 0000000000000..3dced8d2bee6b --- /dev/null +++ b/deps/checksums/cacert-2025-02-25.pem/md5 @@ -0,0 +1 @@ +1a7de82bb9f0fcc779ca18a7a9310898 diff --git a/deps/checksums/cacert-2025-02-25.pem/sha512 b/deps/checksums/cacert-2025-02-25.pem/sha512 new file mode 100644 index 0000000000000..bb59a65af401e --- /dev/null +++ b/deps/checksums/cacert-2025-02-25.pem/sha512 @@ -0,0 +1 @@ +e5fe41820460e6b65e8cd463d1a5f01b7103e1ef66cb75fedc15ebcba3ba6600d77e5e7c2ab94cbb1f11c63b688026a04422bbe2d7a861f7a988f67522ffae3c diff --git a/deps/libgit2.version b/deps/libgit2.version index 6bfb6106e67d2..3f1f7a66fe972 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -11,4 +11,4 @@ LIBGIT2_SHA1=338e6fb681369ff0537719095e22ce9dc602dbf0 # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. # Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. -MOZILLA_CACERT_VERSION := 2024-12-31 +MOZILLA_CACERT_VERSION := 2025-02-25 diff --git a/stdlib/MozillaCACerts_jll/Project.toml b/stdlib/MozillaCACerts_jll/Project.toml index 2f9bf67e22a74..a951435168922 100644 --- a/stdlib/MozillaCACerts_jll/Project.toml +++ b/stdlib/MozillaCACerts_jll/Project.toml @@ -1,7 +1,7 @@ name = "MozillaCACerts_jll" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" # Keep in sync with `deps/libgit2.version`. -version = "2024.12.31" +version = "2025.02.25" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From dc8bc7366c5bb6bac764f7b18a581640a141b2c3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 28 Apr 2025 19:29:22 -0400 Subject: [PATCH 150/662] Update deps to OpenSSL 3.5.0 (#58247) --- deps/checksums/openssl | 76 ++++++++++++++--------------- deps/openssl.version | 2 +- stdlib/Manifest.toml | 2 +- stdlib/OpenSSL_jll/Project.toml | 2 +- stdlib/OpenSSL_jll/test/runtests.jl | 2 +- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/deps/checksums/openssl b/deps/checksums/openssl index 134ad867cbd3f..3b41bfa69231d 100644 --- a/deps/checksums/openssl +++ b/deps/checksums/openssl @@ -1,38 +1,38 @@ -OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/md5/72c29fd0048b0fee44410cfb82ed6235 -OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/sha512/a1d23c15c16d577e7f350004c3e3e8c9f14375ca31256f4e42dcd828b610eeea269eecac58e1c3449c99dcc80b7a8885bbbd39beb5d5ff85167d953c26c10d00 -OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/md5/29bf1097dbe8ac0c42ffb0c1ff9234cf -OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/sha512/b388a13fbfd416feb95bac4f2f4f47605ac8ad971551ec218a8822618cd6e127ad538782524fed5c5f75ab3caf5b86107598967993ae03516706efa6a3af1010 -OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/md5/971ba30a9e0be025433afe3b0aae6260 -OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/sha512/43b6bea8d3e0ab783ed2ec1140fb9054ef0cdd0ddd34e5e95fb36c7b1b72d7e988b2bb17c878c57b0721c44b7783b2db9d4fc614a63bb557b1c32088dd01d506 -OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/md5/3a3d963e16c7efbacdaea9754db640e3 -OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/sha512/7c3d0ed3a7f37e879e3b8f4c5c67cf2766b5e421fab273806a0412ba12ab4de421bce094713aeb3f6c3915260cb8d7fcb35e214344131e9a5b0081cb7bf0d5dc -OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/md5/c94d4882e57cb9d3688127f5f82331a5 -OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/sha512/b8133e873a960b125d0ab8ddd5b4071a6ab269e2b2f5b36e0d723e759875d21f734ac44f4baa4122bc6b94e23f0d82d401f16c88d9c70dac4031f07dcd20597a -OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/md5/149aacf601e86ed15cd4305d035fb7b2 -OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/sha512/dad7bade8fdf62c642ba1a8553f8a02218dbddd15040388b0a0eaac5391fb3c606860378c5dc40b16978c2f8f3837a1698788788d83006a4b312b1c6e73b2a53 -OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/md5/a1d45f34e463df42c2ea77d9084a4360 -OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/sha512/5f5ba285564b1ff1c5db3a2fe4d2051b9b17a77e6c6da37bc739f64051d64a5bff3968d5326760c102f9ddbc3509bf3eeb3ae267acfecbd0a55ff86ec90b5cb0 -OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/md5/81ed6b1188a7ccda1768b67fdfc6e094 -OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/sha512/2424620b462b5596e317e77ebc294377811ec1210c65baf4cd072b62f8d303aa77b2b4f799611ca91202dc371ea4575c2c13cb222e6edd809a036b639c1595c0 -OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/md5/714bab6e53849d0bf6d9922be51871e2 -OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/sha512/141102d20810986b75ba3b2b4446e4a260f4ae38583b7f8dd8a59b8e0ec8b2bac03ee83127b8f4cdb67bd8fd5ebbb102cb581ed182735283109ff85d049cc55b -OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/md5/e4f5f918c011d87626a0f830243324f9 -OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/sha512/128e6c29f0537818a68cbbd262a3735ee99ccca2377874c22978782abd43c3d0f9bd49d68c05d09f37293df52c238aa33d87244276eef080959aefe42fe8c92f -OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/md5/a68b80b7725887ef33ea36e7d19f7cd1 -OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/sha512/91aeb66c49f73eaa00114a61fc2b644e278bc39948f408806c66f61529ad98d3bc1f885152e1cc275a8374dde2d5ace3695e6f70eec718bacb0269560bce83b0 -OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/md5/ec36f9b42c64ab4b1ce862229bc06924 -OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/sha512/71470814b096ca9127fc2055b4ecc6e6acb30f03fe16754010c5c860f8d82cb37c2e723dca3f92f2aa2e9604fd1f4d141eca333d2c7524274e33d98da34326df -OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/md5/5aecf142b6849b8c2cca957830d49f4e -OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/sha512/d25879a4d6b8cc76c8fc4ed49b38d48938c80ba163e7ffe276bb86fdc4bb76cd596f150564bfa3bd88fa90451916a086ca04d31ba884bd978f40f57f0e4332f7 -OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/md5/41c847ee490a5935fac8c4b663d8e325 -OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/sha512/04979622fae5b24b0f0d79b0853ea311e872a7ca465c6e6700e713a34fa90a182ad78db8babacf322fb288cb62caed1a73f8d47e55fd2b074238191649e0141f -OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/md5/b3c143deff5a311740ccc592a1a433e9 -OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/sha512/31819e2e78dbd7aeb95b84664eac9ee29b0ec4e1b0bc9e06a4ea8f7cb65c21929414d9e4e3fa6ce15944bfd7fc68d699d4016cbe5ace16e98394a60fe369541f -OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/md5/bed866803232e6ada4e22eadfd2b98d1 -OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/sha512/ee6f26d150c81e30c93f08de5428d2f92e02e12565e6dcef09bbd1029ff5477bb2176b6cc7616a7cc898dfec8c5d8263108c3558460397c71ca90af8b230e0c1 -OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/md5/ee6f11020b0ce6eac8016877d1635a04 -OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/sha512/dd154fe1e8c537d42919c0889036ac79e181b765c87130edfeffd2ef8414da902995469167812664e6f77f3c89379c799d4311336f4233b05796b6823838e01c -OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/md5/d08f7d27c775eec9c7e4709d0898d60e -OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/sha512/4806c67ff8f629ee6b3a7c358146675310f0812772773aad7e30d0ccee0cc085741c06d252ed16bf8f874fe794d1b8291e0cbcd65c8eacb9e87c0a5f65742c5f -openssl-3.0.16.tar.gz/md5/7b6a9cded21b9fa51877444f5defebd4 -openssl-3.0.16.tar.gz/sha512/5eea2b0c60d870549fc2b8755f1220a57f870d95fbc8d5cc5abb9589f212d10945f355c3e88ff48540a7ee1c4db774b936023ca33d7c799ea82d91eef9c1c16d +OpenSSL.v3.5.0+0.aarch64-apple-darwin.tar.gz/md5/b8dc9909528f769bd9ac56cf2681f387 +OpenSSL.v3.5.0+0.aarch64-apple-darwin.tar.gz/sha512/0d9ea24d8f856c31c8b88afa1de317d13aff1f1f60b309e06e77eea91d195526ec91ed2d077f0dbb75370c17b8875c24d3066e6872bbef04312616e99d0aff3d +OpenSSL.v3.5.0+0.aarch64-linux-gnu.tar.gz/md5/7cf5baeacf4d882b547c229758a9fa9b +OpenSSL.v3.5.0+0.aarch64-linux-gnu.tar.gz/sha512/726ceee82379e667a65abe27c482d3b57e611c630d82b1314f6d385f0f2e8256835ef707c2e015f9204d563d7ee469bed2dee88d245af81dcde2af3b8331b19c +OpenSSL.v3.5.0+0.aarch64-linux-musl.tar.gz/md5/4601e56eaed365548203752a19f4f8e8 +OpenSSL.v3.5.0+0.aarch64-linux-musl.tar.gz/sha512/da349081850d47b9393665c4365787c26f61471362475c2acd3c8205063d09a785f7b6c836ba6793e880440115b19e85821b4d1938e57dafea0cabb45048a70b +OpenSSL.v3.5.0+0.aarch64-unknown-freebsd.tar.gz/md5/6a9e78436727e67af2f537170e18445e +OpenSSL.v3.5.0+0.aarch64-unknown-freebsd.tar.gz/sha512/4dc2f7a39f17255871773d10ed1b74de5c908af0f7a4bd3f94fd71bc12480fd4cdee0bd859a154328218935f004eee20359dacc353e366c47ed890229a579fc4 +OpenSSL.v3.5.0+0.armv6l-linux-gnueabihf.tar.gz/md5/5c751092c27910a48cab31f87700fe19 +OpenSSL.v3.5.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/b44e2356f719549dd831745963b8c74346be173d176ca15ab2ee6f4a1ec7e105086d89115cb76831a3251eb67bf7c5ff5cba3a03fd4614a3501af235a8e03beb +OpenSSL.v3.5.0+0.armv6l-linux-musleabihf.tar.gz/md5/fc05f9645ff000b21e46951f16833fb0 +OpenSSL.v3.5.0+0.armv6l-linux-musleabihf.tar.gz/sha512/8c960294fe542ab9d9ae7dc283c0c30621f348ff8011a9a47f38c1460234b3b128011426c3e5d0cb6c9b02fbee261b7b264d0b0c55bdf3be2a2cd5bdd210d71d +OpenSSL.v3.5.0+0.armv7l-linux-gnueabihf.tar.gz/md5/8928d47a0f549d15240eb934caddf599 +OpenSSL.v3.5.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/4b5dbfb3a4ea4ebe6510cbe63da2de0bb3762a0fc98946acbb059e9a92791ac65a3519577250dcb571fa07f29be182f165a5d4fa05fc96b60270441adab30e74 +OpenSSL.v3.5.0+0.armv7l-linux-musleabihf.tar.gz/md5/1e4c11043d05bea0fcdbf92525152c51 +OpenSSL.v3.5.0+0.armv7l-linux-musleabihf.tar.gz/sha512/e9514cd0c3a8c3659ff87d505490ca3011a65800222b21e4f932bc2a80fb38bb11de1d13925c3a6313f6bea1c2baf35b38b3db18193ac11ec42eb434edee3418 +OpenSSL.v3.5.0+0.i686-linux-gnu.tar.gz/md5/ee699f302edd1f7677baa565ae631c74 +OpenSSL.v3.5.0+0.i686-linux-gnu.tar.gz/sha512/16dd396b192b4ca23d1fad54d130a92ef43a36259482cd3b276001d084711ef8674dcd167c9f832f5989d6197889af69d2ae6bcef3e6b9f538066bf347c89584 +OpenSSL.v3.5.0+0.i686-linux-musl.tar.gz/md5/e6ffea118acb68d39ccb13df51e15125 +OpenSSL.v3.5.0+0.i686-linux-musl.tar.gz/sha512/44335dcaf144d388bd47dd80db08862f4cff1d5a90861f34001127df74d0d16babedbe0ffd02ab398bddd17ecda605f433a940b3cc5159947cb54810a564b0df +OpenSSL.v3.5.0+0.i686-w64-mingw32.tar.gz/md5/8ef4284ac47a6b45f8c5b01d203ae668 +OpenSSL.v3.5.0+0.i686-w64-mingw32.tar.gz/sha512/d7ea8c94d54a139631f2710cb2c47c0387b694e60dc7afddbca3c6705e17d25ec8958a84b4424edd1ea27d6d1c78457fbacd92f7634345f4ccc1a81cf242c28f +OpenSSL.v3.5.0+0.powerpc64le-linux-gnu.tar.gz/md5/21674471f2a3352ede9aef3454381edd +OpenSSL.v3.5.0+0.powerpc64le-linux-gnu.tar.gz/sha512/5615d438db7c3e97dc422b260b3152cd89a2412b7b9b5d7cea36b0ce471fbd3f1a2e8a9d77f399e257f5c38b8b5dfc256acfbdbe2645ba47b89c177dadd066e9 +OpenSSL.v3.5.0+0.riscv64-linux-gnu.tar.gz/md5/7761384fd5991eb56286f24c9a0fbdba +OpenSSL.v3.5.0+0.riscv64-linux-gnu.tar.gz/sha512/e63d5f7ddc368f4cdb03c299361faef7274930c622404907c3560eb04e6110f851b9a201b402bb6e52fdafe64988f909c209f659f84ba77957eb45a933c8baf1 +OpenSSL.v3.5.0+0.x86_64-apple-darwin.tar.gz/md5/a970728a9aa6f25d56db7e43e7b0cae2 +OpenSSL.v3.5.0+0.x86_64-apple-darwin.tar.gz/sha512/8ab5b2dd90914e193d1f7689c8560228d03cb6ee79fd43a48ae9339b61274fea0557a2bf3a7ae4ce4d4b51630aede55d6d6e860f263e1ffc0bfd6141367a9514 +OpenSSL.v3.5.0+0.x86_64-linux-gnu.tar.gz/md5/4530c0e1791b0eaec99b68f2967a3c2f +OpenSSL.v3.5.0+0.x86_64-linux-gnu.tar.gz/sha512/ba952738be38f52ebc23f48c52c12c1bec9c8b81416264612da21ca21f23604c8e59bf49f73d4b80256ea17b6b662179620deadb8660be98d8ad5ed57e346394 +OpenSSL.v3.5.0+0.x86_64-linux-musl.tar.gz/md5/eb49cefbb938d80198dbab90e1ad9108 +OpenSSL.v3.5.0+0.x86_64-linux-musl.tar.gz/sha512/f038e9bd950e4472cdd82b0c39aebbfd60e75cdf24fd8408d39e4db0793813c9d30471d1ca8d112b0bb4049f18f8fb36b4c3069dfce61032dc73cb6568852b77 +OpenSSL.v3.5.0+0.x86_64-unknown-freebsd.tar.gz/md5/25023844dae8c7d326620b1f9e730a07 +OpenSSL.v3.5.0+0.x86_64-unknown-freebsd.tar.gz/sha512/e38f1f7c452903a09b3f0127e377d5e46e538903f9a58076e53dfc53883b2423463d3fdcf13dc961516965b6dbc2d289bfbfa1027f8c3110a61bdee060bccf73 +OpenSSL.v3.5.0+0.x86_64-w64-mingw32.tar.gz/md5/a73f5220598dfc5e71e1eee6b26f7a27 +OpenSSL.v3.5.0+0.x86_64-w64-mingw32.tar.gz/sha512/c028527230b6e9e675b7e22a21997e5d032e1099dd1f3437c6e764b7967fd0196d4cb46d66b36f2f6ddeb8200f445aa8d6a7a61f7be61288ee5e0e510b5800f8 +openssl-3.5.0.tar.gz/md5/51da7d2bdf7f4f508cb024f562eb9b03 +openssl-3.5.0.tar.gz/sha512/39cc80e2843a2ee30f3f5de25cd9d0f759ad8de71b0b39f5a679afaaa74f4eb58d285ae50e29e4a27b139b49343ac91d1f05478f96fb0c6b150f16d7b634676f diff --git a/deps/openssl.version b/deps/openssl.version index 48831039a313e..49c463aad1565 100644 --- a/deps/openssl.version +++ b/deps/openssl.version @@ -3,4 +3,4 @@ OPENSSL_JLL_NAME := OpenSSL ## source build -OPENSSL_VER := 3.0.16 +OPENSSL_VER := 3.5.0 diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index ca3863d974f6f..c68ebcdbe533a 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -168,7 +168,7 @@ version = "0.8.5+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+1" +version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] diff --git a/stdlib/OpenSSL_jll/Project.toml b/stdlib/OpenSSL_jll/Project.toml index 7c8067261c253..28ecf86381213 100644 --- a/stdlib/OpenSSL_jll/Project.toml +++ b/stdlib/OpenSSL_jll/Project.toml @@ -1,6 +1,6 @@ name = "OpenSSL_jll" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.16+0" +version = "3.5.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/OpenSSL_jll/test/runtests.jl b/stdlib/OpenSSL_jll/test/runtests.jl index 8bf67288834c9..e5ae938b68311 100644 --- a/stdlib/OpenSSL_jll/test/runtests.jl +++ b/stdlib/OpenSSL_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, OpenSSL_jll major = ccall((:OPENSSL_version_major, libcrypto), Cuint, ()) minor = ccall((:OPENSSL_version_minor, libcrypto), Cuint, ()) patch = ccall((:OPENSSL_version_patch, libcrypto), Cuint, ()) - @test VersionNumber(major, minor, patch) == v"3.0.16" + @test VersionNumber(major, minor, patch) == v"3.5.0" end From 3e293bcc8cd967af5aade516dbd97d5c3f73e024 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Tue, 29 Apr 2025 01:47:37 -0400 Subject: [PATCH 151/662] Change `mod(x, Inf)` semantics to align with C when `x` is finite (#47102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix #46694 --------- Co-authored-by: Dilum Aluthge Co-authored-by: Lilith Orion Hafner Co-authored-by: Oscar Smith Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- NEWS.md | 2 ++ base/float.jl | 5 +++- base/int.jl | 2 +- test/numbers.jl | 68 +++++++++++++++++++++++++++---------------------- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/NEWS.md b/NEWS.md index c6db2bbfbbe28..1bd8f16f13b76 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,8 @@ New language features Language changes ---------------- +* `mod(x::AbstractFloat, -Inf)` now returns `x` (as long as `x` is finite), this aligns with C standard and +is considered a bug fix ([#47102]) Compiler/Runtime improvements ----------------------------- diff --git a/base/float.jl b/base/float.jl index ff64ca93760ed..79d4a26ef930e 100644 --- a/base/float.jl +++ b/base/float.jl @@ -603,7 +603,10 @@ function rem(x::T, y::T) where {T<:IEEEFloat} end end -function mod(x::T, y::T) where {T<:AbstractFloat} +function mod(x::T, y::T) where T<:AbstractFloat + if isinf(y) && isfinite(x) + return x + end r = rem(x,y) if r == 0 copysign(r,y) diff --git a/base/int.jl b/base/int.jl index 6d6251ac1d64b..c463869ec2dae 100644 --- a/base/int.jl +++ b/base/int.jl @@ -250,7 +250,7 @@ end The reduction of `x` modulo `y`, or equivalently, the remainder of `x` after floored division by `y`, i.e. `x - y*fld(x,y)` if computed without intermediate rounding. -The result will have the same sign as `y`, and magnitude less than `abs(y)` (with some +The result will have the same sign as `y` if `isfinite(y)`, and magnitude less than `abs(y)` (with some exceptions, see note below). !!! note diff --git a/test/numbers.jl b/test/numbers.jl index 510c630c9f4ba..f5767e235b1d0 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -1595,36 +1595,44 @@ end end end - for x=0:5, y=1:5 - @test div(UInt(x),UInt(y)) == div(x,y) - @test div(UInt(x),y) == div(x,y) - @test div(x,UInt(y)) == div(x,y) - @test div(UInt(x),-y) == reinterpret(UInt,div(x,-y)) - @test div(-x,UInt(y)) == div(-x,y) - - @test fld(UInt(x),UInt(y)) == fld(x,y) - @test fld(UInt(x),y) == fld(x,y) - @test fld(x,UInt(y)) == fld(x,y) - @test fld(UInt(x),-y) == reinterpret(UInt,fld(x,-y)) - @test fld(-x,UInt(y)) == fld(-x,y) - - @test cld(UInt(x),UInt(y)) == cld(x,y) - @test cld(UInt(x),y) == cld(x,y) - @test cld(x,UInt(y)) == cld(x,y) - @test cld(UInt(x),-y) == reinterpret(UInt,cld(x,-y)) - @test cld(-x,UInt(y)) == cld(-x,y) - - @test rem(UInt(x),UInt(y)) == rem(x,y) - @test rem(UInt(x),y) == rem(x,y) - @test rem(x,UInt(y)) == rem(x,y) - @test rem(UInt(x),-y) == rem(x,-y) - @test rem(-x,UInt(y)) == rem(-x,y) - - @test mod(UInt(x),UInt(y)) == mod(x,y) - @test mod(UInt(x),y) == mod(x,y) - @test mod(x,UInt(y)) == mod(x,y) - @test mod(UInt(x),-y) == mod(x,-y) - @test mod(-x,UInt(y)) == mod(-x,y) + @test isnan(mod(NaN, Inf)) + @test isnan(mod(NaN, -Inf)) + for x=0:5 + @test mod(x, Inf) == x + @test mod(x, -Inf) == x + @test mod(-x, Inf) == -x + @test mod(-x, -Inf) == -x + for y=1:5 + @test div(UInt(x),UInt(y)) == div(x,y) + @test div(UInt(x),y) == div(x,y) + @test div(x,UInt(y)) == div(x,y) + @test div(UInt(x),-y) == reinterpret(UInt,div(x,-y)) + @test div(-x,UInt(y)) == div(-x,y) + + @test fld(UInt(x),UInt(y)) == fld(x,y) + @test fld(UInt(x),y) == fld(x,y) + @test fld(x,UInt(y)) == fld(x,y) + @test fld(UInt(x),-y) == reinterpret(UInt,fld(x,-y)) + @test fld(-x,UInt(y)) == fld(-x,y) + + @test cld(UInt(x),UInt(y)) == cld(x,y) + @test cld(UInt(x),y) == cld(x,y) + @test cld(x,UInt(y)) == cld(x,y) + @test cld(UInt(x),-y) == reinterpret(UInt,cld(x,-y)) + @test cld(-x,UInt(y)) == cld(-x,y) + + @test rem(UInt(x),UInt(y)) == rem(x,y) + @test rem(UInt(x),y) == rem(x,y) + @test rem(x,UInt(y)) == rem(x,y) + @test rem(UInt(x),-y) == rem(x,-y) + @test rem(-x,UInt(y)) == rem(-x,y) + + @test mod(UInt(x),UInt(y)) == mod(x,y) + @test mod(UInt(x),y) == mod(x,y) + @test mod(x,UInt(y)) == mod(x,y) + @test mod(UInt(x),-y) == mod(x,-y) + @test mod(-x,UInt(y)) == mod(-x,y) + end end @test div(typemax(UInt64) , 1) == typemax(UInt64) From 95ac8c2901a99d7b97e35dd8ff638c7dcb63879f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 29 Apr 2025 15:17:56 +0200 Subject: [PATCH 152/662] add a note to devdocs about `Compiler.@zone` (#58263) --- doc/src/devdocs/external_profilers.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/src/devdocs/external_profilers.md b/doc/src/devdocs/external_profilers.md index 836d821b91df9..5f54c2619559b 100644 --- a/doc/src/devdocs/external_profilers.md +++ b/doc/src/devdocs/external_profilers.md @@ -8,9 +8,21 @@ The currently supported profilers are: ### Adding New Zones +#### From C/C++ code + To add new zones, use the `JL_TIMING` macro. You can find numerous examples throughout the codebase by searching for `JL_TIMING`. To add a new type of zone you add it to `JL_TIMING_OWNERS` (and possibly `JL_TIMING_EVENTS`). +#### From Julia code + +The `Compiler.@zone` macro can be used to add a zone from Julia code, it is used as: + +```julia +Compiler.@zone "ZONE NAME" begin + ... +end +``` + ### Dynamically Enabling and Disabling Zones The [`JULIA_TIMING_SUBSYSTEMS`](@ref JULIA_TIMING_SUBSYSTEMS) environment variable allows you to enable or disable zones for a specific Julia run. For instance, setting the variable to `+GC,-INFERENCE` will enable the `GC` zones and disable the `INFERENCE` From 1aeea1979fac8f8cc9ffcc89d4c62830be55bebf Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 29 Apr 2025 09:21:26 -0400 Subject: [PATCH 153/662] add PARTITION_KIND_BACKDATED_CONST to UndefVarError_hint (#58260) Wording updated based upon timholy's suggestion in #57969. --- base/errorshow.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/errorshow.jl b/base/errorshow.jl index 95ed2a51b57a4..15e616fe482c5 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1169,6 +1169,8 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) print(io, "\nSuggestion: check for spelling errors or missing imports.") elseif Base.is_some_explicit_imported(kind) print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") + elseif kind === Base.PARTITION_KIND_BACKDATED_CONST + print(io, "\nSuggestion: define the const at top-level before running function that uses it (stricter Julia v1.12+ rule).") end elseif scope === :static_parameter print(io, "\nSuggestion: run Test.detect_unbound_args to detect method arguments that do not fully constrain a type parameter.") From 208560b5e3e0d1c8b63f86854b266962835bf3eb Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 29 Apr 2025 11:00:44 -0400 Subject: [PATCH 154/662] Dimensional `any` and `all` reduce with Boolean operations (#58185) and not bitwise ones. Specifically, they are now reductions using these operators: ```julia and_all(x, y) = (x && y)::Bool or_any(x, y) = (x || y)::Bool # As a performance optimization, avoid runtime branches: and_all(x::Bool, y::Bool) = (x & y)::Bool or_any(x::Bool, y::Bool) = (x | y)::Bool ``` Fixes half of #45562. --------- Co-authored-by: Lilith Hafner Co-authored-by: Lilith Orion Hafner --- base/reduce.jl | 8 ++++++++ base/reducedim.jl | 24 ++++++++++++++---------- test/reduce.jl | 21 +++++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/base/reduce.jl b/base/reduce.jl index c132ef385358d..bd29d7eb3d889 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -29,6 +29,12 @@ mul_prod(x::BitSignedSmall, y::BitSignedSmall) = Int(x) * Int(y) mul_prod(x::BitUnsignedSmall, y::BitUnsignedSmall) = UInt(x) * UInt(y) mul_prod(x::Real, y::Real)::Real = x * y +and_all(x, y) = (x && y)::Bool +or_any(x, y) = (x || y)::Bool +# As a performance optimization, avoid runtime branches: +and_all(x::Bool, y::Bool) = (x & y)::Bool +or_any(x::Bool, y::Bool) = (x | y)::Bool + ## foldl && mapfoldl function mapfoldl_impl(f::F, op::OP, nt, itr) where {F,OP} @@ -338,6 +344,8 @@ reduce_empty(::typeof(*), ::Type{T}) where {T} = one(T) reduce_empty(::typeof(*), ::Type{<:AbstractChar}) = "" reduce_empty(::typeof(&), ::Type{Bool}) = true reduce_empty(::typeof(|), ::Type{Bool}) = false +reduce_empty(::typeof(and_all), ::Type{T}) where {T} = true +reduce_empty(::typeof(or_any), ::Type{T}) where {T} = false reduce_empty(::typeof(add_sum), ::Type{T}) where {T} = reduce_empty(+, T) reduce_empty(::typeof(add_sum), ::Type{T}) where {T<:BitSignedSmall} = zero(Int) diff --git a/base/reducedim.jl b/base/reducedim.jl index 0478afe1a46b6..ba3434494bc0b 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -45,7 +45,7 @@ end initarray!(a::AbstractArray{T}, f, ::Union{typeof(min),typeof(max),typeof(_extrema_rf)}, init::Bool, src::AbstractArray) where {T} = (init && mapfirst!(f, a, src); a) -for (Op, initval) in ((:(typeof(&)), true), (:(typeof(|)), false)) +for (Op, initval) in ((:(typeof(and_all)), true), (:(typeof(or_any)), false)) @eval initarray!(a::AbstractArray, ::Any, ::$(Op), init::Bool, src::AbstractArray) = (init && fill!(a, $initval); a) end @@ -173,6 +173,10 @@ end reducedim_init(f::Union{typeof(abs),typeof(abs2)}, op::typeof(max), A::AbstractArray{T}, region) where {T} = reducedim_initarray(A, region, zero(f(zero(T))), _realtype(f, T)) +reducedim_init(f, op::typeof(and_all), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, true) +reducedim_init(f, op::typeof(or_any), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, false) + +# These definitions are wrong in general; Cf. JuliaLang/julia#45562 reducedim_init(f, op::typeof(&), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, true) reducedim_init(f, op::typeof(|), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, false) @@ -883,13 +887,13 @@ julia> A = [true false; true false] 1 0 1 0 -julia> all!([1; 1], A) -2-element Vector{Int64}: +julia> all!(Bool[1; 1], A) +2-element Vector{Bool}: 0 0 -julia> all!([1 1], A) -1×2 Matrix{Int64}: +julia> all!(Bool[1 1], A) +1×2 Matrix{Bool}: 1 0 ``` """ @@ -958,13 +962,13 @@ julia> A = [true false; true false] 1 0 1 0 -julia> any!([1; 1], A) -2-element Vector{Int64}: +julia> any!(Bool[1; 1], A) +2-element Vector{Bool}: 1 1 -julia> any!([1 1], A) -1×2 Matrix{Int64}: +julia> any!(Bool[1 1], A) +1×2 Matrix{Bool}: 1 0 ``` """ @@ -994,7 +998,7 @@ _all(a, ::Colon) = _all(identity, a, :) for (fname, op) in [(:sum, :add_sum), (:prod, :mul_prod), (:maximum, :max), (:minimum, :min), - (:all, :&), (:any, :|), + (:all, :and_all), (:any, :or_any), (:extrema, :_extrema_rf)] fname! = Symbol(fname, '!') _fname = Symbol('_', fname) diff --git a/test/reduce.jl b/test/reduce.jl index f5140c8a34bd9..bb54462bb78f0 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -681,6 +681,27 @@ end end end +@testset "issue #45562" begin + @test all([true, true, true], dims = 1) == [true] + @test any([true, true, true], dims = 1) == [true] + @test_throws TypeError all([3, 3, 3], dims = 1) + @test_throws TypeError any([3, 3, 3], dims = 1) + @test_throws TypeError all(Any[true, 3, 3], dims = 1) + @test_throws TypeError any(Any[false, 3, 3], dims = 1) + @test_throws TypeError all([1, 1, 1], dims = 1) + @test_throws TypeError any([0, 0, 0], dims = 1) + @test_throws TypeError all!([false], [3, 3, 3]) + @test_throws TypeError any!([false], [3, 3, 3]) + @test_throws TypeError all!([false], Any[true, 3, 3]) + @test_throws TypeError any!([false], Any[false, 3, 3]) + @test_throws TypeError all!([false], [1, 1, 1]) + @test_throws TypeError any!([false], [0, 0, 0]) + @test reduce(|, Bool[]) == false + @test reduce(&, Bool[]) == true + @test reduce(|, Bool[], dims=1) == [false] + @test reduce(&, Bool[], dims=1) == [true] +end + # issue #45748 @testset "foldl's stability for nested Iterators" begin a = Iterators.flatten((1:3, 1:3)) From b9a0497e96286fa3e43740294e2c5f6b17b70ae2 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 29 Apr 2025 11:34:49 -0400 Subject: [PATCH 155/662] clarify that time_ns is monotonic (#57129) --- base/Base_compiler.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 28c07379c54d9..ef69cc23139d1 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -188,8 +188,12 @@ end """ time_ns()::UInt64 -Get the time in nanoseconds relative to some arbitrary time in the past. The primary use is for measuring the elapsed time -between two moments in time. +Get the time in nanoseconds relative to some machine-specific arbitrary time in the past. +The primary use is for measuring elapsed times during program execution. The return value is guaranteed to +be monotonic (mod 2⁶⁴) while the system is running, and is unaffected by clock drift or changes to local calendar time, +but it may change arbitrarily across system reboots or suspensions. + +(Although the returned time is always in nanoseconds, the timing resolution is platform-dependent.) """ time_ns() = ccall(:jl_hrtime, UInt64, ()) From 41e0eb09baa458761702c4a22dee8f57c2df9ec7 Mon Sep 17 00:00:00 2001 From: Patrick Jaap Date: Tue, 29 Apr 2025 17:47:12 +0200 Subject: [PATCH 156/662] BigFloat: Print exponent without leading '+' and zeros (#58008) Fixes #56941 With this small change we get: ```julia julia> big"1e6" 1.0e6 ``` --- base/mpfr.jl | 2 +- doc/src/manual/integers-and-floating-point-numbers.md | 2 +- test/mpfr.jl | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/base/mpfr.jl b/base/mpfr.jl index bcdce70a64635..7d0bd2c1b9986 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -1229,7 +1229,7 @@ function _prettify_bigfloat(s::String)::String string(neg ? '-' : "", '0', '.', '0'^(-expo-1), int, frac == "0" ? "" : frac) end else - string(mantissa, 'e', exponent) + string(mantissa, 'e', expo) end end diff --git a/doc/src/manual/integers-and-floating-point-numbers.md b/doc/src/manual/integers-and-floating-point-numbers.md index fa0ee228e873b..0139a33f4854b 100644 --- a/doc/src/manual/integers-and-floating-point-numbers.md +++ b/doc/src/manual/integers-and-floating-point-numbers.md @@ -600,7 +600,7 @@ julia> parse(BigFloat, "1.23456789012345678901") 1.234567890123456789010000000000000000000000000000000000000000000000000000000004 julia> BigFloat(2.0^66) / 3 -2.459565876494606882133333333333333333333333333333333333333333333333333333333344e+19 +2.459565876494606882133333333333333333333333333333333333333333333333333333333344e19 julia> factorial(BigInt(40)) 815915283247897734345611269596115894272000000000 diff --git a/test/mpfr.jl b/test/mpfr.jl index 3bb8768b280d5..8a537d1d4acd8 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -667,16 +667,19 @@ end @test string(parse(BigFloat, "0.1")) == "0.10000002" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.9000015" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end setprecision(40) do @test string(parse(BigFloat, "0.1")) == "0.10000000000002" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.8999999999942" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end setprecision(123) do @test string(parse(BigFloat, "0.1")) == "0.0999999999999999999999999999999999999953" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.8999999999999999999999999999999999997" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end end @testset "eps" begin @@ -998,7 +1001,7 @@ end test_show_bigfloat(big"1.23456789", contains_e=false, starts="1.23") test_show_bigfloat(big"-1.23456789", contains_e=false, starts="-1.23") - test_show_bigfloat(big"2.3457645687563543266576889678956787e10000", starts="2.345", ends="e+10000") + test_show_bigfloat(big"2.3457645687563543266576889678956787e10000", starts="2.345", ends="e10000") test_show_bigfloat(big"-2.3457645687563543266576889678956787e-10000", starts="-2.345", ends="e-10000") test_show_bigfloat(big"42.0", contains_e=false, starts="42.0") test_show_bigfloat(big"420.0", contains_e=false, starts="420.0") # '0's have to be added on the right before point @@ -1006,10 +1009,10 @@ end test_show_bigfloat(big"420000.0", contains_e=false, starts="420000.0") test_show_bigfloat(big"654321.0", contains_e=false, starts="654321.0") test_show_bigfloat(big"-654321.0", contains_e=false, starts="-654321.0") - test_show_bigfloat(big"6543210.0", contains_e=true, starts="6.5", ends="e+06") + test_show_bigfloat(big"6543210.0", contains_e=true, starts="6.5", ends="e6") test_show_bigfloat(big"0.000123", contains_e=false, starts="0.000123") test_show_bigfloat(big"-0.000123", contains_e=false, starts="-0.000123") - test_show_bigfloat(big"0.00001234", contains_e=true, starts="1.23", ends="e-05") + test_show_bigfloat(big"0.00001234", contains_e=true, starts="1.23", ends="e-5") for to_string in [string, x->sprint(show, x), From 2270fcdcc655c8fa7aa9472069434389f895b147 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Tue, 29 Apr 2025 16:50:29 +0100 Subject: [PATCH 157/662] Rename `_aligned_msize` to prevent conflict with mingw64 defintion (#58238) `_aligned_msize` is now declared in mingw64's crt (https://github.com/mingw-w64/mingw-w64/commit/b40e24afb12524ab97ad27d21f9b35f9f4bab678) as `_CRTIMP size_t __cdecl _aligned_msize(void *_Memory,size_t _Alignment,size_t _Offset);`. Renamed our version to prevent the conflict --- src/gc-common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc-common.c b/src/gc-common.c index c07b707b17709..87ddb68abc704 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -126,7 +126,7 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t // but with several fixes to improve the correctness of the computation and remove unnecessary parameters #define SAVED_PTR(x) ((void *)((DWORD_PTR)((char *)x - sizeof(void *)) & \ ~(sizeof(void *) - 1))) -static size_t _aligned_msize(void *p) +static size_t _jl_aligned_msize(void *p) { void *alloc_ptr = *(void**)SAVED_PTR(p); return _msize(alloc_ptr) - ((char*)p - (char*)alloc_ptr); @@ -138,7 +138,7 @@ size_t memory_block_usable_size(void *p, int isaligned) JL_NOTSAFEPOINT { #if defined(_OS_WINDOWS_) if (isaligned) - return _aligned_msize(p); + return _jl_aligned_msize(p); else return _msize(p); #elif defined(_OS_DARWIN_) From d4f2e8ab3758bc614c80535b13d487cac55a6c8b Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 29 Apr 2025 18:01:53 +0200 Subject: [PATCH 158/662] doc: cross-reference `bind` in `Channel` method doc string (#58113) --- base/channels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/channels.jl b/base/channels.jl index ef508bd40e3ed..0bb73e9acba87 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -61,7 +61,7 @@ Channel(sz=0) = Channel{Any}(sz) """ Channel{T=Any}(func::Function, size=0; taskref=nothing, spawn=false, threadpool=nothing) -Create a new task from `func`, bind it to a new channel of type +Create a new task from `func`, [`bind`](@ref) it to a new channel of type `T` and size `size`, and schedule the task, all in a single call. The channel is automatically closed when the task terminates. From f8faac60ee0fbc339a1b6d53e1b8748977c10fde Mon Sep 17 00:00:00 2001 From: Sam Schweigel Date: Tue, 29 Apr 2025 09:24:33 -0700 Subject: [PATCH 159/662] Core._eval_import: don't throw away "from" path Fixes #58272 --- base/module.jl | 7 +++---- test/loading.jl | 12 ++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/base/module.jl b/base/module.jl index 3e95d9882339c..7e7812173060c 100644 --- a/base/module.jl +++ b/base/module.jl @@ -107,15 +107,14 @@ function _eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, pa elseif path.head !== :. fail() end - old_from = from - from, name = eval_import_path(to, from, path, keyword) + m, name = eval_import_path(to, from, path, keyword) if name !== nothing asname = asname === nothing ? name : asname check_macro_rename(name, asname, keyword) - Core._import(to, from, asname, name, imported) + Core._import(to, m, asname, name, imported) else - Core._import(to, from, asname === nothing ? nameof(from) : asname) + Core._import(to, m, asname === nothing ? nameof(m) : asname) end end end diff --git a/test/loading.jl b/test/loading.jl index 2dcf129a741a9..e95138e27f4dc 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1737,3 +1737,15 @@ module M57965 import Random as R end @test M57965.R === Base.require(M57965, :Random) + +# #58272 - _eval_import accidentally reuses evaluated "from" path +module M58272_1 + const x = 1 + module M58272_2 + const y = 3 + const x = 2 + end +end +module M58272_to end +@eval M58272_to import ..M58272_1: M58272_2.y, x +@test @eval M58272_to x === 1 From 117cc81baa11e9e7ad67174b268bd1800bbfa607 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:40:43 -0400 Subject: [PATCH 160/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?JuliaSyntaxHighlighting=20stdlib=20from=20b7a1c63=20to=20f803fb?= =?UTF-8?q?0=20(#58239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: JuliaSyntaxHighlighting URL: https://github.com/julialang/JuliaSyntaxHighlighting.jl.git Stdlib branch: main Julia branch: master Old commit: b7a1c63 New commit: f803fb0 Julia version: 1.13.0-DEV JuliaSyntaxHighlighting version: 1.12.0(Does not match) Bump invoked by: @tecosaur Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/julialang/JuliaSyntaxHighlighting.jl/compare/b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f...f803fb01dae69b116426d163c53b2a00570e1274 ``` $ git log --oneline b7a1c63..f803fb0 f803fb0 Guard against the assumption children gives nodes ``` Co-authored-by: tecosaur <20903656+tecosaur@users.noreply.github.com> --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/JuliaSyntaxHighlighting.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 deleted file mode 100644 index 2a4b55e15ab2d..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ed0ccc4434fc70b06e8ea1ddb8141511 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 deleted file mode 100644 index 456f1ee64ca0b..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -04efea853a1c1bfbf5baf4d2908ce492a5ff3029bca73a004280aa116157b6b678a5f9fd6a115f9c57a625d0841d3fb96c8d68ec467e5bc4a743272bee84c8c7 diff --git a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 new file mode 100644 index 0000000000000..44f1d6505372c --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 @@ -0,0 +1 @@ +b386bcdbabb0b32556d5d84034a6554a diff --git a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 new file mode 100644 index 0000000000000..07498d7ad4964 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 @@ -0,0 +1 @@ +f0d08ae9965ab0b173bd24394d4e8cd852ca3fc158122aabcb9019ebeeaeaf4fafa40abc16ad103e419b43c71bb1fa4337a9b4ca1f24b456bed5d93fc6921959 diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index 1c9bfb131dc0f..9fd6979c624b2 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f +JULIASYNTAXHIGHLIGHTING_SHA1 = f803fb01dae69b116426d163c53b2a00570e1274 JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/julialang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/julialang/JuliaSyntaxHighlighting.jl/tarball/$1 From d46b665067bd9fc352c89c9d0abb591eaa4f7695 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerberg <39104088+nhz2@users.noreply.github.com> Date: Tue, 29 Apr 2025 14:02:24 -0400 Subject: [PATCH 161/662] Note annotated string API is experimental in Julia 1.11 in HISTORY.md (#58134) --- HISTORY.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index e8f3ed01a1baa..0c55b0954a0f8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -293,20 +293,6 @@ New language features * The new macro `Base.Cartesian.@ncallkw` is analogous to `Base.Cartesian.@ncall`, but allows to add keyword arguments to the function call ([#51501]). * Support for Unicode 15.1 ([#51799]). -* Three new types around the idea of text with "annotations" (`Pair{Symbol, Any}` - entries, e.g. `:lang => "en"` or `:face => :magenta`). These annotations - are preserved across operations (e.g. string concatenation with `*`) when - possible. - * `AnnotatedString` is a new `AbstractString` type. It wraps an underlying - string and allows for annotations to be attached to regions of the string. - This type is used extensively in the new `StyledStrings` standard library to - hold styling information. - * `AnnotatedChar` is a new `AbstractChar` type. It wraps another char and - holds a list of annotations that apply to it. - * `AnnotatedIOBuffer` is a new `IO` type that mimics an `IOBuffer`, but has - specialised `read`/`write` methods for annotated content. This can be - thought of both as a "string builder" of sorts and also as glue between - annotated and unannotated content. * `Manifest.toml` files can now be renamed in the format `Manifest-v{major}.{minor}.toml` to be preferentially picked up by the given julia version. i.e. in the same folder, a `Manifest-v1.11.toml` would be used by v1.11 and `Manifest.toml` by every other julia @@ -408,7 +394,20 @@ Standard library changes #### StyledStrings -* A new standard library for handling styling in a more comprehensive and structured way ([#49586]). +* A new experimental standard library for handling styling in a more comprehensive and structured way ([#49586]). +* Three new types around the idea of text with "annotations" (`Pair{Symbol, Any}` + entries, e.g. `:lang => "en"` or `:face => :magenta`). These annotations + are preserved across operations (e.g. string concatenation with `*`) when + possible. + * `AnnotatedString` is a new `AbstractString` type. It wraps an underlying + string and allows for annotations to be attached to regions of the string. + This type is used extensively to hold styling information. + * `AnnotatedChar` is a new `AbstractChar` type. It wraps another char and + holds a list of annotations that apply to it. + * `AnnotatedIOBuffer` is a new `IO` type that mimics an `IOBuffer`, but has + specialised `read`/`write` methods for annotated content. This can be + thought of both as a "string builder" of sorts and also as glue between + annotated and unannotated content. * The new `Faces` struct serves as a container for text styling information (think typeface, as well as color and decoration), and comes with a framework to provide a convenient, extensible (via `addface!`), and customisable (with a From abcfb3bc4f4c611210dc31e25a7ee8da7d68109c Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Fri, 25 Apr 2025 14:33:23 -0300 Subject: [PATCH 162/662] Refactor julia initialization by finding and loading the sysimage earlier --- src/init.c | 180 +++++++++++++++++++++++++----------------------- src/jlapi.c | 26 +++---- src/julia.h | 3 +- src/threading.c | 4 +- 4 files changed, 109 insertions(+), 104 deletions(-) diff --git a/src/init.c b/src/init.c index cf432a6bd6047..ccfe5e0e93e57 100644 --- a/src/init.c +++ b/src/init.c @@ -622,7 +622,7 @@ static const char *absformat(const char *in) return out; } -static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir) { // this function resolves the paths in jl_options to absolute file locations as needed // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths @@ -642,11 +642,13 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) jl_options.julia_bin = (char*)malloc_s(path_size + 1); memcpy((char*)jl_options.julia_bin, free_path, path_size); ((char*)jl_options.julia_bin)[path_size] = '\0'; - if (!jl_options.julia_bindir) { + if (!julia_bindir) { jl_options.julia_bindir = getenv("JULIA_BINDIR"); if (!jl_options.julia_bindir) { jl_options.julia_bindir = dirname(free_path); } + } else { + jl_options.julia_bindir = julia_bindir; } if (jl_options.julia_bindir) jl_options.julia_bindir = absrealpath(jl_options.julia_bindir, 0); @@ -721,8 +723,85 @@ static void restore_fp_env(void) jl_error("Failed to configure floating point environment"); } } +static NOINLINE void _finish_julia_init(jl_image_buf_t sysimage, jl_ptls_t ptls, jl_task_t *ct) +{ + JL_TIMING(JULIA_INIT, JULIA_INIT); + + if (sysimage.kind == JL_IMAGE_KIND_SO) + jl_gc_notify_image_load(sysimage.data, sysimage.size); + + if (jl_options.cpu_target == NULL) + jl_options.cpu_target = "native"; + + // Parse image, perform relocations, and init JIT targets, etc. + jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); + + jl_init_codegen(); + jl_init_common_symbols(); + + if (sysimage.kind != JL_IMAGE_KIND_NONE) { + // Load the .ji or .so sysimage + jl_restore_system_image(&parsed_image, sysimage); + } else { + // No sysimage provided, init a minimal environment + jl_init_types(); + jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; + jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; + } + + jl_init_flisp(); + jl_init_serializer(); + + if (sysimage.kind == JL_IMAGE_KIND_NONE) { + jl_top_module = jl_core_module; + jl_init_intrinsic_functions(); + jl_init_primitives(); + jl_init_main_module(); + jl_load(jl_core_module, "boot.jl"); + jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); + post_boot_hooks(); + } + + if (jl_base_module == NULL) { + // nthreads > 1 requires code in Base + jl_atomic_store_relaxed(&jl_n_threads, 1); + jl_n_markthreads = 0; + jl_n_sweepthreads = 0; + jl_n_gcthreads = 0; + jl_n_threads_per_pool[JL_THREADPOOL_ID_INTERACTIVE] = 0; + jl_n_threads_per_pool[JL_THREADPOOL_ID_DEFAULT] = 1; + } else { + jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); + post_image_load_hooks(); + } + jl_start_threads(); + jl_start_gc_threads(); + uv_barrier_wait(&thread_init_done); + + jl_gc_enable(1); + + if ((sysimage.kind != JL_IMAGE_KIND_NONE) && + (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { + jl_array_t *init_order = jl_module_init_order; + JL_GC_PUSH1(&init_order); + jl_module_init_order = NULL; + int i, l = jl_array_nrows(init_order); + for (i = 0; i < l; i++) { + jl_value_t *mod = jl_array_ptr_ref(init_order, i); + jl_module_run_initializer((jl_module_t*)mod); + } + JL_GC_POP(); + } + + if (jl_options.trim) { + jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); + arraylist_new(jl_entrypoint_mis, 0); + } + + if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) + jl_install_sigint_handler(); +} -static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_task_t *ct); JL_DLLEXPORT int jl_default_debug_info_kind; JL_DLLEXPORT jl_cgparams_t jl_default_cgparams = { @@ -750,7 +829,7 @@ static void init_global_mutexes(void) { JL_MUTEX_INIT(&profile_show_peek_cond_lock, "profile_show_peek_cond_lock"); } -JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) +static void julia_init(jl_image_buf_t sysimage) { // initialize many things, in no particular order // but generally running from simple platform things to optional @@ -860,96 +939,27 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi); #pragma GCC diagnostic pop JL_GC_PROMISE_ROOTED(ct); - _finish_julia_init(rel, ptls, ct); + _finish_julia_init(sysimage, ptls, ct); } -static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_task_t *ct) -{ - JL_TIMING(JULIA_INIT, JULIA_INIT); - jl_resolve_sysimg_location(rel); + +// This function is responsible for loading the image and initializing paths in jl_options +JL_DLLEXPORT void jl_load_image_and_init(JL_IMAGE_SEARCH rel, const char* julia_bindir, void *handle) { + libsupport_init(); + + jl_resolve_sysimg_location(rel, julia_bindir); // loads sysimg if available, and conditionally sets jl_options.cpu_target jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; - if (rel == JL_IMAGE_IN_MEMORY) { + if (handle != NULL) { + sysimage = jl_set_sysimg_so(handle); + } else if (rel == JL_IMAGE_IN_MEMORY) { sysimage = jl_set_sysimg_so(jl_exe_handle); jl_options.image_file = jl_options.julia_bin; - } - else if (jl_options.image_file) + } else if (jl_options.image_file) sysimage = jl_preload_sysimg(jl_options.image_file); - if (sysimage.kind == JL_IMAGE_KIND_SO) - jl_gc_notify_image_load(sysimage.data, sysimage.size); - - if (jl_options.cpu_target == NULL) - jl_options.cpu_target = "native"; - - // Parse image, perform relocations, and init JIT targets, etc. - jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); - - jl_init_codegen(); - jl_init_common_symbols(); - - if (sysimage.kind != JL_IMAGE_KIND_NONE) { - // Load the .ji or .so sysimage - jl_restore_system_image(&parsed_image, sysimage); - } else { - // No sysimage provided, init a minimal environment - jl_init_types(); - jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; - jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; - } - - jl_init_flisp(); - jl_init_serializer(); - - if (sysimage.kind == JL_IMAGE_KIND_NONE) { - jl_top_module = jl_core_module; - jl_init_intrinsic_functions(); - jl_init_primitives(); - jl_init_main_module(); - jl_load(jl_core_module, "boot.jl"); - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); - post_boot_hooks(); - } - - if (jl_base_module == NULL) { - // nthreads > 1 requires code in Base - jl_atomic_store_relaxed(&jl_n_threads, 1); - jl_n_markthreads = 0; - jl_n_sweepthreads = 0; - jl_n_gcthreads = 0; - jl_n_threads_per_pool[JL_THREADPOOL_ID_INTERACTIVE] = 0; - jl_n_threads_per_pool[JL_THREADPOOL_ID_DEFAULT] = 1; - } else { - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); - post_image_load_hooks(); - } - jl_start_threads(); - jl_start_gc_threads(); - uv_barrier_wait(&thread_init_done); - - jl_gc_enable(1); - - if ((sysimage.kind != JL_IMAGE_KIND_NONE) && - (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { - jl_array_t *init_order = jl_module_init_order; - JL_GC_PUSH1(&init_order); - jl_module_init_order = NULL; - int i, l = jl_array_nrows(init_order); - for (i = 0; i < l; i++) { - jl_value_t *mod = jl_array_ptr_ref(init_order, i); - jl_module_run_initializer((jl_module_t*)mod); - } - JL_GC_POP(); - } - - if (jl_options.trim) { - jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); - arraylist_new(jl_entrypoint_mis, 0); - } - - if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) - jl_install_sigint_handler(); + julia_init(sysimage); } #ifdef __cplusplus diff --git a/src/jlapi.c b/src/jlapi.c index 6ef0dd17befed..43b956e5a2500 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -68,6 +68,15 @@ JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) } } +JL_DLLEXPORT void jl_init_with_handle(void *handle) { + if (jl_is_initialized()) + return; + const char *image_path = jl_pathname_for_handle(handle); + jl_enter_threaded_region(); // This should maybe be behind a lock, but it's harmless if done twice + jl_options.image_file = image_path; + jl_load_image_and_init(JL_IMAGE_JULIA_HOME, NULL, handle); + jl_exception_clear(); +} /** * @brief Initialize Julia with a specified system image file. * @@ -87,23 +96,11 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, { if (jl_is_initialized()) return; - libsupport_init(); - if (julia_bindir) { - jl_options.julia_bindir = julia_bindir; - } else { -#ifdef _OS_WINDOWS_ - jl_options.julia_bindir = strdup(jl_get_libdir()); -#else - int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); - if (written < 0) - abort(); // unexpected: memory allocation failed -#endif - } if (image_path != NULL) jl_options.image_file = image_path; else jl_options.image_file = jl_get_default_sysimg_path(); - julia_init(JL_IMAGE_JULIA_HOME); + jl_load_image_and_init(JL_IMAGE_JULIA_HOME, julia_bindir, NULL); jl_exception_clear(); } @@ -1100,8 +1097,7 @@ JL_DLLEXPORT int jl_repl_entrypoint(int argc, char *argv[]) #endif jl_error("Failed to self-execute"); } - - julia_init(jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); + jl_load_image_and_init(jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME, NULL, NULL); if (lisp_prompt) { jl_current_task->world_age = jl_get_world_counter(); jl_lisp_prompt(); diff --git a/src/julia.h b/src/julia.h index faad02c0aa50c..43a263ab566e0 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2223,10 +2223,11 @@ struct _jl_image_t; typedef struct _jl_image_t jl_image_t; JL_DLLIMPORT const char *jl_get_libdir(void); -JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel); JL_DLLEXPORT void jl_init(void); JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, const char *image_path); +JL_DLLEXPORT void jl_load_image_and_init(JL_IMAGE_SEARCH rel, const char* julia_bindir, void *handle); +JL_DLLEXPORT void jl_init_with_handle(void *handle); JL_DLLEXPORT const char *jl_get_default_sysimg_path(void); JL_DLLEXPORT int jl_is_initialized(void); JL_DLLEXPORT void jl_atexit_hook(int status); diff --git a/src/threading.c b/src/threading.c index f01db388f4cc3..b1f92ffb152bb 100644 --- a/src/threading.c +++ b/src/threading.c @@ -461,9 +461,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_autoinit_and_adopt_thread(void) " (this should not happen, please file a bug report)\n"); exit(1); } - const char *image_path = jl_pathname_for_handle(handle); - jl_enter_threaded_region(); // This should maybe be behind a lock, but it's harmless if done twice - jl_init_with_image(NULL, image_path); + jl_init_with_handle(handle); return &jl_get_current_task()->gcstack; } From 201c18c096be5d4930a1f86bcfc1fd083ea38b2d Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 29 Apr 2025 12:51:20 -0400 Subject: [PATCH 163/662] Initializer timing sub-system earlier --- src/init.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index ccfe5e0e93e57..c5418aff5f01c 100644 --- a/src/init.c +++ b/src/init.c @@ -834,7 +834,7 @@ static void julia_init(jl_image_buf_t sysimage) // initialize many things, in no particular order // but generally running from simple platform things to optional // configuration features - jl_init_timing(); + // Make sure we finalize the tls callback before starting any threads. (void)jl_get_pgcstack(); @@ -946,6 +946,7 @@ static void julia_init(jl_image_buf_t sysimage) // This function is responsible for loading the image and initializing paths in jl_options JL_DLLEXPORT void jl_load_image_and_init(JL_IMAGE_SEARCH rel, const char* julia_bindir, void *handle) { libsupport_init(); + jl_init_timing(); jl_resolve_sysimg_location(rel, julia_bindir); From 35ac254715913778b0588acbe3666a60ce2ed3e7 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 29 Apr 2025 14:08:48 -0400 Subject: [PATCH 164/662] Fix `julia_bindir` calculation when embedding --- src/init.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index c5418aff5f01c..62ab2c2e91a0a 100644 --- a/src/init.c +++ b/src/init.c @@ -642,10 +642,16 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bi jl_options.julia_bin = (char*)malloc_s(path_size + 1); memcpy((char*)jl_options.julia_bin, free_path, path_size); ((char*)jl_options.julia_bin)[path_size] = '\0'; - if (!julia_bindir) { + if (julia_bindir == NULL) { jl_options.julia_bindir = getenv("JULIA_BINDIR"); if (!jl_options.julia_bindir) { - jl_options.julia_bindir = dirname(free_path); +#ifdef _OS_WINDOWS_ + jl_options.julia_bindir = strdup(jl_get_libdir()); +#else + int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); + if (written < 0) + abort(); // unexpected: memory allocation failed +#endif } } else { jl_options.julia_bindir = julia_bindir; From 8522727ff54405cb3b9e4b89f885399e5003042d Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 29 Apr 2025 14:44:58 -0400 Subject: [PATCH 165/662] Replace `jl_load_image_and_init` with inlined logic This makes it clearer that sysimage resolution and `jl_options` setup are the responsibility of `jl_api.c`, not the initialization system. --- doc/src/manual/embedding.md | 2 +- src/init.c | 198 +-------------------------------- src/jl_exported_funcs.inc | 4 +- src/jlapi.c | 214 ++++++++++++++++++++++++++++++++++-- src/julia.h | 7 +- src/julia_internal.h | 1 + src/threading.c | 2 +- 7 files changed, 216 insertions(+), 212 deletions(-) diff --git a/doc/src/manual/embedding.md b/doc/src/manual/embedding.md index 64104771cd9cd..7fb40ba5bd649 100644 --- a/doc/src/manual/embedding.md +++ b/doc/src/manual/embedding.md @@ -54,7 +54,7 @@ linking against `libjulia`. The first thing that must be done before calling any other Julia C function is to initialize Julia. This is done by calling `jl_init`, which tries to automatically determine Julia's install location. If you need to specify a custom location, or specify which system -image to load, use `jl_init_with_image` instead. +image to load, use `jl_init_with_image_file` or `jl_init_with_image_handle` instead. The second statement in the test program evaluates a Julia statement using a call to `jl_eval_string`. diff --git a/src/init.c b/src/init.c index 62ab2c2e91a0a..5a8a3eaf5ccf2 100644 --- a/src/init.c +++ b/src/init.c @@ -533,177 +533,6 @@ int jl_isabspath(const char *in) JL_NOTSAFEPOINT return 0; // relative path } -static char *absrealpath(const char *in, int nprefix) -{ // compute an absolute realpath location, so that chdir doesn't change the file reference - // ignores (copies directly over) nprefix characters at the start of abspath -#ifndef _OS_WINDOWS_ - char *out = realpath(in + nprefix, NULL); - if (out) { - if (nprefix > 0) { - size_t sz = strlen(out) + 1; - char *cpy = (char*)malloc_s(sz + nprefix); - memcpy(cpy, in, nprefix); - memcpy(cpy + nprefix, out, sz); - free(out); - out = cpy; - } - } - else { - size_t sz = strlen(in + nprefix) + 1; - if (in[nprefix] == PATHSEPSTRING[0]) { - out = (char*)malloc_s(sz + nprefix); - memcpy(out, in, sz + nprefix); - } - else { - size_t path_size = JL_PATH_MAX; - char *path = (char*)malloc_s(JL_PATH_MAX); - if (uv_cwd(path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving current working directory"); - } - out = (char*)malloc_s(path_size + 1 + sz + nprefix); - memcpy(out, in, nprefix); - memcpy(out + nprefix, path, path_size); - out[nprefix + path_size] = PATHSEPSTRING[0]; - memcpy(out + nprefix + path_size + 1, in + nprefix, sz); - free(path); - } - } -#else - // GetFullPathName intentionally errors if given an empty string so manually insert `.` to invoke cwd - char *in2 = (char*)malloc_s(JL_PATH_MAX); - if (strlen(in) - nprefix == 0) { - memcpy(in2, in, nprefix); - in2[nprefix] = '.'; - in2[nprefix+1] = '\0'; - in = in2; - } - DWORD n = GetFullPathName(in + nprefix, 0, NULL, NULL); - if (n <= 0) { - jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); - } - char *out = (char*)malloc_s(n + nprefix); - DWORD m = GetFullPathName(in + nprefix, n, out + nprefix, NULL); - if (n != m + 1) { - jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); - } - memcpy(out, in, nprefix); - free(in2); -#endif - return out; -} - -// create an absolute-path copy of the input path format string -// formed as `joinpath(replace(pwd(), "%" => "%%"), in)` -// unless `in` starts with `%` -static const char *absformat(const char *in) -{ - if (in[0] == '%' || jl_isabspath(in)) - return in; - // get an escaped copy of cwd - size_t path_size = JL_PATH_MAX; - char path[JL_PATH_MAX]; - if (uv_cwd(path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving current working directory"); - } - size_t sz = strlen(in) + 1; - size_t i, fmt_size = 0; - for (i = 0; i < path_size; i++) - fmt_size += (path[i] == '%' ? 2 : 1); - char *out = (char*)malloc_s(fmt_size + 1 + sz); - fmt_size = 0; - for (i = 0; i < path_size; i++) { // copy-replace pwd portion - char c = path[i]; - out[fmt_size++] = c; - if (c == '%') - out[fmt_size++] = '%'; - } - out[fmt_size++] = PATHSEPSTRING[0]; // path sep - memcpy(out + fmt_size, in, sz); // copy over format, including nul - return out; -} - -static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir) -{ - // this function resolves the paths in jl_options to absolute file locations as needed - // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths - // it may fail, print an error, and exit(1) if any of these paths are longer than JL_PATH_MAX - // - // note: if you care about lost memory, you should call the appropriate `free()` function - // on the original pointer for each `char*` you've inserted into `jl_options`, after - // calling `julia_init()` - char *free_path = (char*)malloc_s(JL_PATH_MAX); - size_t path_size = JL_PATH_MAX; - if (uv_exepath(free_path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving exepath"); - } - if (path_size >= JL_PATH_MAX) { - jl_error("fatal error: jl_options.julia_bin path too long"); - } - jl_options.julia_bin = (char*)malloc_s(path_size + 1); - memcpy((char*)jl_options.julia_bin, free_path, path_size); - ((char*)jl_options.julia_bin)[path_size] = '\0'; - if (julia_bindir == NULL) { - jl_options.julia_bindir = getenv("JULIA_BINDIR"); - if (!jl_options.julia_bindir) { -#ifdef _OS_WINDOWS_ - jl_options.julia_bindir = strdup(jl_get_libdir()); -#else - int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); - if (written < 0) - abort(); // unexpected: memory allocation failed -#endif - } - } else { - jl_options.julia_bindir = julia_bindir; - } - if (jl_options.julia_bindir) - jl_options.julia_bindir = absrealpath(jl_options.julia_bindir, 0); - free(free_path); - free_path = NULL; - if (jl_options.image_file) { - if (rel == JL_IMAGE_JULIA_HOME && !jl_isabspath(jl_options.image_file)) { - // build time path, relative to JULIA_BINDIR - free_path = (char*)malloc_s(JL_PATH_MAX); - int n = snprintf(free_path, JL_PATH_MAX, "%s" PATHSEPSTRING "%s", - jl_options.julia_bindir, jl_options.image_file); - if (n >= JL_PATH_MAX || n < 0) { - jl_error("fatal error: jl_options.image_file path too long"); - } - jl_options.image_file = free_path; - } - if (jl_options.image_file) - jl_options.image_file = absrealpath(jl_options.image_file, 0); - if (free_path) { - free(free_path); - free_path = NULL; - } - } - if (jl_options.outputo) - jl_options.outputo = absrealpath(jl_options.outputo, 0); - if (jl_options.outputji) - jl_options.outputji = absrealpath(jl_options.outputji, 0); - if (jl_options.outputbc) - jl_options.outputbc = absrealpath(jl_options.outputbc, 0); - if (jl_options.outputasm) - jl_options.outputasm = absrealpath(jl_options.outputasm, 0); - if (jl_options.machine_file) - jl_options.machine_file = absrealpath(jl_options.machine_file, 0); - if (jl_options.output_code_coverage) - jl_options.output_code_coverage = absformat(jl_options.output_code_coverage); - if (jl_options.tracked_path) - jl_options.tracked_path = absrealpath(jl_options.tracked_path, 0); - - const char **cmdp = jl_options.cmds; - if (cmdp) { - for (; *cmdp; cmdp++) { - const char *cmd = *cmdp; - if (cmd[0] == 'L') { - *cmdp = absrealpath(cmd, 1); - } - } - } -} - JL_DLLEXPORT int jl_is_file_tracked(jl_sym_t *path) { const char* path_ = jl_symbol_name(path); @@ -729,7 +558,7 @@ static void restore_fp_env(void) jl_error("Failed to configure floating point environment"); } } -static NOINLINE void _finish_julia_init(jl_image_buf_t sysimage, jl_ptls_t ptls, jl_task_t *ct) +static NOINLINE void _finish_jl_init_(jl_image_buf_t sysimage, jl_ptls_t ptls, jl_task_t *ct) { JL_TIMING(JULIA_INIT, JULIA_INIT); @@ -835,7 +664,7 @@ static void init_global_mutexes(void) { JL_MUTEX_INIT(&profile_show_peek_cond_lock, "profile_show_peek_cond_lock"); } -static void julia_init(jl_image_buf_t sysimage) +JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage) { // initialize many things, in no particular order // but generally running from simple platform things to optional @@ -945,28 +774,7 @@ static void julia_init(jl_image_buf_t sysimage) jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi); #pragma GCC diagnostic pop JL_GC_PROMISE_ROOTED(ct); - _finish_julia_init(sysimage, ptls, ct); -} - - -// This function is responsible for loading the image and initializing paths in jl_options -JL_DLLEXPORT void jl_load_image_and_init(JL_IMAGE_SEARCH rel, const char* julia_bindir, void *handle) { - libsupport_init(); - jl_init_timing(); - - jl_resolve_sysimg_location(rel, julia_bindir); - - // loads sysimg if available, and conditionally sets jl_options.cpu_target - jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; - if (handle != NULL) { - sysimage = jl_set_sysimg_so(handle); - } else if (rel == JL_IMAGE_IN_MEMORY) { - sysimage = jl_set_sysimg_so(jl_exe_handle); - jl_options.image_file = jl_options.julia_bin; - } else if (jl_options.image_file) - sysimage = jl_preload_sysimg(jl_options.image_file); - - julia_init(sysimage); + _finish_jl_init_(sysimage, ptls, ct); } #ifdef __cplusplus diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 64dcb2278d240..cdf280c97db6d 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -235,9 +235,11 @@ XX(jl_hrtime) \ XX(jl_idtable_rehash) \ XX(jl_init) \ + XX(jl_init_) \ XX(jl_init_options) \ XX(jl_init_restored_module) \ - XX(jl_init_with_image) \ + XX(jl_init_with_image_file) \ + XX(jl_init_with_image_handle) \ XX(jl_install_sigint_handler) \ XX(jl_instantiate_type_in_env) \ XX(jl_instantiate_unionall) \ diff --git a/src/jlapi.c b/src/jlapi.c index 43b956e5a2500..47d5b84fa5606 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -26,11 +26,13 @@ extern "C" { #include #endif +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir); + /** * @brief Check if Julia is already initialized. * - * Determine if Julia has been previously initialized - * via `jl_init` or `jl_init_with_image`. + * Determine if Julia has been previously initialized via `jl_init` or + * `jl_init_with_image_file` or `jl_init_with_image_handle`. * * @return Returns 1 if Julia is initialized, 0 otherwise. */ @@ -68,13 +70,18 @@ JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) } } -JL_DLLEXPORT void jl_init_with_handle(void *handle) { +JL_DLLEXPORT void jl_init_with_image_handle(void *handle) { if (jl_is_initialized()) return; + const char *image_path = jl_pathname_for_handle(handle); - jl_enter_threaded_region(); // This should maybe be behind a lock, but it's harmless if done twice jl_options.image_file = image_path; - jl_load_image_and_init(JL_IMAGE_JULIA_HOME, NULL, handle); + + jl_resolve_sysimg_location(JL_IMAGE_JULIA_HOME, NULL); + jl_image_buf_t sysimage = jl_set_sysimg_so(handle); + + jl_init_(sysimage); + jl_exception_clear(); } /** @@ -91,8 +98,8 @@ JL_DLLEXPORT void jl_init_with_handle(void *handle) { * @param image_path The path of a system image file (*.so). Interpreted as relative to julia_bindir * or the default Julia home directory if not an absolute path. */ -JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, - const char *image_path) +JL_DLLEXPORT void jl_init_with_image_file(const char *julia_bindir, + const char *image_path) { if (jl_is_initialized()) return; @@ -100,7 +107,12 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, jl_options.image_file = image_path; else jl_options.image_file = jl_get_default_sysimg_path(); - jl_load_image_and_init(JL_IMAGE_JULIA_HOME, julia_bindir, NULL); + + jl_resolve_sysimg_location(JL_IMAGE_JULIA_HOME, julia_bindir); + jl_image_buf_t sysimage = jl_preload_sysimg(jl_options.image_file); + + jl_init_(sysimage); + jl_exception_clear(); } @@ -112,7 +124,7 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, */ JL_DLLEXPORT void jl_init(void) { - jl_init_with_image(NULL, jl_get_default_sysimg_path()); + jl_init_with_image_file(NULL, jl_get_default_sysimg_path()); } static void _jl_exception_clear(jl_task_t *ct) JL_NOTSAFEPOINT @@ -1097,7 +1109,15 @@ JL_DLLEXPORT int jl_repl_entrypoint(int argc, char *argv[]) #endif jl_error("Failed to self-execute"); } - jl_load_image_and_init(jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME, NULL, NULL); + + JL_IMAGE_SEARCH rel = jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME; + jl_resolve_sysimg_location(rel, NULL); + jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; + if (jl_options.image_file) + sysimage = jl_preload_sysimg(jl_options.image_file); + + jl_init_(sysimage); + if (lisp_prompt) { jl_current_task->world_age = jl_get_world_counter(); jl_lisp_prompt(); @@ -1108,6 +1128,180 @@ JL_DLLEXPORT int jl_repl_entrypoint(int argc, char *argv[]) return ret; } +// create an absolute-path copy of the input path format string +// formed as `joinpath(replace(pwd(), "%" => "%%"), in)` +// unless `in` starts with `%` +static const char *absformat(const char *in) +{ + if (in[0] == '%' || jl_isabspath(in)) + return in; + // get an escaped copy of cwd + size_t path_size = JL_PATH_MAX; + char path[JL_PATH_MAX]; + if (uv_cwd(path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving current working directory"); + } + size_t sz = strlen(in) + 1; + size_t i, fmt_size = 0; + for (i = 0; i < path_size; i++) + fmt_size += (path[i] == '%' ? 2 : 1); + char *out = (char*)malloc_s(fmt_size + 1 + sz); + fmt_size = 0; + for (i = 0; i < path_size; i++) { // copy-replace pwd portion + char c = path[i]; + out[fmt_size++] = c; + if (c == '%') + out[fmt_size++] = '%'; + } + out[fmt_size++] = PATHSEPSTRING[0]; // path sep + memcpy(out + fmt_size, in, sz); // copy over format, including nul + return out; +} + +static char *absrealpath(const char *in, int nprefix) +{ // compute an absolute realpath location, so that chdir doesn't change the file reference + // ignores (copies directly over) nprefix characters at the start of abspath +#ifndef _OS_WINDOWS_ + char *out = realpath(in + nprefix, NULL); + if (out) { + if (nprefix > 0) { + size_t sz = strlen(out) + 1; + char *cpy = (char*)malloc_s(sz + nprefix); + memcpy(cpy, in, nprefix); + memcpy(cpy + nprefix, out, sz); + free(out); + out = cpy; + } + } + else { + size_t sz = strlen(in + nprefix) + 1; + if (in[nprefix] == PATHSEPSTRING[0]) { + out = (char*)malloc_s(sz + nprefix); + memcpy(out, in, sz + nprefix); + } + else { + size_t path_size = JL_PATH_MAX; + char *path = (char*)malloc_s(JL_PATH_MAX); + if (uv_cwd(path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving current working directory"); + } + out = (char*)malloc_s(path_size + 1 + sz + nprefix); + memcpy(out, in, nprefix); + memcpy(out + nprefix, path, path_size); + out[nprefix + path_size] = PATHSEPSTRING[0]; + memcpy(out + nprefix + path_size + 1, in + nprefix, sz); + free(path); + } + } +#else + // GetFullPathName intentionally errors if given an empty string so manually insert `.` to invoke cwd + char *in2 = (char*)malloc_s(JL_PATH_MAX); + if (strlen(in) - nprefix == 0) { + memcpy(in2, in, nprefix); + in2[nprefix] = '.'; + in2[nprefix+1] = '\0'; + in = in2; + } + DWORD n = GetFullPathName(in + nprefix, 0, NULL, NULL); + if (n <= 0) { + jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); + } + char *out = (char*)malloc_s(n + nprefix); + DWORD m = GetFullPathName(in + nprefix, n, out + nprefix, NULL); + if (n != m + 1) { + jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); + } + memcpy(out, in, nprefix); + free(in2); +#endif + return out; +} + +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir) +{ + libsupport_init(); + jl_init_timing(); + + // this function resolves the paths in jl_options to absolute file locations as needed + // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths + // it may fail, print an error, and exit(1) if any of these paths are longer than JL_PATH_MAX + // + // note: if you care about lost memory, you should call the appropriate `free()` function + // on the original pointer for each `char*` you've inserted into `jl_options`, after + // calling `jl_init_()` + char *free_path = (char*)malloc_s(JL_PATH_MAX); + size_t path_size = JL_PATH_MAX; + if (uv_exepath(free_path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving exepath"); + } + if (path_size >= JL_PATH_MAX) { + jl_error("fatal error: jl_options.julia_bin path too long"); + } + jl_options.julia_bin = (char*)malloc_s(path_size + 1); + memcpy((char*)jl_options.julia_bin, free_path, path_size); + ((char*)jl_options.julia_bin)[path_size] = '\0'; + if (julia_bindir == NULL) { + jl_options.julia_bindir = getenv("JULIA_BINDIR"); + if (!jl_options.julia_bindir) { +#ifdef _OS_WINDOWS_ + jl_options.julia_bindir = strdup(jl_get_libdir()); +#else + int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); + if (written < 0) + abort(); // unexpected: memory allocation failed +#endif + } + } else { + jl_options.julia_bindir = julia_bindir; + } + if (jl_options.julia_bindir) + jl_options.julia_bindir = absrealpath(jl_options.julia_bindir, 0); + free(free_path); + free_path = NULL; + if (jl_options.image_file) { + if (rel == JL_IMAGE_JULIA_HOME && !jl_isabspath(jl_options.image_file)) { + // build time path, relative to JULIA_BINDIR + free_path = (char*)malloc_s(JL_PATH_MAX); + int n = snprintf(free_path, JL_PATH_MAX, "%s" PATHSEPSTRING "%s", + jl_options.julia_bindir, jl_options.image_file); + if (n >= JL_PATH_MAX || n < 0) { + jl_error("fatal error: jl_options.image_file path too long"); + } + jl_options.image_file = free_path; + } + if (jl_options.image_file) + jl_options.image_file = absrealpath(jl_options.image_file, 0); + if (free_path) { + free(free_path); + free_path = NULL; + } + } + if (jl_options.outputo) + jl_options.outputo = absrealpath(jl_options.outputo, 0); + if (jl_options.outputji) + jl_options.outputji = absrealpath(jl_options.outputji, 0); + if (jl_options.outputbc) + jl_options.outputbc = absrealpath(jl_options.outputbc, 0); + if (jl_options.outputasm) + jl_options.outputasm = absrealpath(jl_options.outputasm, 0); + if (jl_options.machine_file) + jl_options.machine_file = absrealpath(jl_options.machine_file, 0); + if (jl_options.output_code_coverage) + jl_options.output_code_coverage = absformat(jl_options.output_code_coverage); + if (jl_options.tracked_path) + jl_options.tracked_path = absrealpath(jl_options.tracked_path, 0); + + const char **cmdp = jl_options.cmds; + if (cmdp) { + for (; *cmdp; cmdp++) { + const char *cmd = *cmdp; + if (cmd[0] == 'L') { + *cmdp = absrealpath(cmd, 1); + } + } + } +} + #ifdef __cplusplus } #endif diff --git a/src/julia.h b/src/julia.h index 43a263ab566e0..7f7362b9d4a9a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2224,10 +2224,9 @@ typedef struct _jl_image_t jl_image_t; JL_DLLIMPORT const char *jl_get_libdir(void); JL_DLLEXPORT void jl_init(void); -JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, - const char *image_path); -JL_DLLEXPORT void jl_load_image_and_init(JL_IMAGE_SEARCH rel, const char* julia_bindir, void *handle); -JL_DLLEXPORT void jl_init_with_handle(void *handle); +JL_DLLEXPORT void jl_init_with_image_file(const char *julia_bindir, + const char *image_path); +JL_DLLEXPORT void jl_init_with_image_handle(void *handle); JL_DLLEXPORT const char *jl_get_default_sysimg_path(void); JL_DLLEXPORT int jl_is_initialized(void); JL_DLLEXPORT void jl_atexit_hook(int status); diff --git a/src/julia_internal.h b/src/julia_internal.h index b588f84d05319..573ccb3306bd0 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -196,6 +196,7 @@ void JL_UV_LOCK(void); extern _Atomic(unsigned) _threadedregion; extern _Atomic(uint16_t) io_loop_tid; +JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage); JL_DLLEXPORT void jl_enter_threaded_region(void); JL_DLLEXPORT void jl_exit_threaded_region(void); int jl_running_under_rr(int recheck) JL_NOTSAFEPOINT; diff --git a/src/threading.c b/src/threading.c index b1f92ffb152bb..4256115214fc2 100644 --- a/src/threading.c +++ b/src/threading.c @@ -461,7 +461,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_autoinit_and_adopt_thread(void) " (this should not happen, please file a bug report)\n"); exit(1); } - jl_init_with_handle(handle); + jl_init_with_image_handle(handle); return &jl_get_current_task()->gcstack; } From e5631ff04604b8de0544884fe4fc4a921b3b2ec6 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Wed, 30 Apr 2025 07:11:10 +0530 Subject: [PATCH 166/662] Define `isdebugbuild` before it is imported (#58206) Following on from https://github.com/JuliaLang/julia/pull/58088, this ensures that `isdebugbuild` is defined in `Base` before it is imported in `Libdl`. --- base/Base.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/base/Base.jl b/base/Base.jl index 3bb8988d36804..afa5a3d93d27c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -127,6 +127,12 @@ include("missing.jl") # version include("version.jl") +#= +isdebugbuild is defined here as this is imported in libdl.jl (included in libc.jl) +The method is added in util.jl +=# +function isdebugbuild end + # system & environment include("sysinfo.jl") include("libc.jl") From 1f1839129b6efbf6f280b56d71f761642763a445 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 29 Apr 2025 21:58:25 -0400 Subject: [PATCH 167/662] OncePerX: Improve initializer type for DataType constructors (#58265) Generalized from https://github.com/JuliaData/Parsers.jl/pull/197/ --- base/lock.jl | 6 ++++++ test/threads.jl | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/base/lock.jl b/base/lock.jl index 24c09ec289018..993b771d73f1e 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -735,7 +735,9 @@ mutable struct OncePerProcess{T, F} <: Function return once end end +OncePerProcess{T}(initializer::Type{U}) where {T, U} = OncePerProcess{T, Type{U}}(initializer) OncePerProcess{T}(initializer::F) where {T, F} = OncePerProcess{T, F}(initializer) +OncePerProcess(initializer::Type{U}) where U = OncePerProcess{Base.promote_op(initializer), Type{U}}(initializer) OncePerProcess(initializer) = OncePerProcess{Base.promote_op(initializer), typeof(initializer)}(initializer) @inline function (once::OncePerProcess{T,F})() where {T,F} state = (@atomic :acquire once.state) @@ -842,7 +844,9 @@ mutable struct OncePerThread{T, F} <: Function return once end end +OncePerThread{T}(initializer::Type{U}) where {T, U} = OncePerThread{T,Type{U}}(initializer) OncePerThread{T}(initializer::F) where {T, F} = OncePerThread{T,F}(initializer) +OncePerThread(initializer::Type{U}) where U = OncePerThread{Base.promote_op(initializer), Type{U}}(initializer) OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof(initializer)}(initializer) @inline (once::OncePerThread{T,F})() where {T,F} = once[Threads.threadid()] @inline function getindex(once::OncePerThread{T,F}, tid::Integer) where {T,F} @@ -961,8 +965,10 @@ false mutable struct OncePerTask{T, F} <: Function const initializer::F + OncePerTask{T}(initializer::Type{U}) where {T, U} = new{T,Type{U}}(initializer) OncePerTask{T}(initializer::F) where {T, F} = new{T,F}(initializer) OncePerTask{T,F}(initializer::F) where {T, F} = new{T,F}(initializer) + OncePerTask(initializer::Type{U}) where U = new{Base.promote_op(initializer), Type{U}}(initializer) OncePerTask(initializer) = new{Base.promote_op(initializer), typeof(initializer)}(initializer) end @inline function (once::OncePerTask{T,F})() where {T,F} diff --git a/test/threads.jl b/test/threads.jl index 52d0546f0e31b..fa0b33a6352f3 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -447,6 +447,12 @@ let once = OncePerProcess(() -> return [nothing]) @atomic once.state = 0x01 @test x === once() end +let once1 = OncePerProcess(BigFloat), once2 = OncePerProcess{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerProcess with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerProcess{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerProcess{BigFloat,Type{BigFloat}} +end let once = OncePerProcess{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("OncePerProcess initializer failed previously") once() @@ -551,6 +557,12 @@ let e = Base.Event(true), @test_throws ArgumentError once[-1] end +let once1 = OncePerThread(BigFloat), once2 = OncePerThread{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerThread with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerThread{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerThread{BigFloat,Type{BigFloat}} +end let once = OncePerThread{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("OncePerThread initializer failed previously") once() @@ -563,6 +575,12 @@ let once = OncePerTask(() -> return [nothing]) delete!(task_local_storage(), once) @test x !== once() === once() end +let once1 = OncePerTask(BigFloat), once2 = OncePerTask{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerTask with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerTask{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerTask{BigFloat,Type{BigFloat}} +end let once = OncePerTask{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("expected") once() From fc8e6e731c59f6b62f695a6616f37bfb951d7dca Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Apr 2025 12:10:06 +0200 Subject: [PATCH 168/662] bpart: Fix a hang in a particular corner case (#58271) The `jl_get_binding_partition_with_hint` version of the bpart lookup did not properly set the `max_world` of the gap, leaving it at `~(size_t)0`. Having a `gap` with that `max_world`, but with `gap.replace` set is inconsistent and ended up causing an integer overflow downstream in the computation of `expected_prev_min_world`, which looked like a concurrent modification causing an infinite retry. Fixes #58257 --- Compiler/test/inference.jl | 17 +++++++++++++++++ src/module.c | 20 +++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 249a614fccc5b..c5f44e22d55ed 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6428,4 +6428,21 @@ global invalid_setglobal!_exct_modeling::Int setglobal!(@__MODULE__, :invalid_setglobal!_exct_modeling, x) end == ErrorException +# Issue #58257 - Hang in inference during BindingPartition resolution +module A58257 + module B58257 + using ..A58257 + # World age here is N + end + using .B58257 + # World age here is N+1 + @eval f() = $(GlobalRef(B58257, :get!)) +end + +## The sequence of events is critical here. +A58257.get! # Creates binding partition in A, N+1:∞ +A58257.B58257.get! # Creates binding partition in A.B, N+1:∞ +Base.invoke_in_world(UInt(38678), getglobal, A58257, :get!) # Expands binding partition in A through restriction == resolution.binding_or_const && prev->kind == new_kind) { +retry: if (!jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, new_min_world)) { if (expected_prev_min_world <= new_min_world) { return prev; } else if (expected_prev_min_world <= new_max_world) { - // Concurrent modification by another thread - bail. - return NULL; + // Concurrent modification of the partition. However, our lookup is still valid, + // so we should still be able to extend the partition. + goto retry; } // There remains a gap - proceed } else { @@ -154,7 +157,7 @@ static jl_binding_partition_t *jl_implicit_import_resolved(jl_binding_t *b, stru for (;;) { // We've updated the previous partition - check if we've closed a gap size_t next_max_world = jl_atomic_load_relaxed(&next->max_world); - if (next_max_world == expected_prev_min_world-1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { + if (next_max_world >= expected_prev_min_world-1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { if (jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, next_min_world)) { jl_binding_partition_t *nextnext = jl_atomic_load_relaxed(&next->next); if (!jl_atomic_cmpswap(&prev->next, &next, nextnext)) { @@ -370,7 +373,7 @@ JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t bpart->kind = resolution.ultimate_kind; } -STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, size_t max_world, modstack_t *st) JL_GLOBALLY_ROOTED { assert(jl_is_binding(b)); struct implicit_search_gap gap; @@ -378,7 +381,7 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b gap.insert = insert; gap.inherited_flags = 0; gap.min_world = 0; - gap.max_world = ~(size_t)0; + gap.max_world = max_world; while (1) { gap.replace = jl_atomic_load_relaxed(gap.insert); jl_binding_partition_t *bpart = jl_get_binding_partition__(b, world, &gap); @@ -395,15 +398,14 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) if (!b) return NULL; // Duplicate the code for the entry frame for branch prediction - return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, NULL); + return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, ~(size_t)0, NULL); } jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b, jl_binding_partition_t *prev, size_t world) JL_GLOBALLY_ROOTED { // Helper for getting a binding partition for an older world after we've already looked up the partition for a newer world assert(b); - // TODO: Is it possible for a concurrent lookup to have expanded this bpart, making this false? - assert(jl_atomic_load_relaxed(&prev->min_world) > world); - return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, NULL); + size_t prev_min_world = jl_atomic_load_relaxed(&prev->min_world); + return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, prev_min_world-1, NULL); } jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min_world, size_t max_world) { From 252c65e8c82d4a4795ee423fd4c031b61b52c5aa Mon Sep 17 00:00:00 2001 From: Helios De Rosario Date: Wed, 30 Apr 2025 14:27:26 +0200 Subject: [PATCH 169/662] Add explanation on arithmetic operations with `Bool` values in the manual (#58281) --- doc/src/manual/mathematical-operations.md | 36 +++++++++++++++-------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/doc/src/manual/mathematical-operations.md b/doc/src/manual/mathematical-operations.md index d2cef68bd6fff..20abf40917a18 100644 --- a/doc/src/manual/mathematical-operations.md +++ b/doc/src/manual/mathematical-operations.md @@ -47,18 +47,6 @@ julia> 3*2/12 operators. For instance, we would generally write `-x + 2` to reflect that first `x` gets negated, and then `2` is added to that result.) -When used in multiplication, `false` acts as a *strong zero*: - -```jldoctest -julia> NaN * false -0.0 - -julia> false * Inf -0.0 -``` - -This is useful for preventing the propagation of `NaN` values in quantities that are known to be zero. See [Knuth (1992)](https://arxiv.org/abs/math/9205211) for motivation. - ## Boolean Operators The following [Boolean operators](https://en.wikipedia.org/wiki/Boolean_algebra#Operations) are supported on [`Bool`](@ref) types: @@ -71,7 +59,29 @@ The following [Boolean operators](https://en.wikipedia.org/wiki/Boolean_algebra# Negation changes `true` to `false` and vice versa. The short-circuiting operations are explained on the linked page. -Note that `Bool` is an integer type and all the usual promotion rules and numeric operators are also defined on it. +## Arithmetic operations with `Bool` values + +Note that `Bool` is an integer type, such that `false` is numerically equal to `0` and `true` is numerically equal to `1`. All the usual promotion rules and numeric operators are also defined on it, with a special behavior of arithmetic (non-Boolean) operations when all the arguments are `Bool`: in those cases, the arguments are promoted to `Int` instead of keeping their type. Compare e.g. the following equivalent operations with `Bool` and with a different numeric type (`UInt8`): + +```jldoctest +julia> true - true +0 + +julia> 0x01 - 0x01 +0x00 +``` + +Also, when used in multiplication, `false` acts as a *strong zero*: + +```jldoctest +julia> NaN * false +0.0 + +julia> false * Inf +0.0 +``` + +This is useful for preventing the propagation of `NaN` values in quantities that are known to be zero. See [Knuth (1992)](https://arxiv.org/abs/math/9205211) for motivation. ## Bitwise Operators From 89003fe49c67508122934041c2ca0b2906a5f7c4 Mon Sep 17 00:00:00 2001 From: Nathan Boyer <65452054+nathanrboyer@users.noreply.github.com> Date: Wed, 30 Apr 2025 08:28:44 -0400 Subject: [PATCH 170/662] Update rounding.jl docstring (#58277) --- base/rounding.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/rounding.jl b/base/rounding.jl index 0d8f576ded398..88966c82fb3a6 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -337,7 +337,7 @@ Without keyword arguments, `x` is rounded to an integer value, returning a value thrown if the value is not representable by `T`, similar to [`convert`](@ref). If the `digits` keyword argument is provided, it rounds to the specified number of digits -after the decimal place (or before if negative), in base `base`. +after the decimal place (or before if `digits` is negative), in base `base`. If the `sigdigits` keyword argument is provided, it rounds to the specified number of significant digits, in base `base`. From d7cd2710f6d919cd99e3b26a8056bbaeee74560c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Apr 2025 18:33:04 +0200 Subject: [PATCH 171/662] Strengthen language around `@assume_effects` :consistent (#58254) The current language of `:consistent` for assume_effects is problematic, because it explicitly promises something false: That you're allowed to mark a method as `:consistent` even in the face of redefinitions. To understand this problem a bit better, we need to think about the primary use case of the `:consistent` annotation: To perform constant propagation evaluation of fuctions with constant arguments. However, since constant evaluation only runs in a single world (unlike the abstract interpreter, which symbolically runs in multiple worlds and keeps track of which world range its result is valid for), we run into a bit of a problem. In principle, for full correctness and under the current definition of consistentcy, we can only claim that we know the result for a single world age (the one that we ran in). This is problematic, because it would force us to re-infer the function for every new world age. In practice however, ignoring `@assume_effects` for the moment, we have a stronger guarantee. When inference sets the :consistent effect bit, it guarantees consistentcy across the entire computed min_world:max_world range. If there is a redefinition, the world range will be terminated, even if it is `:consistent` both before and after and even if the redefinition would not otherwise affect inference's computed information. This is useful, because it lets us conclude that the information derived from concrete evluation is valid for the entire min_world:max_world range of the corresponding code instance. Coming back to `@assume_effects` we run into a problem however, because we have no way to provide the required edges to inference. In practice inference will probably be able to figure it out, but this is insufficient as a semantic guarantee. After some discussion within the compiler team, we came to the conclusion that the best resolution was to change the documented semantics of `@assume_effects :consistent` to require consistent-cy across the entire defined world range of the method, not just world-age-by-world-age. This is a significantly stronger guarantee, but appears necessary for semantic correctness. In the future we may want to consider `:consistent` annotations for particular if blocks, which would not require the same restrictions (because it would still rely on inference to add appropriate edges for redefinitions). Closes #46156 --- Compiler/src/abstractinterpretation.jl | 14 +++++++++++++ Compiler/src/typeinfer.jl | 29 +++++++++++++++++++------- base/expr.jl | 14 ++++++++----- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 3a2d04a127607..188085f52d091 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3499,6 +3499,20 @@ function merge_override_effects!(interp::AbstractInterpreter, effects::Effects, # It is possible for arguments (GlobalRef/:static_parameter) to throw, # but these will be recomputed during SSA construction later. override = decode_statement_effects_override(sv) + if override.consistent + m = sv.linfo.def + if isa(m, Method) + # N.B.: We'd like deleted_world here, but we can't add an appropriate edge at this point. + # However, in order to reach here in the first place, ordinary method lookup would have + # had to add an edge and appropriate invalidation trigger. + valid_worlds = WorldRange(m.primary_world, typemax(Int)) + if sv.world.this in valid_worlds + update_valid_age!(sv, valid_worlds) + else + override = EffectsOverride(override, consistent=false) + end + end + end effects = override_effects(effects, override) set_curr_ssaflag!(sv, flags_for_effects(effects), IR_FLAGS_EFFECTS) merge_effects!(interp, sv, effects) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 2b4ada7805140..a5b87c86a3ac2 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -368,11 +368,17 @@ function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) return typ end -function adjust_effects(ipo_effects::Effects, def::Method) +function adjust_effects(ipo_effects::Effects, def::Method, world::UInt) # override the analyzed effects using manually annotated effect settings override = decode_effects_override(def.purity) + valid_worlds = WorldRange(0, typemax(UInt)) if is_effect_overridden(override, :consistent) - ipo_effects = Effects(ipo_effects; consistent=ALWAYS_TRUE) + # See note on `typemax(Int)` instead of `deleted_world` in adjust_effects! + override_valid_worlds = WorldRange(def.primary_world, typemax(Int)) + if world in override_valid_worlds + ipo_effects = Effects(ipo_effects; consistent=ALWAYS_TRUE) + valid_worlds = override_valid_worlds + end end if is_effect_overridden(override, :effect_free) ipo_effects = Effects(ipo_effects; effect_free=ALWAYS_TRUE) @@ -400,7 +406,7 @@ function adjust_effects(ipo_effects::Effects, def::Method) if is_effect_overridden(override, :nortcall) ipo_effects = Effects(ipo_effects; nortcall=true) end - return ipo_effects + return (ipo_effects, valid_worlds) end function adjust_effects(sv::InferenceState) @@ -454,7 +460,8 @@ function adjust_effects(sv::InferenceState) # override the analyzed effects using manually annotated effect settings def = sv.linfo.def if isa(def, Method) - ipo_effects = adjust_effects(ipo_effects, def) + (ipo_effects, valid_worlds) = adjust_effects(ipo_effects, def, sv.world.this) + update_valid_age!(sv, valid_worlds) end return ipo_effects @@ -492,9 +499,9 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: end end result = me.result - result.valid_worlds = me.world.valid_worlds result.result = bestguess ipo_effects = result.ipo_effects = me.ipo_effects = adjust_effects(me) + result.valid_worlds = me.world.valid_worlds result.exc_result = me.exc_bestguess = refine_exception_type(me.exc_bestguess, ipo_effects) me.src.rettype = widenconst(ignorelimited(bestguess)) me.src.ssaflags = me.ssaflags @@ -957,8 +964,13 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize update_valid_age!(caller, frame.world.valid_worlds) local isinferred = is_inferred(frame) local edge = isinferred ? edge_ci : nothing - local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects - adjust_effects(effects_for_cycle(frame.ipo_effects), method) + local effects, valid_worlds + if isinferred + effects = frame.result.ipo_effects # effects are adjusted already within `finish` for ipo_effects + else + (effects, valid_worlds) = adjust_effects(effects_for_cycle(frame.ipo_effects), method, frame.world.this) + update_valid_age!(caller, valid_worlds) + end local bestguess = frame.bestguess local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: @@ -981,7 +993,8 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize # return the current knowledge about this cycle frame = frame::InferenceState update_valid_age!(caller, frame.world.valid_worlds) - effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) + (effects, valid_worlds) = adjust_effects(effects_for_cycle(frame.ipo_effects), method, frame.world.this) + update_valid_age!(caller, valid_worlds) bestguess = frame.bestguess exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, nothing, edgecycle, edgelimited)) diff --git a/base/expr.jl b/base/expr.jl index dd357736bc529..ca9c6b46b41b2 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -540,16 +540,20 @@ The `:consistent` setting asserts that for egal (`===`) inputs: contents) are not egal. !!! note - The `:consistent`-cy assertion is made world-age wise. More formally, write - ``fᵢ`` for the evaluation of ``f`` in world-age ``i``, then this setting requires: + The `:consistent`-cy assertion is made with respect to a particular world range `R`. + More formally, write ``fᵢ`` for the evaluation of ``f`` in world-age ``i``, then this setting requires: ```math - ∀ i, x, y: x ≡ y → fᵢ(x) ≡ fᵢ(y) + ∀ i ∈ R, j ∈ R, x, y: x ≡ y → fᵢ(x) ≡ fⱼ(y) ``` - However, for two world ages ``i``, ``j`` s.t. ``i ≠ j``, we may have ``fᵢ(x) ≢ fⱼ(y)``. + + For `@assume_effects`, the range `R` is `m.primary_world:m.deleted_world` of + the annotated or containing method. + + For ordinary code instances, `R` is `ci.min_world:ci.max_world`. A further implication is that `:consistent` functions may not make their return value dependent on the state of the heap or any other global state - that is not constant for a given world age. + that is not constant over the given world age range. !!! note The `:consistent`-cy includes all legal rewrites performed by the optimizer. From 1e248e120ab6dac851e23128c4376c2683f3c1af Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Apr 2025 18:34:35 +0200 Subject: [PATCH 172/662] Add manual chapter on world age and binding partition (#58253) Co-authored-by: Alex Arslan Co-authored-by: Nick Robinson --- base/Base_compiler.jl | 2 +- doc/src/manual/methods.md | 69 +-------- doc/src/manual/modules.md | 62 ++++++++ doc/src/manual/worldage.md | 295 +++++++++++++++++++++++++++++++++++++ 4 files changed, 360 insertions(+), 68 deletions(-) create mode 100644 doc/src/manual/worldage.md diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index ef69cc23139d1..aa18535d505a5 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -265,7 +265,7 @@ support libraries, etc. In these cases it can be useful to prevent unwanted method invalidation and recompilation latency, and to prevent the user from breaking supporting infrastructure by mistake. -The current world age can be queried using [`Base.get_world_counter()`](@ref) +The global world age can be queried using [`Base.get_world_counter()`](@ref) and stored for later use within the lifetime of the current Julia session, or when serializing and reloading the system image. diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index e7f6514a64506..7538649f2b8df 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -578,73 +578,8 @@ However, future calls to `tryeval` will continue to see the definition of `newfu You may want to try this for yourself to see how it works. -The implementation of this behavior is a "world age counter". -This monotonically increasing value tracks each method definition operation. -This allows describing "the set of method definitions visible to a given runtime environment" -as a single number, or "world age". -It also allows comparing the methods available in two worlds just by comparing their ordinal value. -In the example above, we see that the "current world" (in which the method `newfun` exists), -is one greater than the task-local "runtime world" that was fixed when the execution of `tryeval` started. - -Sometimes it is necessary to get around this (for example, if you are implementing the above REPL). -Fortunately, there is an easy solution: call the function using [`Base.invokelatest`](@ref) or -the macro version [`Base.@invokelatest`](@ref): - -```jldoctest -julia> function tryeval2() - @eval newfun2() = 2 - @invokelatest newfun2() - end -tryeval2 (generic function with 1 method) - -julia> tryeval2() -2 -``` - -Finally, let's take a look at some more complex examples where this rule comes into play. -Define a function `f(x)`, which initially has one method: - -```jldoctest redefinemethod -julia> f(x) = "original definition" -f (generic function with 1 method) -``` - -Start some other operations that use `f(x)`: - -```jldoctest redefinemethod -julia> g(x) = f(x) -g (generic function with 1 method) - -julia> t = @async f(wait()); yield(); -``` - -Now we add some new methods to `f(x)`: - -```jldoctest redefinemethod -julia> f(x::Int) = "definition for Int" -f (generic function with 2 methods) - -julia> f(x::Type{Int}) = "definition for Type{Int}" -f (generic function with 3 methods) -``` - -Compare how these results differ: - -```jldoctest redefinemethod -julia> f(1) -"definition for Int" - -julia> g(1) -"definition for Int" - -julia> fetch(schedule(t, 1)) -"original definition" - -julia> t = @async f(wait()); yield(); - -julia> fetch(schedule(t, 1)) -"definition for Int" -``` +The implementation of this behavior is a "world age counter", which is further described in the [Worldage](@ref man-worldage) +manual chapter. ## Design Patterns with Parametric Methods diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index 00b55c23af905..72b9d586945f9 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -322,6 +322,68 @@ Here, Julia cannot decide which `f` you are referring to, so you have to make a 3. When the names in question *do* share a meaning, it is common for one module to import it from another, or have a lightweight “base” package with the sole function of defining an interface like this, which can be used by other packages. It is conventional to have such package names end in `...Base` (which has nothing to do with Julia's `Base` module). +### Precedence order of definitions + +There are in general four kinds of binding definitions: + 1. Those provided via implicit import through `using M` + 2. Those provided via explicit import (e.g. `using M: x`, `import M: x`) + 3. Those declared implicitly as global (via `global x` without type specification) + 4. Those declared explicitly using definition syntax (`const`, `global x::T`, `struct`, etc.) + +Syntactically, we divide these into three precedence levels (from weakest to strongest) + 1. Implicit imports + 2. Implicit declarations + 3. Explicit declarations and imports + +In general, we permit replacement of weaker bindings by stronger ones: + +```julia-repl +julia> module M1; const x = 1; export x; end +Main.M1 + +julia> using .M1 + +julia> x # Implicit import from M1 +1 + +julia> begin; f() = (global x; x = 1) end + +julia> x # Implicit declaration +ERROR: UndefVarError: `x` not defined in `Main` +Suggestion: add an appropriate import or assignment. This global was declared but not assigned. + +julia> const x = 2 # Explicit declaration +2 +``` + +However, within the explicit precedence level, replacement is syntactically disallowed: +```julia-repl +julia> module M1; const x = 1; export x; end +Main.M1 + +julia> import .M1: x + +julia> const x = 2 +ERROR: cannot declare Main.x constant; it was already declared as an import +Stacktrace: + [1] top-level scope + @ REPL[3]:1 +``` + +or ignored: + +```julia-repl +julia> const y = 2 +2 + +julia> import .M1: x as y +WARNING: import of M1.x into Main conflicts with an existing identifier; ignored. +``` + +The resolution of an implicit binding depends on the set of all `using`'d modules visible +in the current world age. See [the manual chapter on world age](@ref man-worldage) for more +details. + ### Default top-level definitions and bare modules Modules automatically contain `using Core`, `using Base`, and definitions of the [`eval`](@ref) diff --git a/doc/src/manual/worldage.md b/doc/src/manual/worldage.md new file mode 100644 index 0000000000000..378b5a1b8f4e9 --- /dev/null +++ b/doc/src/manual/worldage.md @@ -0,0 +1,295 @@ +# The World Age mechanism + +!!! note + World age is an advanced concept. For the vast majority of Julia users, the world age + mechanism operates invisibly in the background. This documentation is intended for the + few users who may encounter world-age related issues or error messages. + +!!! compat "Julia 1.12" + Prior to Julia 1.12, the world age mechanism did not apply to changes to the global binding table. + The documentation in this chapter is specific to Julia 1.12+. + +!!! warning + This manual chapter uses internal functions to introspect world age and runtime data structures + as an explanatory aid. In general, unless otherwise noted the world age mechanism is not a stable + interface and should be interacted with in packages through stable APIs (e.g. `invokelatest`) only. + In particular, do not assume that world ages are always integers or that they have a linear order. + +## World age in general + +The "world age counter" is a monotonically increasing counter that is incremented for every +change to the global method table or the global binding table (e.g. through method definition, +type definition, `import`/`using` declaration, creation of (typed) globals or definition of constants). + +The current value of the global world age counter can be retrieved using the (internal) function [`Base.get_world_counter`](@ref). + +```julia-repl +julia> Base.get_world_counter() +0x0000000000009632 + +julia> const x = 1 + +julia> Base.get_world_counter() +0x0000000000009633 +``` + +In addition, each [`Task`](@ref) stores a local world age that determines which modifications to +the global binding and method tables are currently visible to the running task. The world age of +the running task will never exceed the global world age counter, but may run arbitrarily behind it. +In general the term "current world age" refers to the local world age of the currently running task. +The current world age may be retrieved using the (internal) function [`Base.tls_world_age`](@ref) + +```julia-repl +julia> function f end +f (generic function with 0 methods) + +julia> begin + @show (Int(Base.get_world_counter()), Int(Base.tls_world_age())) + Core.eval(@__MODULE__, :(f() = 1)) + @show (Int(Base.get_world_counter()), Int(Base.tls_world_age())) + f() + end +(Int(Base.get_world_counter()), Int(Base.tls_world_age())) = (38452, 38452) +(Int(Base.get_world_counter()), Int(Base.tls_world_age())) = (38453, 38452) +ERROR: MethodError: no method matching f() +The applicable method may be too new: running in current world age 38452, while global world is 38453. + +Closest candidates are: + f() (method too new to be called from this world context.) + @ Main REPL[2]:3 + +Stacktrace: + [1] top-level scope + @ REPL[2]:5 + +julia> (f(), Int(Base.tls_world_age())) +(1, 38453) +``` + +Here the definition of the method `f` raised the global world counter, but the current world +age did not change. As a result, the definition of `f` was not visible in the currently +executing task and a [`MethodError`](@ref) resulted. + +!!! note + The method error printing provided additional information that `f()` is available in a newer world age. + This information is added by the error display, not the task that threw the `MethodError`. + The thrown `MethodError` is identical whether or not a matching definition of `f()` exists + in a newer world age. + +However, note that the definition of `f()` was subsequently available at the next REPL prompt, because +the current task's world age had been raised. In general, certain syntactic constructs (in particular most definitions) +will raise the current task's world age to the latest global world age, thus making all changes +(both from the current task and any concurrently executing other tasks) visible. The following statements +raise the current world age: + + 1. An explicit invocation of `Core.@latestworld` + 2. The start of every top-level statement + 3. The start of every REPL prompt + 4. Any type or struct definition + 5. Any method definition + 6. Any constant declaration + 7. Any global variable declaration (but not a global variable assignment) + 8. Any `using`, `import`, `export` or `public` statement + 9. Certain other macros like [`@eval`](@ref) (depends on the macro implementation) + +Note, however, that the current task's world age may only ever be permanently incremented at +top level. As a general rule, using any of the above statements in non-top-level scope is a syntax error: + +```julia-repl +julia> f() = Core.@latestworld +ERROR: syntax: World age increment not at top level +Stacktrace: + [1] top-level scope + @ REPL[5]:1 +``` + +When it isn't (for example for `@eval`), the world age side effect is ignored. + +As a result of these rules, Julia may assume that the world age does not change +within the execution of an ordinary function. + +```julia +function my_function() + before = Base.tls_world_age() + # Any arbitrary code + after = Base.tls_world_age() + @assert before === after # always true +end +``` + +This is the key invariant that allows Julia to optimize based on the current state +of its global data structures, while still having the well-defined ability to change +these data structures. + +## Temporarily raising the world age using `invokelatest` + +As described above, it is not possible to permanently raise the world age for the remainder of +a `Task`'s execution unless the task is executing top-level statements. However, it is possible to +temporarily change the world age in a scoped manner using `invokelatest`: + +```jldoctest +julia> function f end +f (generic function with 0 methods) + +julia> begin + Core.eval(@__MODULE__, :(f() = 1)) + invokelatest(f) + end +1 +``` + +`invokelatest` will temporarily raise the current task's world age to the latest global world age (at +entry to `invokelatest`) and execute the provided function. Note that the world age will return +to its prior value upon exit from `invokelatest`. + +## World age and const struct redefinitions + +The semantics described above for method redefinition also apply to redefinition of constants: + +```jldoctest +julia> const x = 1 +1 + +julia> get_const() = x +get_const (generic function with 1 method) + +julia> begin + @show get_const() + Core.eval(@__MODULE__, :(const x = 2)) + @show get_const() + Core.@latestworld + @show get_const() + end +get_const() = 1 +get_const() = 1 +get_const() = 2 +2 +``` + +However, for the avoidance of doubt, they do not apply to ordinary assignment to global variables, which becomes visible immediately: +```jldoctest +julia> global y = 1 +1 + +julia> get_global() = y +get_global (generic function with 1 method) + +julia> begin + @show get_global() + Core.eval(@__MODULE__, :(y = 2)) + @show get_global() + end +get_global() = 1 +get_global() = 2 +2 +``` + +One particular special case of constant reassignment is the redefinition of struct types: + +```jldoctest; filter = r"\@world\(MyStruct, \d+\:\d+\)" +julia> struct MyStruct + x::Int + end + +julia> const one_field = MyStruct(1) +MyStruct(1) + +julia> struct MyStruct + x::Int + y::Float64 + end + +julia> const two_field = MyStruct(1, 2.0) +MyStruct(1, 2.0) + +julia> one_field +@world(MyStruct, 38452:38455)(1) + +julia> two_field +MyStruct(1, 2.0) +``` + +Internally the two definitions of `MyStruct` are entirely separate types. However, +after the new `MyStruct` type is defined, there is no longer any default binding +for the original definition of `MyStruct`. To nevertheless facilitate access to +these types, the special [`@world`](@ref) macro may be used to access the meaning +of a name in a previous world. However, this facility is intended for introspection +only and in particular note that world age numbers are not stable across precompilation +and should in general be treated opaquely. + +### Binding partition introspection + +In certain cases, it can be helpful to introspect the system's understanding of what +a binding means in any particular world age. The default display printing of `Core.Binding` +provides a helpful summary (e.g. on the `MyStruct` example from above): + +```julia-repl +julia> convert(Core.Binding, GlobalRef(@__MODULE__, :MyStruct)) +Binding Main.MyStruct + 38456:∞ - constant binding to MyStruct + 38452:38455 - constant binding to @world(MyStruct, 38452:38455) + 38451:38451 - backdated constant binding to @world(MyStruct, 38452:38455) + 0:38450 - backdated constant binding to @world(MyStruct, 38452:38455) +``` + +## World age and `using`/`import` + +Bindings provided via `using` and `import` also operate via the world age mechanism. +Binding resolution is a stateless function of the `import` and `using` definitions +visible in the current world age. For example: + +```julia-repl +julia> module M1; const x = 1; export x; end + +julia> module M2; const x = 2; export x; end + +julia> using .M1 + +julia> x +1 + +julia> using .M2 + +julia> x +ERROR: UndefVarError: `x` not defined in `Main` +Hint: It looks like two or more modules export different bindings with this name, resulting in ambiguity. Try explicitly importing it from a particular module, or qualifying the name with the module it should come from. + +julia> convert(Core.Binding, GlobalRef(@__MODULE__, :x)) +Binding Main.x + 38458:∞ - ambiguous binding - guard entry + 38457:38457 - implicit `using` resolved to constant 1 +``` + +## World age capture + +Certain language features capture the current task's world age. Perhaps the most common of +these is creation of new tasks. Newly created tasks will inherit the creating task's local +world age at creation time and will retain said world age (unless explicitly raised) even +if the originating tasks raises its world age: + +```julia-repl +julia> const x = 1 + +julia> t = @task (wait(); println("Running now"); x); + +julia> const x = 2 + +julia> schedule(t); +Running now + +julia> x +2 + +julia> fetch(t) +1 +``` + +In addition to tasks, opaque closures also capture their world age at creation. See [`Base.Experimental.@opaque`](@ref). + +```@docs +Base.@world +Base.get_world_counter +Base.tls_world_age +Base.invoke_in_world +Base.Experimental.@opaque +``` From 3864b18af6a066400a1cb89b01902ef289fb524b Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Apr 2025 18:35:27 +0200 Subject: [PATCH 173/662] Reland #57979 (and following #58083 #58082) (#58203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts JuliaLang/julia#58182. The API changes were intentional and desirable. Let's figure out why nansoldier was upset and re-apply this. --------- Co-authored-by: Cédric Belmant --- .../CompilerDevTools/src/CompilerDevTools.jl | 2 +- Compiler/src/optimize.jl | 98 ++++-------- Compiler/src/ssair/inlining.jl | 2 +- Compiler/src/typeinfer.jl | 146 +++++++++++++++--- Compiler/test/codegen.jl | 1 + Compiler/test/inline.jl | 4 + 6 files changed, 165 insertions(+), 88 deletions(-) diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index e08d08779e097..ddf202f378fb5 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -47,7 +47,7 @@ end function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) opt = result.src::Compiler.OptimizationState - ir = opt.result.ir::Compiler.IRCode + ir = opt.optresult.ir::Compiler.IRCode override = with_new_compiler for inst in ir.stmts stmt = inst[:stmt] diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 612105061c38d..fdf97c447559d 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -116,11 +116,14 @@ function inline_cost_clamp(x::Int) return convert(InlineCostType, x) end +const SRC_FLAG_DECLARED_INLINE = 0x1 +const SRC_FLAG_DECLARED_NOINLINE = 0x2 + is_declared_inline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 1 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE is_declared_noinline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 2 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE ##################### # OptimizationState # @@ -157,6 +160,7 @@ code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.wor mutable struct OptimizationResult ir::IRCode + inline_flag::UInt8 simplified::Bool # indicates whether the IR was processed with `cfg_simplify!` end @@ -168,7 +172,7 @@ end mutable struct OptimizationState{Interp<:AbstractInterpreter} linfo::MethodInstance src::CodeInfo - result::Union{Nothing, OptimizationResult} + optresult::Union{Nothing, OptimizationResult} stmt_info::Vector{CallInfo} mod::Module sptypes::Vector{VarState} @@ -236,13 +240,29 @@ include("ssair/EscapeAnalysis.jl") include("ssair/passes.jl") include("ssair/irinterp.jl") +function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector) + ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult)) +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType) + src = ir_to_codeinf!(opt, edges) + src.inlining_cost = inlining_cost + src +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector) + src = ir_to_codeinf!(opt) + src.edges = edges + src +end + function ir_to_codeinf!(opt::OptimizationState) - (; linfo, src, result) = opt - if result === nothing + (; linfo, src, optresult) = opt + if optresult === nothing return src end - src = ir_to_codeinf!(src, result.ir) - opt.result = nothing + src = ir_to_codeinf!(src, optresult.ir) + opt.optresult = nothing opt.src = src maybe_validate_code(linfo, src, "optimized") return src @@ -485,63 +505,12 @@ end abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = types(src)[s] """ - finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) + finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) -Post-process information derived by Julia-level optimizations for later use. -In particular, this function determines the inlineability of the optimized code. +Called at the end of optimization to store the resulting IR back into the OptimizationState. """ -function finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) - (; src, linfo) = opt - (; def, specTypes) = linfo - - force_noinline = is_declared_noinline(src) - - # compute inlining and other related optimizations - result = caller.result - @assert !(result isa LimitedAccuracy) - result = widenslotwrapper(result) - - opt.result = OptimizationResult(ir, false) - - # determine and cache inlineability - if !force_noinline - sig = unwrap_unionall(specTypes) - if !(isa(sig, DataType) && sig.name === Tuple.name) - force_noinline = true - end - if !is_declared_inline(src) && result === Bottom - force_noinline = true - end - end - if force_noinline - set_inlineable!(src, false) - elseif isa(def, Method) - if is_declared_inline(src) && isdispatchtuple(specTypes) - # obey @inline declaration if a dispatch barrier would not help - set_inlineable!(src, true) - else - # compute the cost (size) of inlining this code - params = OptimizationParams(interp) - cost_threshold = default = params.inline_cost_threshold - if ⊑(optimizer_lattice(interp), result, Tuple) && !isconcretetype(widenconst(result)) - cost_threshold += params.inline_tupleret_bonus - end - # if the method is declared as `@inline`, increase the cost threshold 20x - if is_declared_inline(src) - cost_threshold += 19*default - end - # a few functions get special treatment - if def.module === _topmod(def.module) - name = def.name - if name === :iterate || name === :unsafe_convert || name === :cconvert - cost_threshold += 4*default - end - end - src.inlining_cost = inline_cost(ir, params, cost_threshold) - end - end +function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) + opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false) return nothing end @@ -1015,7 +984,8 @@ end function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) - return finish(interp, opt, ir, caller) + finishopt!(interp, opt, ir) + return nothing end const ALL_PASS_NAMES = String[] @@ -1466,7 +1436,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod return thiscost end -function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Int) +function inline_cost_model(ir::IRCode, params::OptimizationParams, cost_threshold::Int) bodycost = 0 for i = 1:length(ir.stmts) stmt = ir[SSAValue(i)][:stmt] diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 3830197aef458..c7e052ed17218 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -976,7 +976,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, ir::IRCode, preserve_local return ir, spec_info, DebugInfo(ir.debuginfo, length(ir.stmts)) end function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, preserve_local_sources::Bool) - result = opt.result + result = opt.optresult if result !== nothing !result.simplified && simplify_ir!(result) return retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index a5b87c86a3ac2..d2709259c8824 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -104,7 +104,10 @@ end function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) - if isdefined(result, :ci) + if caller.cache_mode === CACHE_MODE_LOCAL + @assert !isdefined(result, :ci) + result.src = transform_result_for_local_cache(interp, result) + elseif isdefined(result, :ci) edges = result_edges(interp, caller) ci = result.ci # if we aren't cached, we don't need this edge @@ -115,21 +118,31 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation store_backedges(ci, edges) end inferred_result = nothing - uncompressed = inferred_result + uncompressed = result.src const_flag = is_result_constabi_eligible(result) + debuginfo = nothing discard_src = caller.cache_mode === CACHE_MODE_NULL || const_flag if !discard_src inferred_result = transform_result_for_cache(interp, result, edges) + if inferred_result !== nothing + uncompressed = inferred_result + debuginfo = get_debuginfo(inferred_result) + # Inlining may fast-path the global cache via `VolatileInferenceResult`, so store it back here + result.src = inferred_result + else + if isa(result.src, OptimizationState) + debuginfo = get_debuginfo(ir_to_codeinf!(result.src)) + elseif isa(result.src, CodeInfo) + debuginfo = get_debuginfo(result.src) + end + end # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? if inferred_result isa CodeInfo - result.src = inferred_result if may_compress(interp) nslots = length(inferred_result.slotflags) resize!(inferred_result.slottypes::Vector{Any}, nslots) resize!(inferred_result.slotnames, nslots) end - di = inferred_result.debuginfo - uncompressed = inferred_result inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing @@ -137,18 +150,21 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation inferred_result = nothing end end - if !@isdefined di - di = DebugInfo(result.linfo) + if debuginfo === nothing + debuginfo = DebugInfo(result.linfo) end time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), - result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, di, edges) + result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) engine_reject(interp, ci) codegen = codegen_cache(interp) - if !discard_src && codegen !== nothing && uncompressed isa CodeInfo + if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) + if isa(uncompressed, OptimizationState) + uncompressed = ir_to_codeinf!(uncompressed, edges) + end # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work codegen[ci] = uncompressed if bootstrapping_compiler && inferred_result == nothing @@ -299,36 +315,123 @@ function adjust_cycle_frame!(sv::InferenceState, cycle_valid_worlds::WorldRange, return nothing end +function get_debuginfo(src) + isa(src, CodeInfo) && return src.debuginfo + isa(src, OptimizationState) && return src.src.debuginfo + return nothing +end + function is_result_constabi_eligible(result::InferenceResult) result_type = result.result return isa(result_type, Const) && is_foldable_nothrow(result.ipo_effects) && is_inlineable_constant(result_type.val) end -function transform_result_for_cache(::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult) + src = result.src + isa(src, OptimizationState) || return MAX_INLINE_COST + compute_inlining_cost(interp, result, src.optresult) +end + +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult, optresult#=::OptimizationResult=#) + return inline_cost_model(interp, result, optresult.inline_flag, optresult.ir) +end + +function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, + inline_flag::UInt8, ir::IRCode) + + inline_flag === SRC_FLAG_DECLARED_NOINLINE && return MAX_INLINE_COST + + mi = result.linfo + (; def, specTypes) = mi + if !isa(def, Method) + return MAX_INLINE_COST + end + + declared_inline = inline_flag === SRC_FLAG_DECLARED_INLINE + + rt = result.result + @assert !(rt isa LimitedAccuracy) + rt = widenslotwrapper(rt) + + sig = unwrap_unionall(specTypes) + if !(isa(sig, DataType) && sig.name === Tuple.name) + return MAX_INLINE_COST + end + if !declared_inline && rt === Bottom + return MAX_INLINE_COST + end + + if declared_inline && isdispatchtuple(specTypes) + # obey @inline declaration if a dispatch barrier would not help + return MIN_INLINE_COST + else + # compute the cost (size) of inlining this code + params = OptimizationParams(interp) + cost_threshold = default = params.inline_cost_threshold + if ⊑(optimizer_lattice(interp), rt, Tuple) && !isconcretetype(widenconst(rt)) + cost_threshold += params.inline_tupleret_bonus + end + # if the method is declared as `@inline`, increase the cost threshold 20x + if declared_inline + cost_threshold += 19*default + end + # a few functions get special treatment + if def.module === _topmod(def.module) + name = def.name + if name === :iterate || name === :unsafe_convert || name === :cconvert + cost_threshold += 4*default + end + end + return inline_cost_model(ir, params, cost_threshold) + end +end + +function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) + if is_result_constabi_eligible(result) + return nothing + end + src = result.src + if isa(src, OptimizationState) + # Compute and store any information required to determine the inlineability of the callee. + opt = src + opt.src.inlining_cost = compute_inlining_cost(interp, result) + end + return src +end + +function transform_result_for_cache(interp::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) + inlining_cost = nothing src = result.src if isa(src, OptimizationState) - src = ir_to_codeinf!(src) + opt = src + inlining_cost = compute_inlining_cost(interp, result, opt.optresult) + discard_optimized_result(interp, opt, inlining_cost) && return nothing + src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) src.edges = edges + if inlining_cost !== nothing + src.inlining_cost = inlining_cost + elseif may_optimize(interp) + src.inlining_cost = compute_inlining_cost(interp, result) + end end return src end +function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) + may_discard_trees(interp) || return false + return inlining_cost == MAX_INLINE_COST +end + function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo) def = mi.def isa(def, Method) || return ci # don't compress toplevel code can_discard_trees = may_discard_trees(interp) cache_the_tree = !can_discard_trees || is_inlineable(ci) - if cache_the_tree - if may_compress(interp) - return ccall(:jl_compress_ir, String, (Any, Any), def, ci) - else - return ci - end - else - return nothing - end + cache_the_tree || return nothing + may_compress(interp) && return ccall(:jl_compress_ir, String, (Any, Any), def, ci) + return ci end function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) @@ -1116,8 +1219,7 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - src = ir_to_codeinf!(opt) - src.edges = Core.svec(opt.inlining.edges...) + src = ir_to_codeinf!(opt, frame, Core.svec(opt.inlining.edges...)) end result.src = frame.src = src end diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index fd8bbae70a346..45db9e73d5a3f 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -4,6 +4,7 @@ using Random using InteractiveUtils +using InteractiveUtils: code_llvm, code_native using Libdl using Test diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index 92e389ff0dc04..0a88907965f5a 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +module inline_tests + using Test using Base.Meta using Core: ReturnNode @@ -2311,3 +2313,5 @@ g_noinline_invoke(x) = f_noinline_invoke(x) let src = code_typed1(g_noinline_invoke, (Union{Symbol,Nothing},)) @test !any(@nospecialize(x)->isa(x,GlobalRef), src.code) end + +end # module inline_tests From ccef01a75367acede2f7325416e54407a8255408 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Wed, 30 Apr 2025 11:17:50 -0700 Subject: [PATCH 174/662] Print fallback REPL prompt before user input (#58210) Using the fallback REPL interactively prints the prompt only after user input, making the scrollback confusing: ``` $ JULIA_FALLBACK_REPL=1 julia 1 + 2 julia> 3 ``` --------- Co-authored-by: Jameson Nash Co-authored-by: Dilum Aluthge --- base/client.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/client.jl b/base/client.jl index 853402c916d12..5bf658bead437 100644 --- a/base/client.jl +++ b/base/client.jl @@ -442,11 +442,12 @@ function run_fallback_repl(interactive::Bool) eval_user_input(stderr, ex, true) end else - while !eof(input) + while true if interactive print("julia> ") flush(stdout) end + eof(input) && break try line = "" ex = nothing From ea6a465488d6a2266cd63f316dffe790d31266ba Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 3 Apr 2025 16:15:01 -0400 Subject: [PATCH 175/662] trim: Add `Core.finalizer` support This adds the changes required to guess the MethodInstance that the runtime will eventually invoke for this call, enqueue it for codegen, and then verify its presence in the verifier. --- Compiler/src/typeinfer.jl | 54 +++++++++++++++++++++++++++++++++++--- Compiler/src/verifytrim.jl | 14 ++++++++-- base/runtime_internals.jl | 8 +++++- src/gf.c | 1 + 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index d2709259c8824..f5433ddb9f4a2 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1374,8 +1374,29 @@ function typeinf_type(interp::AbstractInterpreter, mi::MethodInstance) return ci.rettype end +# Resolve a call, as described by `argtype` to a single matching +# Method and return a compilable MethodInstance for the call, if +# it will be runtime-dispatched to exactly that MethodInstance +function compileable_specialization_for_call(interp::AbstractInterpreter, @nospecialize(argtype)) + matches = findall(argtype, method_table(interp); limit = 1) + matches === nothing && return nothing + length(matches.matches) == 0 && return nothing + match = only(matches.matches) + + compileable_atype = get_compileable_sig(match.method, match.spec_types, match.sparams) + compileable_atype === nothing && return nothing + if match.spec_types !== compileable_atype + sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), compileable_atype, method.sig)::SimpleVector + sparams = sp_[2]::SimpleVector + mi = specialize_method(match.method, compileable_atype, sparams) + else + mi = specialize_method(match.method, compileable_atype, match.sparams) + end + return mi +end + # collect a list of all code that is needed along with CodeInstance to codegen it fully -function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo) +function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo, sptypes::Vector{VarState}) src = ci.code for i = 1:length(src) stmt = src[i] @@ -1384,6 +1405,31 @@ function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo) edge = stmt.args[1] edge isa CodeInstance && isdefined(edge, :inferred) && push!(wq, edge) end + + if isexpr(stmt, :call) + farg = stmt.args[1] + !applicable(argextype, farg, ci, sptypes) && continue # TODO: Why is this failing during bootstrap + ftyp = widenconst(argextype(farg, ci, sptypes)) + if ftyp <: Builtin + # TODO: Make interp elsewhere + interp = NativeInterpreter(Base.get_world_counter()) + if Core.finalizer isa ftyp && length(stmt.args) == 3 + finalizer = argextype(stmt.args[2], ci, sptypes) + obj = argextype(stmt.args[3], ci, sptypes) + atype = argtypes_to_type(Any[finalizer, obj]) + + mi = compileable_specialization_for_call(interp, atype) + mi === nothing && continue + + if mi.def.primary_world <= Base.get_world_counter() <= mi.def.deleted_world + ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) + # TODO: separate workqueue for NativeInterpreter + ci isa CodeInstance && push!(wq, ci) + end + # push!(wq, mi) + end + end + end # TODO: handle other StmtInfo like @cfunction and OpaqueClosure? end end @@ -1420,8 +1466,9 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn end end push!(inspected, callee) - collectinvokes!(tocompile, src) mi = get_ci_mi(callee) + sptypes = sptypes_from_meth_instance(mi) + collectinvokes!(tocompile, src, sptypes) if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(interp))::CodeInstance if cached === callee @@ -1519,7 +1566,8 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m end push!(inspected, callee) if src isa CodeInfo - collectinvokes!(tocompile, src) + sptypes = sptypes_from_meth_instance(mi) + collectinvokes!(tocompile, src, sptypes) # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, this_world)::CodeInstance diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 9b49c4b4500c1..298c52113d3a6 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -199,6 +199,8 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec if !may_dispatch(ftyp) continue end + # TODO: Make interp elsewhere + interp = NativeInterpreter(Base.get_world_counter()) if Core._apply_iterate isa ftyp if length(stmt.args) >= 3 # args[1] is _apply_iterate object @@ -219,9 +221,17 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec end elseif Core.finalizer isa ftyp if length(stmt.args) == 3 - # TODO: check that calling `args[1](args[2])` is defined before warning + finalizer = argextype(stmt.args[2], ci, sptypes) + obj = argextype(stmt.args[3], ci, sptypes) + atype = argtypes_to_type(Any[finalizer, obj]) + + mi = compileable_specialization_for_call(interp, atype) + if mi !== nothing + ci = get(caches, mi, nothing) + ci isa CodeInstance && continue + end + error = "unresolved finalizer registered" - warn = true end else error = "unresolved call to builtin" diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index d8c90b41842c8..9aae6b2a2b78c 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1622,12 +1622,18 @@ end is_nospecialized(method::Method) = method.nospecialize ≠ 0 is_nospecializeinfer(method::Method) = method.nospecializeinfer && is_nospecialized(method) + +""" +Return MethodInstance corresponding to `atype` and `sparams`. + +No widening / narrowing / compileable-normalization of `atype` is performed. +""" function specialize_method(method::Method, @nospecialize(atype), sparams::SimpleVector; preexisting::Bool=false) @inline if isa(atype, UnionAll) atype, sparams = normalize_typevars(method, atype, sparams) end - if is_nospecializeinfer(method) + if is_nospecializeinfer(method) # TODO: this shouldn't be here atype = get_nospecializeinfer_sig(method, atype, sparams) end if preexisting diff --git a/src/gf.c b/src/gf.c index b130fbe05ca6d..a42c73e618e83 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3211,6 +3211,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc } // compile-time method lookup +// intersect types with the MT, and return a single compileable specialization that covers the intersection. jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache) { if (jl_has_free_typevars((jl_value_t*)types)) From e4e86a2a6e7442405749526545d254a55a0ec5a5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 4 Apr 2025 15:15:57 -0400 Subject: [PATCH 176/662] some fixes --- Compiler/src/typeinfer.jl | 2 +- Compiler/src/verifytrim.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index f5433ddb9f4a2..d79007949d0f9 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1386,7 +1386,7 @@ function compileable_specialization_for_call(interp::AbstractInterpreter, @nospe compileable_atype = get_compileable_sig(match.method, match.spec_types, match.sparams) compileable_atype === nothing && return nothing if match.spec_types !== compileable_atype - sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), compileable_atype, method.sig)::SimpleVector + sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), compileable_atype, match.method.sig)::SimpleVector sparams = sp_[2]::SimpleVector mi = specialize_method(match.method, compileable_atype, sparams) else diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 298c52113d3a6..19d8bcd623275 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import ..Compiler: verify_typeinf_trim +import ..Compiler: verify_typeinf_trim, NativeInterpreter, argtypes_to_type, compileable_specialization_for_call using ..Compiler: # operators @@ -221,8 +221,8 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec end elseif Core.finalizer isa ftyp if length(stmt.args) == 3 - finalizer = argextype(stmt.args[2], ci, sptypes) - obj = argextype(stmt.args[3], ci, sptypes) + finalizer = argextype(stmt.args[2], codeinfo, sptypes) + obj = argextype(stmt.args[3], codeinfo, sptypes) atype = argtypes_to_type(Any[finalizer, obj]) mi = compileable_specialization_for_call(interp, atype) From d34fb93e4a2837b04a6662295ed9550f8f281005 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Fri, 4 Apr 2025 15:55:13 -0400 Subject: [PATCH 177/662] typeinfer: add separate compile workqueue for native / call_latest code --- Compiler/src/typeinfer.jl | 210 ++++++++++++++++++++++++-------------- 1 file changed, 131 insertions(+), 79 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index d79007949d0f9..c3a6e40da937c 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1378,6 +1378,12 @@ end # Method and return a compilable MethodInstance for the call, if # it will be runtime-dispatched to exactly that MethodInstance function compileable_specialization_for_call(interp::AbstractInterpreter, @nospecialize(argtype)) + mt = ccall(:jl_method_table_for, Any, (Any,), argtype) + if mt === nothing + # this would require scanning all method tables, so give up instead + return nothing + end + matches = findall(argtype, method_table(interp); limit = 1) matches === nothing && return nothing length(matches.matches) == 0 && return nothing @@ -1392,18 +1398,44 @@ function compileable_specialization_for_call(interp::AbstractInterpreter, @nospe else mi = specialize_method(match.method, compileable_atype, match.sparams) end + return mi end +const QueueItems = Union{CodeInstance,MethodInstance,SimpleVector} + +struct CompilationQueue + tocompile::Vector{QueueItems} + inspected::IdSet{QueueItems} + interp::Union{AbstractInterpreter,Nothing} + + CompilationQueue(; + interp::Union{AbstractInterpreter,Nothing} + ) = new(QueueItems[], IdSet{QueueItems}(), interp) + + CompilationQueue(queue::CompilationQueue; + interp::Union{AbstractInterpreter,Nothing} + ) = new(empty!(queue.tocompile), empty!(queue.inspected), interp) +end + +Base.push!(queue::CompilationQueue, item) = push!(queue.tocompile, item) +Base.append!(queue::CompilationQueue, items) = append!(queue.tocompile, items) +Base.pop!(queue::CompilationQueue) = pop!(queue.tocompile) +Base.empty!(queue::CompilationQueue) = (empty!(queue.tocompile); empty!(queue.inspected)) +markinspected!(queue::CompilationQueue, item) = push!(queue.inspected, item) +isinspected(queue::CompilationQueue, item) = item in queue.inspected +Base.isempty(queue::CompilationQueue) = isempty(queue.tocompile) + # collect a list of all code that is needed along with CodeInstance to codegen it fully -function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo, sptypes::Vector{VarState}) +function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vector{VarState}; + invokelatest_queue::Union{CompilationQueue,Nothing} = nothing) src = ci.code for i = 1:length(src) stmt = src[i] isexpr(stmt, :(=)) && (stmt = stmt.args[2]) if isexpr(stmt, :invoke) || isexpr(stmt, :invoke_modify) edge = stmt.args[1] - edge isa CodeInstance && isdefined(edge, :inferred) && push!(wq, edge) + edge isa CodeInstance && isdefined(edge, :inferred) && push!(workqueue, edge) end if isexpr(stmt, :call) @@ -1411,22 +1443,19 @@ function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo, sptypes::Vector !applicable(argextype, farg, ci, sptypes) && continue # TODO: Why is this failing during bootstrap ftyp = widenconst(argextype(farg, ci, sptypes)) if ftyp <: Builtin - # TODO: Make interp elsewhere - interp = NativeInterpreter(Base.get_world_counter()) if Core.finalizer isa ftyp && length(stmt.args) == 3 finalizer = argextype(stmt.args[2], ci, sptypes) obj = argextype(stmt.args[3], ci, sptypes) atype = argtypes_to_type(Any[finalizer, obj]) - mi = compileable_specialization_for_call(interp, atype) - mi === nothing && continue + invokelatest_queue === nothing && continue + let workqueue = invokelatest_queue + # make a best-effort attempt to enqueue the relevant code for the finalizer + mi = compileable_specialization_for_call(workqueue.interp, atype) + mi === nothing && continue - if mi.def.primary_world <= Base.get_world_counter() <= mi.def.deleted_world - ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) - # TODO: separate workqueue for NativeInterpreter - ci isa CodeInstance && push!(wq, ci) + push!(workqueue, mi) end - # push!(wq, mi) end end end @@ -1439,14 +1468,13 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn ci isa CodeInstance && !ci_has_invoke(ci) || return ci codegen = codegen_cache(interp) codegen === nothing && return ci - inspected = IdSet{CodeInstance}() - tocompile = Vector{CodeInstance}() - push!(tocompile, ci) - while !isempty(tocompile) + workqueue = CompilationQueue(; interp) + push!(workqueue, ci) + while !isempty(workqueue) # ci_has_real_invoke(ci) && return ci # optimization: cease looping if ci happens to get compiled (not just jl_fptr_wait_for_compiled, but fully jl_is_compiled_codeinst) - callee = pop!(tocompile) + callee = pop!(workqueue) ci_has_invoke(callee) && continue - callee in inspected && continue + isinspected(workqueue, callee) && continue src = get(codegen, callee, nothing) if !isa(src, CodeInfo) src = @atomic :monotonic callee.inferred @@ -1454,26 +1482,26 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn src = _uncompressed_ir(callee, src) end if !isa(src, CodeInfo) - newcallee = typeinf_ext(interp, callee.def, source_mode) # always SOURCE_MODE_ABI + newcallee = typeinf_ext(workqueue.interp, callee.def, source_mode) # always SOURCE_MODE_ABI if newcallee isa CodeInstance callee === ci && (ci = newcallee) # ci stopped meeting the requirements after typeinf_ext last checked, try again with newcallee - push!(tocompile, newcallee) + push!(workqueue, newcallee) end if newcallee !== callee - push!(inspected, callee) + markinspected!(workqueue, callee) end continue end end - push!(inspected, callee) + markinspected!(workqueue, callee) mi = get_ci_mi(callee) sptypes = sptypes_from_meth_instance(mi) - collectinvokes!(tocompile, src, sptypes) + collectinvokes!(workqueue, src, sptypes) if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(interp))::CodeInstance + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance if cached === callee # make sure callee is gc-rooted and cached, as required by jl_add_codeinst_to_jit - code_cache(interp)[mi] = callee + code_cache(workqueue.interp)[mi] = callee else # use an existing CI from the cache, if there is available one that is compatible callee === ci && (ci = cached) @@ -1497,57 +1525,45 @@ function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt return typeinf_ext_toplevel(interp, mi, source_mode) end -# This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches -# The trim_mode can be any of: -const TRIM_NO = 0 -const TRIM_SAFE = 1 -const TRIM_UNSAFE = 2 -const TRIM_UNSAFE_WARN = 3 -function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) - inspected = IdSet{CodeInstance}() - tocompile = Vector{CodeInstance}() - codeinfos = [] - # first compute the ABIs of everything - latest = true # whether this_world == world_counter() - for this_world in reverse(sort!(worlds)) - interp = NativeInterpreter( - this_world; - inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) - ) - for i = 1:length(methods) - # each item in this list is either a MethodInstance indicating something - # to compile, or an svec(rettype, sig) describing a C-callable alias to create. - item = methods[i] - if item isa MethodInstance - # if this method is generally visible to the current compilation world, - # and this is either the primary world, or not applicable in the primary world - # then we want to compile and emit this - if item.def.primary_world <= this_world <= item.def.deleted_world - ci = typeinf_ext(interp, item, SOURCE_MODE_GET_SOURCE) - ci isa CodeInstance && push!(tocompile, ci) - end - elseif item isa SimpleVector && latest - (rt::Type, sig::Type) = item - # make a best-effort attempt to enqueue the relevant code for the ccallable - ptr = ccall(:jl_get_specialization1, - #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), - sig, this_world, #= mt_cache =# 0) - if ptr !== C_NULL - mi = unsafe_pointer_to_objref(ptr)::MethodInstance - ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) - ci isa CodeInstance && push!(tocompile, ci) - end - # additionally enqueue the ccallable entrypoint / adapter, which implicitly - # invokes the above ci - push!(codeinfos, item) +function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; + invokelatest_queue::Union{CompilationQueue,Nothing} = nothing, +) + interp = workqueue.interp + world = get_inference_world(interp) + while !isempty(workqueue) + item = pop!(workqueue) + # each item in this list is either a MethodInstance indicating something + # to compile, or an svec(rettype, sig) describing a C-callable alias to create. + if item isa MethodInstance + isinspected(workqueue, item) && continue + # if this method is generally visible to the current compilation world, + # and this is either the primary world, or not applicable in the primary world + # then we want to compile and emit this + if item.def.primary_world <= world <= item.def.deleted_world + ci = typeinf_ext(interp, item, SOURCE_MODE_GET_SOURCE) + ci isa CodeInstance && push!(workqueue, ci) end - end - while !isempty(tocompile) - callee = pop!(tocompile) - callee in inspected && continue - # now make sure everything has source code, if desired + markinspected!(workqueue, item) + elseif item isa SimpleVector + invokelatest_queue === nothing && continue + (rt::Type, sig::Type) = item + # make a best-effort attempt to enqueue the relevant code for the ccallable + ptr = ccall(:jl_get_specialization1, + #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), + sig, world, #= mt_cache =# 0) + if ptr !== C_NULL + mi = unsafe_pointer_to_objref(ptr)::MethodInstance + ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) + ci isa CodeInstance && push!(invokelatest_queue, ci) + end + # additionally enqueue the ccallable entrypoint / adapter, which implicitly + # invokes the above ci + push!(codeinfos, item) + elseif item isa CodeInstance + callee = item + isinspected(workqueue, callee) && continue mi = get_ci_mi(callee) - def = mi.def + # now make sure everything has source code, if desired if use_const_api(callee) src = codeinfo_for_const(interp, mi, callee.rettype_const) else @@ -1556,21 +1572,21 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m newcallee = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) if newcallee isa CodeInstance @assert use_const_api(newcallee) || haskey(interp.codegen, newcallee) - push!(tocompile, newcallee) + push!(workqueue, newcallee) end if newcallee !== callee - push!(inspected, callee) + markinspected!(workqueue, callee) end continue end end - push!(inspected, callee) + markinspected!(workqueue, callee) if src isa CodeInfo sptypes = sptypes_from_meth_instance(mi) - collectinvokes!(tocompile, src, sptypes) + collectinvokes!(workqueue, src, sptypes; invokelatest_queue) # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, this_world)::CodeInstance + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, world)::CodeInstance if cached === callee code_cache(interp)[mi] = callee else @@ -1581,9 +1597,45 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m push!(codeinfos, callee) push!(codeinfos, src) end + else @assert false "unexpected item in queue" end + end + return codeinfos +end + +# This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches +# The trim_mode can be any of: +const TRIM_NO = 0 +const TRIM_SAFE = 1 +const TRIM_UNSAFE = 2 +const TRIM_UNSAFE_WARN = 3 +function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + invokelatest_queue = CompilationQueue(; + interp = NativeInterpreter(get_world_counter(); inf_params) + ) + codeinfos = [] + is_latest_world = true # whether this_world == world_counter() + workqueue = CompilationQueue(; interp = nothing) + for (i, this_world) in enumerate(sort!(worlds)) + workqueue = CompilationQueue(workqueue; + interp = NativeInterpreter(this_world; inf_params) + ) + + append!(workqueue, methods) + + is_latest_world = (i == length(worlds)) + if is_latest_world + # Provide the `invokelatest` queue so that we trigger "best-effort" code generation + # for, e.g., finalizers and cfunction. + # + # The queue is intentionally aliased, to handle e.g. a `finalizer` calling `Core.finalizer` + # (it will enqueue into itself and immediately drain) + compile!(codeinfos, workqueue; invokelatest_queue = workqueue) + else + compile!(codeinfos, workqueue) end - latest = false end + if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE verify_typeinf_trim(codeinfos, trim_mode == TRIM_UNSAFE_WARN) end From 27a8799793e7b4584ff8dfd4cb008451a0c03a0b Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 8 Apr 2025 22:09:58 -0400 Subject: [PATCH 178/662] Make `empty!(::BitSet)` work during bootstrap --- base/idset.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/base/idset.jl b/base/idset.jl index c2f26b528020d..963da8e7c0f92 100644 --- a/base/idset.jl +++ b/base/idset.jl @@ -92,8 +92,17 @@ function sizehint!(s::IdSet, newsz) nothing end +function _zero!(a::Memory{<:BitInteger}) + t = @_gc_preserve_begin a + p = unsafe_convert(Ptr{Cvoid}, a) + T = eltype(a) + memset(p, 0x0, (sizeof(T) * length(a)) % UInt) + @_gc_preserve_end t + return a +end + function empty!(s::IdSet) - fill!(s.idxs, 0x00) + _zero!(s.idxs) list = s.list for i = 1:s.max _unsetindex!(list, i) From ce56baa3b54dde4e3dc96da5571daf5841d4295f Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Fri, 11 Apr 2025 20:39:15 -0400 Subject: [PATCH 179/662] Collect invoke only for `ftyp === typeof(finalizer)` Otherwise this will match `ftyp === Builtin`, which is pretty silly since that could just as well be any other built-in. --- Compiler/src/typeinfer.jl | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index c3a6e40da937c..bf1cc0b1220e8 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1438,25 +1438,27 @@ function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vec edge isa CodeInstance && isdefined(edge, :inferred) && push!(workqueue, edge) end + invokelatest_queue === nothing && continue if isexpr(stmt, :call) farg = stmt.args[1] !applicable(argextype, farg, ci, sptypes) && continue # TODO: Why is this failing during bootstrap ftyp = widenconst(argextype(farg, ci, sptypes)) - if ftyp <: Builtin - if Core.finalizer isa ftyp && length(stmt.args) == 3 - finalizer = argextype(stmt.args[2], ci, sptypes) - obj = argextype(stmt.args[3], ci, sptypes) - atype = argtypes_to_type(Any[finalizer, obj]) - - invokelatest_queue === nothing && continue - let workqueue = invokelatest_queue - # make a best-effort attempt to enqueue the relevant code for the finalizer - mi = compileable_specialization_for_call(workqueue.interp, atype) - mi === nothing && continue - - push!(workqueue, mi) - end - end + + if ftyp === typeof(Core.finalizer) && length(stmt.args) == 3 + finalizer = argextype(stmt.args[2], ci, sptypes) + obj = argextype(stmt.args[3], ci, sptypes) + atype = argtypes_to_type(Any[finalizer, obj]) + else + # No dynamic dispatch to resolve / enqueue + continue + end + + let workqueue = invokelatest_queue + # make a best-effort attempt to enqueue the relevant code for the finalizer + mi = compileable_specialization_for_call(workqueue.interp, atype) + mi === nothing && continue + + push!(workqueue, mi) end end # TODO: handle other StmtInfo like @cfunction and OpaqueClosure? From 748775b5689deaa4acac7929e0a31a80d1c564f4 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:09:15 -0700 Subject: [PATCH 180/662] Add/improve docs for memorynew, pointerref, pointerset, getfield (#58290) Just the docs changes from #57295. --- base/docs/basedocs.jl | 21 ++++++++++++-------- base/docs/intrinsicsdocs.jl | 38 +++++++++++++++++++++++++++++++++++++ doc/src/base/arrays.md | 1 + doc/src/devdocs/builtins.md | 34 ++++++++++++++++++++++++--------- 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 60770c42bae7d..1445eee7e28b4 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -2470,14 +2470,19 @@ julia> Tuple(Real[1, 2, pi]) # takes a collection tuple """ - getfield(value, name::Symbol, [order::Symbol]) - getfield(value, i::Int, [order::Symbol]) - -Extract a field from a composite `value` by name or position. Optionally, an -ordering can be defined for the operation. If the field was declared `@atomic`, -the specification is strongly recommended to be compatible with the stores to -that location. Otherwise, if not declared as `@atomic`, this parameter must be -`:not_atomic` if specified. + getfield(value, name::Symbol, [boundscheck::Bool=true], [order::Symbol]) + getfield(value, i::Int, [boundscheck::Bool=true], [order::Symbol]) + +Extract a field from a composite `value` by name or position. + +Optionally, an ordering can be defined for the operation. If the field was +declared `@atomic`, the specification is strongly recommended to be compatible +with the stores to that location. Otherwise, if not declared as `@atomic`, this +parameter must be `:not_atomic` if specified. + +The bounds check may be disabled, in which case the behavior of this function is +undefined if `i` is out of bounds. + See also [`getproperty`](@ref Base.getproperty) and [`fieldnames`](@ref). # Examples diff --git a/base/docs/intrinsicsdocs.jl b/base/docs/intrinsicsdocs.jl index 3e7b982e85377..db54c1d0dc437 100644 --- a/base/docs/intrinsicsdocs.jl +++ b/base/docs/intrinsicsdocs.jl @@ -22,6 +22,15 @@ The `Core.Intrinsics` module holds the `Core.IntrinsicFunction` objects. """ Core.Intrinsics +""" + Core.memorynew(::Type{T} where T <: GenericMemory, n::Int) + +Construct an uninitialized [`GenericMemory`](@ref) of length `n`. + +See also [`Memory`](@ref Core.Memory), [`Memory{T}(undef, n)`](@ref Core.Memory(::UndefInitializer, ::Int)). +""" +Core.memorynew + """ Core.memoryrefnew(::GenericMemory) Core.memoryrefnew(::GenericMemoryRef, index::Int, [boundscheck::Bool]) @@ -129,6 +138,35 @@ See also [`setpropertyonce!`](@ref Base.replaceproperty!) and [`Core.memoryrefse """ Core.memoryrefsetonce! + +""" + Core.Intrinsics.pointerref(p::Ptr{T}, i::Int, align::Int) + +Load a value of type `T` from the address of the `i`th element (1-indexed) +starting at `p`. This is equivalent to the C expression `p[i-1]`. + +The alignment must be a power of two, or 0, indicating the default alignment +for `T`. If `p[i-1]` is out of bounds, invalid, or is not aligned, the behavior +is undefined. An alignment of 1 is always safe. + +See also [`unsafe_load`](@ref). +""" +Core.Intrinsics.pointerref + +""" + Core.Intrinsics.pointerset(p::Ptr{T}, x::T, i::Int, align::Int) + +Store a value of type `T` to the address of the `i`th element (1-indexed) +starting at `p`. This is equivalent to the C expression `p[i-1] = x`. + +The alignment must be a power of two, or `0`, indicating the default alignment +for `T`. If `p[i-1]` is out of bounds, invalid, or is not aligned, the behavior +is undefined. An alignment of 1 is always safe. + +See also [`unsafe_store!`](@ref). +""" +Core.Intrinsics.pointerset + """ Core.Intrinsics.atomic_pointerref(pointer::Ptr{T}, order::Symbol) --> T diff --git a/doc/src/base/arrays.md b/doc/src/base/arrays.md index defe497daf00c..982c4b49cb2c4 100644 --- a/doc/src/base/arrays.md +++ b/doc/src/base/arrays.md @@ -32,6 +32,7 @@ Base.StridedMatrix Base.StridedVecOrMat Base.GenericMemory Base.Memory +Base.Memory(::UndefInitializer, ::Int) Base.memoryref Base.Slices Base.RowSlices diff --git a/doc/src/devdocs/builtins.md b/doc/src/devdocs/builtins.md index e53321f3e70a0..bb6b1ae0d82a2 100644 --- a/doc/src/devdocs/builtins.md +++ b/doc/src/devdocs/builtins.md @@ -1,12 +1,25 @@ # [Core.Builtins](@id lib-builtins) -## Builtin Function APIs +The following builtin functions are considered unstable, but provide the basic +definitions for what defines the abilities and behaviors of a Julia +program. They are typically accessed through a higher level generic API. -The following Builtin function APIs are considered unstable, but provide the basic -definitions for what defines the abilities and behaviors of a Julia program. They are -typically accessed through a higher level generic API. +## Raw access to memory ```@docs +Core.Intrinsics.pointerref +Core.Intrinsics.pointerset +Core.Intrinsics.atomic_pointerref +Core.Intrinsics.atomic_pointerset +Core.Intrinsics.atomic_pointerswap +Core.Intrinsics.atomic_pointermodify +Core.Intrinsics.atomic_pointerreplace +``` + +## Managed memory + +```@docs +Core.memorynew Core.memoryrefnew Core.memoryrefoffset Core.memoryrefget @@ -16,12 +29,15 @@ Core.memoryrefswap! Core.memoryrefmodify! Core.memoryrefreplace! Core.memoryrefsetonce! -Core.Intrinsics.atomic_pointerref -Core.Intrinsics.atomic_pointerset -Core.Intrinsics.atomic_pointerswap -Core.Intrinsics.atomic_pointermodify -Core.Intrinsics.atomic_pointerreplace +``` + +## Module bindings + Core.get_binding_type + +## Other + +```@docs Core.IntrinsicFunction Core.Intrinsics Core.IR From d54649f57af9240a4968b00ad900d535981e497d Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 16 Apr 2025 20:03:33 +0000 Subject: [PATCH 181/662] typeinfer: Work around inference bug See https://github.com/JuliaLang/julia/issues/58143 for details. --- Compiler/src/typeinfer.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index bf1cc0b1220e8..9a3b312346225 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1618,14 +1618,12 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m codeinfos = [] is_latest_world = true # whether this_world == world_counter() workqueue = CompilationQueue(; interp = nothing) - for (i, this_world) in enumerate(sort!(worlds)) + for this_world in reverse!(sort!(worlds)) workqueue = CompilationQueue(workqueue; interp = NativeInterpreter(this_world; inf_params) ) append!(workqueue, methods) - - is_latest_world = (i == length(worlds)) if is_latest_world # Provide the `invokelatest` queue so that we trigger "best-effort" code generation # for, e.g., finalizers and cfunction. @@ -1636,6 +1634,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m else compile!(codeinfos, workqueue) end + is_latest_world = false end if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE From fa4e1e0bd57228696dea39d650dd2e2f48209f61 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 30 Apr 2025 09:27:53 -0400 Subject: [PATCH 182/662] Fix trimming test --- test/trimming/init.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/trimming/init.c b/test/trimming/init.c index ea1b02f8e5c8f..889d71d47d540 100644 --- a/test/trimming/init.c +++ b/test/trimming/init.c @@ -1,9 +1,11 @@ +#define _GNU_SOURCE +#include #include __attribute__((constructor)) void static_init(void) { if (jl_is_initialized()) return; - julia_init(JL_IMAGE_IN_MEMORY); + jl_init_with_image_handle((void *)RTLD_DEFAULT); jl_exception_clear(); } From 0578395d9c980f496b188a383cc9266bce86d689 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 16 Apr 2025 21:50:43 +0000 Subject: [PATCH 183/662] Adjust `--trim` tests for new functionality --- Compiler/test/verifytrim.jl | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index 2462b50b0559a..ad32224be7e15 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -18,31 +18,42 @@ let infos = Any[] @test isempty(parents) end +make_cfunction() = @cfunction(+, Float64, (Int64,Int64)) + # use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel -let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) - @test_broken length(infos) > 4 +let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE) errors, parents = get_verify_typeinf_trim(infos) @test_broken isempty(errors) # missing cfunction - #resize!(infos, 4) - #errors, = get_verify_typeinf_trim(infos) desc = only(errors) @test desc.first desc = desc.second @test desc isa CallMissing - @test occursin("finalizer", desc.desc) + @test occursin("cfunction", desc.desc) repr = sprint(verify_print_error, desc, parents) @test occursin( - r"""^unresolved finalizer registered from statement \(Core.finalizer\)\(Base.final_shred!, %new\(\)::Base.SecretBuffer\)::Nothing + r"""^unresolved cfunction from statement \$\(Expr\(:cfunction, Ptr{Nothing}, :\(\$\(QuoteNode\(\+\)\)\), Float64, :\(svec\(Int64, Int64\)::Core.SimpleVector\), :\(:ccall\)\)\)::Ptr{Nothing} Stacktrace: - \[1\] finalizer\(f::typeof\(Base.final_shred!\), o::Base.SecretBuffer\) - @ Base gcutils.jl:(\d+) \[inlined\] - \[2\] Base.SecretBuffer\(; sizehint::Int\d\d\) - @ Base secretbuffer.jl:(\d+) \[inlined\] - \[3\] Base.SecretBuffer\(\) - @ Base secretbuffer.jl:(\d+) + \[1\] make_cfunction\(\)""", repr) + + resize!(infos, 1) + @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type + errors, parents = get_verify_typeinf_trim(infos) + desc = only(errors) + @test !desc.first + desc = desc.second + @test desc isa CCallableMissing + @test desc.rt == Ptr{Cvoid} + @test desc.sig == Tuple{typeof(make_cfunction)} + @test occursin("unresolved ccallable", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test repr == "unresolved ccallable for Tuple{$(typeof(make_cfunction))} => Ptr{Nothing}\n\n" +end - $""", repr) +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) + @test length(infos) > 4 + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) resize!(infos, 1) @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type From ff4e43c989d7e77b58a0d1cc9f2b99e9dd6724e3 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 22 Apr 2025 12:48:14 -0400 Subject: [PATCH 184/662] trimming: Remove `init.c` from test This accidentally diverged when #58141 was merged without adjusting the test to match (these tests shouldn't be re-implementing the `juliac` link step anyway, but they currently do) Unfortunately, this hid a bug where `jl_pathname_for_handle` does not understand the main executable. --- test/trimming/Makefile | 13 +++++-------- test/trimming/init.c | 11 ----------- 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 test/trimming/init.c diff --git a/test/trimming/Makefile b/test/trimming/Makefile index c12fe8ee04c56..63114a2764570 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -35,24 +35,21 @@ release: hello$(EXE) basic_jll$(EXE) hello-o.a: $(SRCDIR)/hello.jl $(BUILDSCRIPT) $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true -init.o: $(SRCDIR)/init.c - $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) - basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true -hello$(EXE): hello-o.a init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +hello$(EXE): hello-o.a + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -basic_jll$(EXE): basic_jll-o.a init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +basic_jll$(EXE): basic_jll-o.a + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) check: hello$(EXE) basic_jll$(EXE) $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming clean: - -rm -f hello$(EXE) basic_jll$(EXE) init.o hello-o.a basic_jll-o.a + -rm -f hello$(EXE) basic_jll$(EXE) hello-o.a basic_jll-o.a .PHONY: release clean check diff --git a/test/trimming/init.c b/test/trimming/init.c deleted file mode 100644 index 889d71d47d540..0000000000000 --- a/test/trimming/init.c +++ /dev/null @@ -1,11 +0,0 @@ -#define _GNU_SOURCE -#include -#include - -__attribute__((constructor)) void static_init(void) -{ - if (jl_is_initialized()) - return; - jl_init_with_image_handle((void *)RTLD_DEFAULT); - jl_exception_clear(); -} From 7e515b139ecc6217bfb131f0fa61705abebb6b42 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 2 May 2025 15:00:42 +0900 Subject: [PATCH 185/662] inference: prevent nested slot wrappers in abstract_call_builtin (#58249) --- Compiler/src/abstractinterpretation.jl | 9 +++++---- Compiler/test/inference.jl | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 188085f52d091..8fe30d4891e9b 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -1402,8 +1402,9 @@ function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance, if slotid !== nothing # using union-split signature, we may be able to narrow down `Conditional` sigt = widenconst(slotid > nargs ? argtypes[slotid] : cache_argtypes[slotid]) - thentype = tmeet(cnd.thentype, sigt) - elsetype = tmeet(cnd.elsetype, sigt) + ⊓ = meet(𝕃) + thentype = cnd.thentype ⊓ sigt + elsetype = cnd.elsetype ⊓ sigt if thentype === Bottom && elsetype === Bottom # we accidentally proved this method match is impossible # TODO bail out here immediately rather than just propagating Bottom ? @@ -2119,7 +2120,7 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs else thentype = form_partially_defined_struct(𝕃ᵢ, argtype2, argtypes[3]) if thentype !== nothing - elsetype = argtype2 + elsetype = widenslotwrapper(argtype2) if rt === Const(false) thentype = Bottom elseif rt === Const(true) @@ -3254,7 +3255,7 @@ function abstract_eval_isdefined_expr(interp::AbstractInterpreter, e::Expr, ssta elseif !vtyp.undef rt = Const(true) # definitely assigned previously else # form `Conditional` to refine `vtyp.undef` in the then branch - rt = Conditional(sym, vtyp.typ, vtyp.typ; isdefined=true) + rt = Conditional(sym, widenslotwrapper(vtyp.typ), widenslotwrapper(vtyp.typ); isdefined=true) end return RTEffects(rt, Union{}, EFFECTS_TOTAL) end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index c5f44e22d55ed..163e84266fe11 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -2264,12 +2264,18 @@ struct AliasableFields{S,T} f1::S f2::T end +struct NullableAliasableFields{S,T} + f1::S + f2::T + NullableAliasableFields(f1::S, f2::T) where {S,T} = new{S,T}(f1, f2) + NullableAliasableFields(f1::S) where {S} = new{S,Union{}}(f1) +end mutable struct AliasableConstField{S,T} const f1::S f2::T end -import .Compiler: +using .Compiler: InferenceLattice, MustAliasesLattice, InterMustAliasesLattice, BaseInferenceLattice, SimpleInferenceLattice, IPOResultLattice, typeinf_lattice, ipo_lattice, optimizer_lattice @@ -2282,7 +2288,7 @@ Compiler.optimizer_lattice(::MustAliasInterpreter) = SimpleInferenceLattice.inst # lattice # ------- -import .Compiler: MustAlias, Const, PartialStruct, ⊑, tmerge +using .Compiler: MustAlias, Const, PartialStruct, ⊑, tmerge let 𝕃ᵢ = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance)) ⊑(@nospecialize(a), @nospecialize(b)) = Compiler.:⊑(𝕃ᵢ, a, b) tmerge(@nospecialize(a), @nospecialize(b)) = Compiler.tmerge(𝕃ᵢ, a, b) @@ -2520,6 +2526,15 @@ jet509_hasitems(list) = length(list) >= 1 error("list is empty") end |> only == Vector{Int} +# don't form nested slot wrappers +@test Base.infer_return_type((NullableAliasableFields{NullableAliasableFields},); interp=MustAliasInterpreter()) do x + y = getfield(x, :f1) + if isdefined(y, :f2) && isa(getfield(y, :f2), Int) + return getfield(y, :f2) + end + return 0 +end == Int + # === constraint # -------------- From bb7b6e7f4a790357fe30fe684c2c2d079a75670d Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Fri, 2 May 2025 08:20:10 -0400 Subject: [PATCH 186/662] Speed up extrema 3-50x (#58280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This method is no longer an optimization over what Julia can do with the naive definition on most (if not all) architectures. Like #58267, I asked for a smattering of crowdsourced multi-architecture benchmarking of this simple example: ```julia using BenchmarkTools A = rand(10000); b1 = @benchmark extrema($A) b2 = @benchmark mapreduce(x->(x,x),((min1, max1), (min2, max2))->(min(min1, min2), max(max1, max2)), $A) println("$(Sys.CPU_NAME): $(round(median(b1).time/median(b2).time, digits=1))x faster") ``` With results: ```txt cortex-a72: 13.2x faster cortex-a76: 15.8x faster neoverse-n1: 16.4x faster neoverse-v2: 23.4x faster a64fx: 46.5x faster apple-m1: 54.9x faster apple-m4*: 43.7x faster znver2: 8.6x faster znver4: 12.8x faster znver5: 16.7x faster haswell (32-bit): 3.5x faster skylake-avx512: 7.4x faster rocketlake: 7.8x faster alderlake: 5.2x faster cascadelake: 8.8x faster cascadelake: 7.1x faster ``` The results are even more dramatic for Float32s, here on my M1: ```julia julia> A = rand(Float32, 10000); julia> @benchmark extrema($A) BenchmarkTools.Trial: 10000 samples with 1 evaluation per sample. Range (min … max): 49.083 μs … 151.750 μs ┊ GC (min … max): 0.00% … 0.00% Time (median): 49.375 μs ┊ GC (median): 0.00% Time (mean ± σ): 49.731 μs ± 2.350 μs ┊ GC (mean ± σ): 0.00% ± 0.00% ▅██▅▁ ▁▂▂ ▁▂▁ ▂ ██████▇▇▇▇█▇████▇▆▆▆▇▇███▇▇▆▆▆▅▆▅▃▄▃▄▅▄▄▆▆▅▃▁▄▃▅▄▅▄▄▁▄▄▅▃▄▁▄ █ 49.1 μs Histogram: log(frequency) by time 56.8 μs < Memory estimate: 0 bytes, allocs estimate: 0. julia> @benchmark mapreduce(x->(x,x),((min1, max1), (min2, max2))->(min(min1, min2), max(max1, max2)), $A) BenchmarkTools.Trial: 10000 samples with 191 evaluations per sample. Range (min … max): 524.435 ns … 1.104 μs ┊ GC (min … max): 0.00% … 0.00% Time (median): 525.089 ns ┊ GC (median): 0.00% Time (mean ± σ): 529.323 ns ± 20.876 ns ┊ GC (mean ± σ): 0.00% ± 0.00% █▃ ▁ ▃▃ ▁ █████▇███▇███▇▇▇▇▇▇▇▇▅▆▆▆▆▆▅▅▄▆▃▄▄▃▅▅▄▃▅▄▄▄▅▅▅▃▅▄▄▁▄▄▅▆▄▄▅▄▅ █ 524 ns Histogram: log(frequency) by time 609 ns < Memory estimate: 0 bytes, allocs estimate: 0. ``` Closes #34790, closes #31442, closes #44606. --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- base/reduce.jl | 8 -------- test/numbers.jl | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/base/reduce.jl b/base/reduce.jl index bd29d7eb3d889..f85cb285ef9ce 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -860,14 +860,6 @@ ExtremaMap(::Type{T}) where {T} = ExtremaMap{Type{T}}(T) @inline (f::ExtremaMap)(x) = (y = f.f(x); (y, y)) @inline _extrema_rf((min1, max1), (min2, max2)) = (min(min1, min2), max(max1, max2)) -# optimization for IEEEFloat -function _extrema_rf(x::NTuple{2,T}, y::NTuple{2,T}) where {T<:IEEEFloat} - (x1, x2), (y1, y2) = x, y - anynan = isnan(x1)|isnan(y1) - z1 = ifelse(anynan, x1-y1, ifelse(signbit(x1-y1), x1, y1)) - z2 = ifelse(anynan, x1-y1, ifelse(signbit(x2-y2), y2, x2)) - z1, z2 -end ## findmax, findmin, argmax & argmin diff --git a/test/numbers.jl b/test/numbers.jl index f5767e235b1d0..702cad3f4e0f1 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -154,7 +154,8 @@ end x = unorded[i], unorded[i] y = unorded[j], unorded[j] z = Base._extrema_rf(x, y) - @test z === x || z === y + @test (z[1] === x[1] || z[1] === y[1]) && + (z[2] === x[1] || z[2] === y[1]) end end end From 35bfba9229ef0a07799ef1e493937f97829d5242 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Fri, 2 May 2025 17:23:37 +0200 Subject: [PATCH 187/662] fix: give access to displaysize when displaying at REPL (#58121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In #53959, an new `LimitIO` object was defined to display at the REPL, but it hid the displaysize of the underlying `io`. This patch just forwards this information. Before: ``` julia> Dict(1 => fill(UInt(0), 4)) Dict{Int64, Vector{UInt64}} with 1 entry: 1 => [0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x000000000… ``` After: ``` Dict{Int64, Vector{UInt64}} with 1 entry: 1 => [0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000] ``` --- stdlib/REPL/src/REPL.jl | 2 ++ stdlib/REPL/test/repl.jl | 3 +++ 2 files changed, 5 insertions(+) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 8e4bf4fd9db85..5fc74f77d2971 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -480,6 +480,8 @@ function Base.showerror(io::IO, e::LimitIOException) print(io, "$LimitIOException: aborted printing after attempting to print more than $(Base.format_bytes(e.maxbytes)) within a `LimitIO`.") end +Base.displaysize(io::LimitIO) = _displaysize(io.io) + function Base.write(io::LimitIO, v::UInt8) io.n > io.maxbytes && throw(LimitIOException(io.maxbytes)) n_bytes = write(io.io, v) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index b2172096a2e6b..8dab038b01c50 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1950,6 +1950,9 @@ end @test output == "…[printing stopped after displaying 0 bytes; $hint]" @test sprint(io -> show(REPL.LimitIO(io, 5), "abc")) == "\"abc\"" @test_throws REPL.LimitIOException(1) sprint(io -> show(REPL.LimitIO(io, 1), "abc")) + + # displaying objects at the REPL sometimes needs access to displaysize, like Dict + @test displaysize(IOContext(REPL.LimitIO(stdout, 100), stdout)) == displaysize(stdout) finally REPL.SHOW_MAXIMUM_BYTES = previous end From 48bd67353ddf660469e8b3d5b33eff8b6e36bcf2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 2 May 2025 13:12:45 -0400 Subject: [PATCH 188/662] Prevent type infer hang of "simple" recursive functions (#58273) Prevent infinite inference when dealing with LimitedAccuracy by declining to cache them. Presence in the cache triggers further compilation to resolve it, so removing infinite work from the cache also prevents infinite work at pre-compile/jit compile time. Fix #57098 Fix #57873 --- Compiler/src/typeinfer.jl | 15 +++++++-------- Compiler/test/inference.jl | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 9a3b312346225..93b29a22ab7be 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -613,18 +613,17 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: istoplevel = !(me.linfo.def isa Method) istoplevel || compute_edges!(me) # don't add backedges to toplevel method instance - if limited_ret - # a parent may be cached still, but not this intermediate work: - # we can throw everything else away now + if limited_ret || limited_src + # A parent may be cached still, but not this intermediate work: + # we can throw everything else away now. Caching anything can confuse later + # heuristics to consider it worth trying to pursue compiling this further and + # finding infinite work as a result. Avoiding caching helps to ensure there is only + # a finite amount of work that can be discovered later (although potentially still a + # large multiplier on it). result.src = nothing result.tombstone = true me.cache_mode = CACHE_MODE_NULL set_inlineable!(me.src, false) - elseif limited_src - # a type result will be cached still, but not this intermediate work: - # we can throw everything else away now - result.src = nothing - set_inlineable!(me.src, false) else # annotate fulltree with type information, # either because we are the outermost code, or we might use this later diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 163e84266fe11..a0e436144e0a9 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6460,4 +6460,29 @@ A58257.B58257.get! # Creates binding partition in A.B, N+1:∞ Base.invoke_in_world(UInt(38678), getglobal, A58257, :get!) # Expands binding partition in A through Date: Fri, 2 May 2025 17:30:40 -0400 Subject: [PATCH 189/662] move growend_internal! and growbeg_internal! out to proper functions (#58211) @gbaraldi has told me that this leads to a better calling convention (and is clearer to read) --- base/array.jl | 129 ++++++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/base/array.jl b/base/array.jl index c56a3757b0103..9bf0bbc87aba9 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1066,6 +1066,41 @@ end array_new_memory(mem::Memory, newlen::Int) = typeof(mem)(undef, newlen) # when implemented, this should attempt to first expand mem +function _growbeg_internal!(a::Vector, delta::Int, len::Int) + @_terminates_locally_meta + ref = a.ref + mem = ref.mem + offset = memoryrefoffset(ref) + newlen = len + delta + memlen = length(mem) + if offset + len - 1 > memlen || offset < 1 + throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) + end + # since we will allocate the array in the middle of the memory we need at least 2*delta extra space + # the +1 is because I didn't want to have an off by 1 error. + newmemlen = max(overallocation(len), len + 2 * delta + 1) + newoffset = div(newmemlen - newlen, 2) + 1 + # If there is extra data after the end of the array we can use that space so long as there is enough + # space at the end that there won't be quadratic behavior with a mix of growth from both ends. + # Specifically, we want to ensure that we will only do this operation once before + # increasing the size of the array, and that we leave enough space at both the beginning and the end. + if newoffset + newlen < memlen + newoffset = div(memlen - newlen, 2) + 1 + newmem = mem + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) + for j in offset:newoffset+delta-1 + @inbounds _unsetindex!(mem, j) + end + else + newmem = array_new_memory(mem, newmemlen) + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) + end + if ref !== a.ref + throw(ConcurrencyViolationError("Vector can not be resized concurrently")) + end + setfield!(a, :ref, @inbounds memoryref(newmem, newoffset)) +end + function _growbeg!(a::Vector, delta::Integer) @_noub_meta delta = Int(delta) @@ -1081,40 +1116,46 @@ function _growbeg!(a::Vector, delta::Integer) if delta <= offset - 1 setfield!(a, :ref, @inbounds memoryref(ref, 1 - delta)) else - @noinline (function() - @_terminates_locally_meta - memlen = length(mem) - if offset + len - 1 > memlen || offset < 1 - throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) - end - # since we will allocate the array in the middle of the memory we need at least 2*delta extra space - # the +1 is because I didn't want to have an off by 1 error. - newmemlen = max(overallocation(len), len + 2 * delta + 1) - newoffset = div(newmemlen - newlen, 2) + 1 - # If there is extra data after the end of the array we can use that space so long as there is enough - # space at the end that there won't be quadratic behavior with a mix of growth from both ends. - # Specifically, we want to ensure that we will only do this operation once before - # increasing the size of the array, and that we leave enough space at both the beginning and the end. - if newoffset + newlen < memlen - newoffset = div(memlen - newlen, 2) + 1 - newmem = mem - unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) - for j in offset:newoffset+delta-1 - @inbounds _unsetindex!(mem, j) - end - else - newmem = array_new_memory(mem, newmemlen) - unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) - end - if ref !== a.ref - @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) - end - setfield!(a, :ref, @inbounds memoryref(newmem, newoffset)) - end)() + @noinline _growbeg_internal!(a, delta, len) end return end +function _growend_internal!(a::Vector, delta::Int, len::Int) + ref = a.ref + mem = ref.mem + memlen = length(mem) + newlen = len + delta + offset = memoryrefoffset(ref) + newmemlen = offset + newlen - 1 + if offset + len - 1 > memlen || offset < 1 + throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) + end + + if offset - 1 > div(5 * newlen, 4) + # If the offset is far enough that we can copy without resizing + # while maintaining proportional spacing on both ends of the array + # note that this branch prevents infinite growth when doing combinations + # of push! and popfirst! (i.e. when using a Vector as a queue) + newmem = mem + newoffset = div(newlen, 8) + 1 + else + # grow either by our computed overallocation factor + # or exactly the requested size, whichever is larger + # TODO we should possibly increase the offset if the current offset is nonzero. + newmemlen2 = max(overallocation(memlen), newmemlen) + newmem = array_new_memory(mem, newmemlen2) + newoffset = offset + end + newref = @inbounds memoryref(newmem, newoffset) + unsafe_copyto!(newref, ref, len) + if ref !== a.ref + @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) + end + setfield!(a, :ref, newref) +return +end + function _growend!(a::Vector, delta::Integer) @_noub_meta delta = Int(delta) @@ -1128,33 +1169,7 @@ function _growend!(a::Vector, delta::Integer) setfield!(a, :size, (newlen,)) newmemlen = offset + newlen - 1 if memlen < newmemlen - @noinline (function() - if offset + len - 1 > memlen || offset < 1 - throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) - end - - if offset - 1 > div(5 * newlen, 4) - # If the offset is far enough that we can copy without resizing - # while maintaining proportional spacing on both ends of the array - # note that this branch prevents infinite growth when doing combinations - # of push! and popfirst! (i.e. when using a Vector as a queue) - newmem = mem - newoffset = div(newlen, 8) + 1 - else - # grow either by our computed overallocation factor - # or exactly the requested size, whichever is larger - # TODO we should possibly increase the offset if the current offset is nonzero. - newmemlen2 = max(overallocation(memlen), newmemlen) - newmem = array_new_memory(mem, newmemlen2) - newoffset = offset - end - newref = @inbounds memoryref(newmem, newoffset) - unsafe_copyto!(newref, ref, len) - if ref !== a.ref - @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) - end - setfield!(a, :ref, newref) - end)() + @noinline _growend_internal!(a, delta, len) end return end From 1237a9c8ed781809832af5b06cb2467096809fca Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Fri, 2 May 2025 20:46:25 -0400 Subject: [PATCH 190/662] improve isdefined precision for 0 field types (#58220) alternate to https://github.com/JuliaLang/julia/pull/58214. --------- Co-authored-by: Jeff Bezanson --- Compiler/src/tfuncs.jl | 4 ++++ Compiler/test/inference.jl | 1 + base/runtime_internals.jl | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index f39601c1e729d..f3ce3b010a345 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -450,6 +450,10 @@ end return Const(true) end end + # datatype_fieldcount is what `fieldcount` uses internally + # and returns nothing (!==0) for non-definite field counts. + elseif datatype_fieldcount(a1) === 0 + return Const(false) end elseif isa(a1, Union) # Results can only be `Const` or `Bool` diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index a0e436144e0a9..8333b191c75c9 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -1210,6 +1210,7 @@ let isdefined_tfunc(@nospecialize xs...) = @test isdefined_tfunc(Union{UnionIsdefinedA,UnionIsdefinedB}, Const(:x)) === Const(true) @test isdefined_tfunc(Union{UnionIsdefinedA,UnionIsdefinedB}, Const(:y)) === Const(false) @test isdefined_tfunc(Union{UnionIsdefinedA,Nothing}, Const(:x)) === Bool + @test isdefined_tfunc(Nothing, Any) === Const(false) end # https://github.com/aviatesk/JET.jl/issues/379 diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 9aae6b2a2b78c..db8fd114a3493 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1141,7 +1141,7 @@ function datatype_fieldcount(t::DataType) return length(names) end if types isa DataType && types <: Tuple - return fieldcount(types) + return datatype_fieldcount(types) end return nothing elseif isabstracttype(t) From 7b64cec5385d9099762ad7449c340eaac4fccb41 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Sat, 3 May 2025 02:19:09 -0700 Subject: [PATCH 191/662] Fix lowering failure with type parameter in opaque closure (#58307) --- src/julia-syntax.scm | 22 ++++++++++++++++------ test/opaque_closure.jl | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 3b05f19e5456e..0268306f147f0 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2599,11 +2599,13 @@ (typ-svec (caddr sig-svec)) (tvars (cddr (cadddr sig-svec))) (argtypes (cdddr typ-svec)) - (functionloc (cadr (caddddr sig-svec)))) - (let* ((argtype (foldl (lambda (var ex) `(call (core UnionAll) ,var ,ex)) - (expand-forms `(curly (core Tuple) ,@argtypes)) - (reverse tvars)))) - `(_opaque_closure ,(or argt argtype) ,rt_lb ,rt_ub ,isva ,(length argtypes) ,allow-partial ,functionloc ,lam)))) + (functionloc (cadr (caddddr sig-svec))) + (argtype (foldl (lambda (var ex) `(call (core UnionAll) ,var ,ex)) + (expand-forms `(curly (core Tuple) ,@argtypes)) + (reverse tvars))) + (argtype (or argt argtype)) + (argtype (if (null? stmts) argtype `(block ,@stmts ,argtype)))) + `(_opaque_closure ,argtype ,rt_lb ,rt_ub ,isva ,(length argtypes) ,allow-partial ,functionloc ,lam))) 'block (lambda (e) @@ -5232,6 +5234,14 @@ f(x) = yt(x) (define (set-lineno! lineinfo num) (set-car! (cddr lineinfo) num)) +;; note that the 'list and 'block atoms make all lists 1-indexed. +;; returns a 5-element vector containing: +;; code: `(block ,@(n expressions)) +;; locs: list of line-table index, where code[i] has lineinfo line-table[locs[i]] +;; line-table: list of `(lineinfo file.jl 123 0)' +;; ssavalue-table: table of (ssa-num . code-index) +;; where ssavalue references in `code` need this remapping +;; label-table: table of (label . code-index) (define (compact-ir body file line) (let ((code '(block)) (locs '(list)) @@ -5338,7 +5348,7 @@ f(x) = yt(x) e) ((ssavalue? e) (let ((idx (get ssavalue-table (cadr e) #f))) - (if (not idx) (begin (prn e) (prn lam) (error "ssavalue with no def"))) + (if (not idx) (error "internal bug: ssavalue with no def")) `(ssavalue ,idx))) ((eq? (car e) 'goto) `(goto ,(get label-table (cadr e)))) diff --git a/test/opaque_closure.jl b/test/opaque_closure.jl index 7b02578a86621..0dc2ed95b8872 100644 --- a/test/opaque_closure.jl +++ b/test/opaque_closure.jl @@ -407,3 +407,6 @@ let f = f54357(+, Tuple{Int,Int}) @test g isa Core.OpaqueClosure @test g(32.0, 34.0) === 66.0 end + +# 49659: signature-scoped typevar shouldn't fail in lowering +@test_throws "must be a tuple type" @opaque ((x::T,y::T) where {T}) -> 123 From 71574073759506b0a3171640ea5e1b468f2757c9 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Sat, 3 May 2025 12:33:46 -0300 Subject: [PATCH 192/662] Make build_id.lo more random (#58258) This does not fix the underlying issue that can occur here which is a collision of build_ids.lo between modules in IR decompression. Fixing that requires a somewhat significant overhaul to the serialization of IR (probably using the module identity as a key). This does mean we use a lot more of the bits available here so it makes collisions a lot less likely( they were already extremely rare) but hrtime does tend to only use the lower bits of a 64 bit integer and this will hopefully add some more randomness and make this even less likely --- src/module.c | 3 ++- src/staticdata.c | 6 +++++- src/staticdata_utils.c | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 9dedb250db083..850cd8d6aa885 100644 --- a/src/module.c +++ b/src/module.c @@ -483,7 +483,8 @@ static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) m->istopmod = 0; m->uuid = uuid_zero; static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing - m->build_id.lo = jl_hrtime() + (++mcounter); + // TODO: this is used for ir decompression and is liable to hash collisions so use more of the bits + m->build_id.lo = bitmix(jl_hrtime() + (++mcounter), jl_rand()); if (!m->build_id.lo) m->build_id.lo++; // build id 0 is invalid m->build_id.hi = ~(uint64_t)0; diff --git a/src/staticdata.c b/src/staticdata.c index a8ebf1d9bc345..df38ad1fc0277 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -4410,7 +4410,11 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im JL_SIGATOMIC_END(); // Add roots to methods - jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); + int failed = jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); + if (failed != 0) { + jl_printf(JL_STDERR, "Error copying roots to methods from Module: %s\n", pkgname); + abort(); + } // Insert method extensions and handle edges int new_methods = jl_array_nrows(extext_methods) > 0; if (!new_methods) { diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 95a64964eccba..c3f6a7e98a550 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -736,17 +736,31 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size } } -static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) +static int jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) { size_t i, l = jl_array_nrows(method_roots_list); + int failed = 0; for (i = 0; i < l; i+=2) { jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(method_roots_list, i); jl_array_t *roots = (jl_array_t*)jl_array_ptr_ref(method_roots_list, i+1); if (roots) { assert(jl_is_array(roots)); + if (m->root_blocks) { + // check for key collision + uint64_t *blocks = jl_array_data(m->root_blocks, uint64_t); + size_t nx2 = jl_array_nrows(m->root_blocks); + for (size_t i = 0; i < nx2; i+=2) { + if (blocks[i] == key) { + // found duplicate block + failed = -1; + } + } + } + jl_append_method_roots(m, key, roots); } } + return failed; } static jl_value_t *read_verify_mod_list(ios_t *s, jl_array_t *depmods) From 06821587b0fde7f87432e90d50158c562ac61c0d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 3 May 2025 12:33:45 -0400 Subject: [PATCH 193/662] add backdate_admonition to depwarn=error (#58266) This is close to the expected behavior after deprecations are removed (other than that the b->globalref->mod in the printed message here will be the source module instead of the destination module, which may sometimes cause confusing printing here, but probably rarely). I also needed this recently to find a place this warning occurred, so I think it should be merged now and get feedback later. Closes #57969 --- src/module.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 850cd8d6aa885..0271a645aa4ed 100644 --- a/src/module.c +++ b/src/module.c @@ -845,12 +845,15 @@ JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT { + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_undefined_var_error(b->globalref->name, (jl_value_t*)b->globalref->mod); jl_safe_printf( "WARNING: Detected access to binding `%s.%s` in a world prior to its definition world.\n" " Julia 1.12 has introduced more strict world age semantics for global bindings.\n" " !!! This code may malfunction under Revise.\n" " !!! This code will error in future versions of Julia.\n" - "Hint: Add an appropriate `invokelatest` around the access to this binding.\n", + "Hint: Add an appropriate `invokelatest` around the access to this binding.\n" + "To make this warning an error, and hence obtain a stack trace, use `julia --depwarn=error`.\n", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); } From b680b4ebaac3cc8b5896e1ec7861004c653ea22b Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 4 May 2025 21:30:57 +0200 Subject: [PATCH 194/662] faster rand(::RandomDevice, NTuple{N, UInt}) (#58288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RandomDevice` uses `Libc.getrandom!` to get randomness, but such a system call is very expensive. There was already a specialization for `rand!` on `Array`s of builtin integers to have only one such call; this commit adds a similar specialization for homogeneous tuples of builtin integer types. One motivation is to instantiate `Xoshiro()` faster from `RandomDevice()`, as requesting 4*64 bits of entropy from the system in one go is, at least on my system, roughly 4 times faster that in 4 calls: ``` julia> @btime Xoshiro() 1.225 μs (1 allocation: 48 bytes) # master 319.068 ns (1 allocation: 48 bytes) # PR ``` --- stdlib/Random/src/RNGs.jl | 7 +++++++ stdlib/Random/src/Xoshiro.jl | 7 +------ stdlib/Random/test/runtests.jl | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index b99430adbdb46..81b3fe0da85b4 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -16,6 +16,13 @@ seed!(rng::RandomDevice, ::Nothing) = rng rand(rd::RandomDevice, sp::SamplerBoolBitInteger) = Libc.getrandom!(Ref{sp[]}())[] rand(rd::RandomDevice, ::SamplerType{Bool}) = rand(rd, UInt8) % Bool + +# specialization for homogeneous tuple types of builtin integers, to avoid +# repeated system calls +rand(rd::RandomDevice, sp::SamplerTag{Ref{Tuple{Vararg{T, N}}}, Tuple{S}} + ) where {T, N, S <: SamplerUnion(Base.BitInteger_types...)} = + Libc.getrandom!(Ref{gentype(sp)}())[] + function rand!(rd::RandomDevice, A::Array{Bool}, ::SamplerType{Bool}) Libc.getrandom!(A) # we need to mask the result so that only the LSB in each byte can be non-zero diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 40e42d9fd7743..088af4adac452 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -245,12 +245,7 @@ hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2 function seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) # as we get good randomness from RandomDevice, we can skip hashing - rd = RandomDevice() - s0 = rand(rd, UInt64) - s1 = rand(rd, UInt64) - s2 = rand(rd, UInt64) - s3 = rand(rd, UInt64) - initstate!(rng, (s0, s1, s2, s3)) + initstate!(rng, rand(RandomDevice(), NTuple{4, UInt64})) end seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) = diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index a28197bdfe833..fc3469d13262f 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -824,6 +824,20 @@ end @inferred rand(Tuple{Int32,Int64,Float64}) @inferred rand(NTuple{20,Int}) @test_throws TypeError rand(Tuple{1:2,3:4}) + + @testset "rand(::RandomDevice, ::Type{NTuple{N, Int}})" begin + # RandomDevice has a specialization for homogeneous tuple types of builtin integers + rd = RandomDevice() + @test () == rand(rd, Tuple{}) + xs = rand(rd, Tuple{Int, Int}) + @test xs isa Tuple{Int, Int} && xs[1] != xs[2] + xs = rand(rd, NTuple{2, Int}) + @test xs isa Tuple{Int, Int} && xs[1] != xs[2] + xs = rand(rd, Tuple{Int, UInt}) # not NTuple + @test xs isa Tuple{Int, UInt} && xs[1] != xs[2] + xs = rand(rd, Tuple{Bool}) # not included in the specialization + @test xs isa Tuple{Bool} + end end @testset "GLOBAL_RNG" begin From f2e223f65f73f90f844d681f2ff1a8dd7eff1c2f Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 1 Oct 2023 14:09:09 +0200 Subject: [PATCH 195/662] Random: allow seeding from an RNG Seeding from an RNG is the most flexible way: the caller doesn't need to know anything about the internals of the fed RNG, which can request all the entropy it needs from the parent PRG in whatever form it needs. --- stdlib/Random/src/RNGs.jl | 9 +++++---- stdlib/Random/src/Random.jl | 17 ++++++++++------- stdlib/Random/src/Xoshiro.jl | 8 ++++---- stdlib/Random/test/runtests.jl | 8 +++++++- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 81b3fe0da85b4..8e1f7bffede78 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -365,12 +365,13 @@ function initstate!(r::MersenneTwister, data::StridedVector, seed) end # When a seed is not provided, we generate one via `RandomDevice()` rather -# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, whatever)` because the +# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, 8)` because the # seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a -# small overhead compared to `initstate!`, so this simple solution is fine. -# A random seed with 128 bits is a good compromise for almost surely always getting distinct +# small overhead compared to `initstate!`. +# A random seed with 128 bits is a good compromise for almost surely getting distinct # seeds, while having them printed reasonably tersely. -seed!(r::MersenneTwister, ::Nothing) = seed!(r, rand(RandomDevice(), UInt128)) +seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128)) +seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice()) seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed) diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 2d75f49480a7b..5721aad85557d 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -414,8 +414,10 @@ sequence of numbers if and only if a `seed` is provided. Some RNGs don't accept a seed, like `RandomDevice`. After the call to `seed!`, `rng` is equivalent to a newly created object initialized with the same seed. + The types of accepted seeds depend on the type of `rng`, but in general, -integer seeds should work. +integer seeds should work. Providing `nothing` as the seed should be +equivalent to not providing one. If `rng` is not specified, it defaults to seeding the state of the shared task-local generator. @@ -455,11 +457,12 @@ julia> rand(Xoshiro(), Bool) # not reproducible either true ``` """ -seed!(rng::AbstractRNG) = seed!(rng, nothing) -#= -We have this generic definition instead of the alternative option -`seed!(rng::AbstractRNG, ::Nothing) = seed!(rng)` -because it would lead too easily to ambiguities, e.g. when we define `seed!(::Xoshiro, seed)`. -=# +function seed!(rng::AbstractRNG, seed=nothing) + if seed === nothing + seed!(rng, RandomDevice()) + else + throw(MethodError(seed!, (rng, seed))) + end +end end # module diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 088af4adac452..efeb39f65ab88 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -243,10 +243,10 @@ copy!(dst::Union{TaskLocalRNG, Xoshiro}, src::Union{TaskLocalRNG, Xoshiro}) = se # use a magic (random) number to scramble `h` so that `hash(x)` is distinct from `hash(getstate(x))` hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2dda6fa9be % UInt) -function seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) - # as we get good randomness from RandomDevice, we can skip hashing - initstate!(rng, rand(RandomDevice(), NTuple{4, UInt64})) -end +seed!(rng::Union{TaskLocalRNG, Xoshiro}, seeder::AbstractRNG) = + initstate!(rng, rand(seeder, NTuple{4, UInt64})) + +seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) = @invoke seed!(rng::AbstractRNG, nothing::Any) seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) = initstate!(rng, reinterpret(UInt64, hash_seed(seed))) diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index fc3469d13262f..2b99e50eb7e73 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -675,6 +675,7 @@ end @test Random.seed!(m..., typemax(UInt)) === m2 @test Random.seed!(m..., typemax(UInt128)) === m2 @test Random.seed!(m..., "a random seed") === m2 + @test Random.seed!(m..., Random.default_rng()) === m2 end end @@ -728,7 +729,7 @@ end @test rand(m, Int) ∉ (a, b, c, d) end -@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister,Xoshiro) +@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro) seeds = Any[0, 1, 2, 10000, 10001, rand(UInt32, 8), randstring(), randstring(), rand(UInt128, 3)...] if RNG == Xoshiro push!(seeds, rand(UInt64, rand(1:4))) @@ -739,6 +740,11 @@ end Random.seed!(m, seed) @test a == [rand(m) for _=1:100] end + # rng as a seed + m = RNG(Xoshiro(0)) + a = [rand(m) for _=1:100] + Random.seed!(m, Xoshiro(0)) + @test a == [rand(m) for _=1:100] end @testset "Random.seed!(seed) sets Random.GLOBAL_SEED" begin From 856025e8c65e144ca9cd0335118aa3af28a9958e Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 1 Oct 2023 17:10:30 +0200 Subject: [PATCH 196/662] Random: make a new "strong" RNG using SHA The most convenient way to define `seed!` for new RNGs is via an another RNG, with `seed!(rng::AbstractRNG, seeder::AbstractRNG)`. But RNGs want to also support more usual seeds. In order to allow them to only define the method above, a new `SeedHasher` RNG is implemented, whose purpose is to convert an initial given seed into a stream of random numbers. Given that it's not always "safe" to seed an RNG from another RNG, `SeedHasher` uses a strong cryptographic hash (SHA2) to produces random streams. The generic `seed!(rng::AbstractRNG, seed)` method now takes care of forwarding the call to `seed!(rng, SeedHasher(seed))`. --- stdlib/Random/src/RNGs.jl | 104 ++++++++++++++++++++++++++++----- stdlib/Random/src/Random.jl | 9 ++- stdlib/Random/src/Xoshiro.jl | 5 -- stdlib/Random/test/runtests.jl | 52 +++++++++++------ 4 files changed, 128 insertions(+), 42 deletions(-) diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 8e1f7bffede78..b8f71f3e2f3c8 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -285,10 +285,81 @@ end ### seeding +""" + Random.SeedHasher(seed=nothing) + +Create a `Random.SeedHasher` RNG object, which generates random bytes with the help +of a cryptographic hash function (SHA2), via calls to [`Random.hash_seed`](@ref). + +Given two seeds `s1` and `s2`, the random streams generated by +`SeedHasher(s1)` and `SeedHasher(s2)` should be distinct if and only if +`s1` and `s2` are distinct. + +This RNG is used by default in `Random.seed!(::AbstractRNG, seed::Any)`, such that +RNGs usually need only to implement `seed!(rng, ::AbstractRNG)`. + +This is an internal type, subject to change. +""" +mutable struct SeedHasher <: AbstractRNG + bytes::Vector{UInt8} + idx::Int + cnt::Int64 + + SeedHasher(seed=nothing) = seed!(new(), seed) +end + +seed!(rng::SeedHasher, seeder::AbstractRNG) = seed!(rng, rand(seeder, UInt64, 4)) +seed!(rng::SeedHasher, ::Nothing) = seed!(rng, RandomDevice()) + +function seed!(rng::SeedHasher, seed) + # typically, no more than 256 bits will be needed, so use + # SHA2_256 because it's faster + ctx = SHA2_256_CTX() + hash_seed(seed, ctx) + rng.bytes = SHA.digest!(ctx)::Vector{UInt8} + rng.idx = 0 + rng.cnt = 0 + rng +end + +@noinline function rehash!(rng::SeedHasher) + # more random bytes are necessary, from now on use SHA2_512 to generate + # more bytes at once + ctx = SHA2_512_CTX() + SHA.update!(ctx, rng.bytes) + # also hash the counter, just for the extremely unlikely case where the hash of + # rng.bytes is equal to rng.bytes (i.e. rng.bytes is a "fixed point"), or more generally + # if there is a small cycle + SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, rng.cnt += 1)) + rng.bytes = SHA.digest!(ctx) + rng.idx = 0 + rng +end + +function rand(rng::SeedHasher, ::SamplerType{UInt8}) + rng.idx < length(rng.bytes) || rehash!(rng) + rng.bytes[rng.idx += 1] +end + +for TT = Base.BitInteger_types + TT === UInt8 && continue + @eval function rand(rng::SeedHasher, ::SamplerType{$TT}) + xx = zero($TT) + for ii = 0:sizeof($TT)-1 + xx |= (rand(rng, UInt8) % $TT) << (8 * ii) + end + xx + end +end + +rand(rng::SeedHasher, ::SamplerType{Bool}) = rand(rng, UInt8) % Bool + +rng_native_52(::SeedHasher) = UInt64 + + #### hash_seed() -function hash_seed(seed::Integer) - ctx = SHA.SHA2_256_CTX() +function hash_seed(seed::Integer, ctx::SHA_CTX) neg = signbit(seed) if neg seed = ~seed @@ -302,21 +373,18 @@ function hash_seed(seed::Integer) end # make sure the hash of negative numbers is different from the hash of positive numbers neg && SHA.update!(ctx, (0x01,)) - SHA.digest!(ctx) + nothing end -function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}) - ctx = SHA.SHA2_256_CTX() +function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}, ctx::SHA_CTX) for xx in seed SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, UInt64(xx))) end # discriminate from hash_seed(::Integer) SHA.update!(ctx, (0x10,)) - SHA.digest!(ctx) end -function hash_seed(str::AbstractString) - ctx = SHA.SHA2_256_CTX() +function hash_seed(str::AbstractString, ctx::SHA_CTX) # convert to String such that `codeunits(str)` below is consistent between equal # strings of different types str = String(str) @@ -331,25 +399,29 @@ function hash_seed(str::AbstractString) SHA.update!(ctx, (pad % UInt8,)) end SHA.update!(ctx, (0x05,)) - SHA.digest!(ctx) end """ - hash_seed(seed)::AbstractVector{UInt8} + Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8} + +Update `ctx` via `SHA.update!` with the content of `seed`. +This function is used by the [`SeedHasher`](@ref) RNG to produce +random bytes. -Return a cryptographic hash of `seed` of size 256 bits (32 bytes). `seed` can currently be of type `Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`, but modules can extend this function for types they own. -`hash_seed` is "injective" : if `n != m`, then `hash_seed(n) != `hash_seed(m)`. -Moreover, if `n == m`, then `hash_seed(n) == hash_seed(m)`. - -This is an internal function subject to change. +`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`, +if `n != m`, then `cn` and `cm` will be distinct after calling +`hash_seed(n, cn); hash_seed(m, cm)`. +Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling +`hash_seed(n, cn); hash_seed(m, cm)`. """ hash_seed + #### seed!() function initstate!(r::MersenneTwister, data::StridedVector, seed) @@ -372,7 +444,7 @@ end # seeds, while having them printed reasonably tersely. seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128)) seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice()) -seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed) +seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed) ### Global RNG diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 5721aad85557d..796b3edfe6321 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -13,7 +13,7 @@ include("DSFMT.jl") using .DSFMT using Base.GMP.MPZ using Base.GMP: Limb -import SHA +using SHA: SHA, SHA2_256_CTX, SHA2_512_CTX, SHA_CTX using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing import Base: copymutable, copy, copy!, ==, hash, convert, @@ -457,11 +457,14 @@ julia> rand(Xoshiro(), Bool) # not reproducible either true ``` """ -function seed!(rng::AbstractRNG, seed=nothing) +function seed!(rng::AbstractRNG, seed::Any=nothing) if seed === nothing seed!(rng, RandomDevice()) - else + elseif seed isa AbstractRNG + # avoid getting into an infinite recursive call from the other branches throw(MethodError(seed!, (rng, seed))) + else + seed!(rng, SeedHasher(seed)) end end diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index efeb39f65ab88..63a027046c545 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -246,11 +246,6 @@ hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2 seed!(rng::Union{TaskLocalRNG, Xoshiro}, seeder::AbstractRNG) = initstate!(rng, rand(seeder, NTuple{4, UInt64})) -seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) = @invoke seed!(rng::AbstractRNG, nothing::Any) - -seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) = - initstate!(rng, reinterpret(UInt64, hash_seed(seed))) - @inline function rand(x::Union{TaskLocalRNG, Xoshiro}, ::SamplerType{UInt64}) s0, s1, s2, s3 = getstate(x) diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index 2b99e50eb7e73..cbd8eb41f5515 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -11,8 +11,9 @@ using Random using Random.DSFMT using Random: default_rng, Sampler, SamplerRangeFast, SamplerRangeInt, SamplerRangeNDL, MT_CACHE_F, MT_CACHE_I -using Random: jump_128, jump_192, jump_128!, jump_192! +using Random: jump_128, jump_192, jump_128!, jump_192!, SeedHasher +import SHA import Future # randjump function test_uniform(xs::AbstractArray{T}) where {T<:AbstractFloat} @@ -297,7 +298,7 @@ for f in (:<, :<=, :>, :>=, :(==), :(!=)) end # test all rand APIs -for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro()]) +for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro(0)], [SeedHasher(0)]) realrng = rng == [] ? default_rng() : only(rng) ftypes = [Float16, Float32, Float64, FakeFloat64, BigFloat] cftypes = [ComplexF16, ComplexF32, ComplexF64, ftypes...] @@ -453,7 +454,8 @@ function hist(X, n) end @testset "uniform distribution of floats" begin - for rng in [MersenneTwister(), RandomDevice(), Xoshiro()], + seed = rand(UInt128) + for rng in [MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)], T in [Float16, Float32, Float64, BigFloat], prec in (T == BigFloat ? [3, 53, 64, 100, 256, 1000] : [256]) @@ -480,7 +482,8 @@ end # but also for 3 linear combinations of positions (for the array version) lcs = unique!.([rand(1:n, 2), rand(1:n, 3), rand(1:n, 5)]) aslcs = zeros(Int, 3) - for rng = (MersenneTwister(), RandomDevice(), Xoshiro()) + seed = rand(UInt128) + for rng = (MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)) for scalar = [false, true] fill!(a, 0) fill!(as, 0) @@ -662,7 +665,7 @@ end @testset "Random.seed!(rng, ...) returns rng" begin # issue #21248 seed = rand(UInt) - for m = ([MersenneTwister(seed)], [Xoshiro(seed)], []) + for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [SeedHasher(seed)], []) m2 = m == [] ? default_rng() : m[1] @test Random.seed!(m...) === m2 @test Random.seed!(m..., rand(UInt)) === m2 @@ -708,7 +711,7 @@ end # this shouldn't crash (#22403) @test_throws MethodError rand!(Union{UInt,Int}[1, 2, 3]) -@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro) +@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro, SeedHasher) m = RNG() a = rand(m, Int) m = RNG() @@ -729,7 +732,7 @@ end @test rand(m, Int) ∉ (a, b, c, d) end -@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro) +@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro, SeedHasher) seeds = Any[0, 1, 2, 10000, 10001, rand(UInt32, 8), randstring(), randstring(), rand(UInt128, 3)...] if RNG == Xoshiro push!(seeds, rand(UInt64, rand(1:4))) @@ -769,7 +772,10 @@ struct RandomStruct23964 end @test_throws MethodError rand(RandomStruct23964()) end -@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), RandomDevice(), Xoshiro()), +@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), + RandomDevice(), + Xoshiro(rand(UInt128)), + SeedHasher(rand(UInt128))), T ∈ (Bool, Int8, Int16, Int32, UInt32, Int64, Int128, UInt128) if T === Bool @test rand(RNG, false:true) ∈ (false, true) @@ -912,8 +918,11 @@ end @test rand(rng) == rand(GLOBAL_RNG) end -@testset "RNGs broadcast as scalars: T" for T in (MersenneTwister, RandomDevice) - @test length.(rand.(T(), 1:3)) == 1:3 +@testset "RNGs broadcast as scalars: $(typeof(RNG))" for RNG in (MersenneTwister(0), + RandomDevice(), + Xoshiro(0), + SeedHasher(0)) + @test length.(rand.(RNG, 1:3)) == 1:3 end @testset "generated scalar integers do not overlap" begin @@ -1211,7 +1220,14 @@ end end end + @testset "seed! and hash_seed" begin + function hash_seed(seed) + ctx = SHA.SHA2_256_CTX() + Random.hash_seed(seed, ctx) + bytes2hex(SHA.digest!(ctx)) + end + # Test that: # 1) if n == m, then hash_seed(n) == hash_seed(m) # 2) if n != m, then hash_seed(n) != hash_seed(m) @@ -1224,12 +1240,12 @@ end T <: Signed && push!(seeds, T(0), T(1), T(2), T(-1), T(-2)) end - vseeds = Dict{Vector{UInt8}, BigInt}() + vseeds = Dict{String, BigInt}() for seed = seeds bigseed = big(seed) - vseed = Random.hash_seed(bigseed) + vseed = hash_seed(bigseed) # test property 1) above - @test Random.hash_seed(seed) == vseed + @test hash_seed(seed) == vseed # test property 2) above @test bigseed == get!(vseeds, vseed, bigseed) # test that the property 1) is actually inherited by `seed!` @@ -1241,16 +1257,16 @@ end end seed32 = rand(UInt32, rand(1:9)) - hash32 = Random.hash_seed(seed32) - @test Random.hash_seed(map(UInt64, seed32)) == hash32 + hash32 = hash_seed(seed32) + @test hash_seed(map(UInt64, seed32)) == hash32 @test hash32 ∉ keys(vseeds) seed_str = randstring() seed_gstr = GenericString(seed_str) - @test Random.hash_seed(seed_str) == Random.hash_seed(seed_gstr) - string_seeds = Set{Vector{UInt8}}() + @test hash_seed(seed_str) == hash_seed(seed_gstr) + string_seeds = Set{String}() for ch = 'A':'z' - vseed = Random.hash_seed(string(ch)) + vseed = hash_seed(string(ch)) @test vseed ∉ keys(vseeds) @test vseed ∉ string_seeds push!(string_seeds, vseed) From f03df78bb0dff0d18628a85439bfbbf064857b32 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Mon, 5 May 2025 10:16:13 +0900 Subject: [PATCH 197/662] docs: some minor follow-up to JuliaLang/julia#58253 (#58314) - indent the numbered list so that it is rendered nicely - add the world age docs to the manual index --- doc/make.jl | 1 + doc/src/manual/worldage.md | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index 068531be24a1d..f149e4b37a9e9 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -173,6 +173,7 @@ Manual = [ "manual/noteworthy-differences.md", "manual/unicode-input.md", "manual/command-line-interface.md", + "manual/worldage.md", ] BaseDocs = [ diff --git a/doc/src/manual/worldage.md b/doc/src/manual/worldage.md index 378b5a1b8f4e9..26853b84b3031 100644 --- a/doc/src/manual/worldage.md +++ b/doc/src/manual/worldage.md @@ -82,15 +82,15 @@ will raise the current task's world age to the latest global world age, thus mak (both from the current task and any concurrently executing other tasks) visible. The following statements raise the current world age: - 1. An explicit invocation of `Core.@latestworld` - 2. The start of every top-level statement - 3. The start of every REPL prompt - 4. Any type or struct definition - 5. Any method definition - 6. Any constant declaration - 7. Any global variable declaration (but not a global variable assignment) - 8. Any `using`, `import`, `export` or `public` statement - 9. Certain other macros like [`@eval`](@ref) (depends on the macro implementation) +1. An explicit invocation of `Core.@latestworld` +2. The start of every top-level statement +3. The start of every REPL prompt +4. Any type or struct definition +5. Any method definition +6. Any constant declaration +7. Any global variable declaration (but not a global variable assignment) +8. Any `using`, `import`, `export` or `public` statement +9. Certain other macros like [`@eval`](@ref) (depends on the macro implementation) Note, however, that the current task's world age may only ever be permanently incremented at top level. As a general rule, using any of the above statements in non-top-level scope is a syntax error: From 8c5952b465f8a3aa0126019e463604407431e3ad Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 5 May 2025 08:09:42 +0200 Subject: [PATCH 198/662] Add 1.12 NEWS entry on binding partition (#58297) Will need to be manually backported since this is in NEWS.md on 1.12 rather than HISTORY. Closes #57596 --------- Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> --- HISTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.md b/HISTORY.md index 0c55b0954a0f8..d6b5b380ab364 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,7 @@ New language features * New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). +* Redefinition of constants is now well defined and follows world age semantics. Additional redefinitions (e.g. of structs) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). * A new keyword argument `usings::Bool` has been added to `names`, returning all names visible via `using` ([#54609]). * The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, From a909276f92e06e9e65b3a019c0a933e69da450fc Mon Sep 17 00:00:00 2001 From: NegaScout <42321060+NegaScout@users.noreply.github.com> Date: Mon, 5 May 2025 11:47:42 +0200 Subject: [PATCH 199/662] RFC: fix Base.GMP.MPZ.export! on uninitialized vectors (#57787) fixing Base.GMP.MPZ.export! not initializing memory when resizing vector fixing Base.GMP.MPZ.export! not initializing memory, nonzero vectors are only partially overwritten by :__gmpz_export, leading to unexpected results fixing Base.GMP.MPZ.export! not initializing memory when `n` is zero. if `n` is zero :__gmpz_export does not touch target vector adding test cases for fixed issues --------- Co-authored-by: Rafael Fourquet --- base/gmp.jl | 1 + test/gmp.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/base/gmp.jl b/base/gmp.jl index 465b4b6a9772d..dcd50c6022d84 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -257,6 +257,7 @@ function export!(a::AbstractVector{T}, n::BigInt; order::Integer=-1, nails::Inte stride(a, 1) == 1 || throw(ArgumentError("a must have stride 1")) ndigits = cld(sizeinbase(n, 2), 8*sizeof(T) - nails) length(a) < ndigits && resize!(a, ndigits) + fill!(a, zero(T)) count = Ref{Csize_t}() ccall((:__gmpz_export, libgmp), Ptr{T}, (Ptr{T}, Ref{Csize_t}, Cint, Csize_t, Cint, Csize_t, mpz_t), a, count, order, sizeof(T), endian, nails, n) diff --git a/test/gmp.jl b/test/gmp.jl index 0812775672969..8caf69b36fcc1 100644 --- a/test/gmp.jl +++ b/test/gmp.jl @@ -481,6 +481,18 @@ end bytes_to_export_to = Vector{UInt8}(undef, 2) Base.GMP.MPZ.export!(bytes_to_export_to, int_to_export_from, order=0) @test all(bytes_to_export_to .== bytes_to_import_from) + + # test export of 0 is T[0] + zero_to_export = BigInt(0) + bytes_to_export_to = Vector{UInt8}(undef, 0) + Base.GMP.MPZ.export!(bytes_to_export_to, zero_to_export, order=0) + @test bytes_to_export_to == UInt8[0] + + # test export on nonzero vector + x_to_export = BigInt(6) + bytes_to_export_to = UInt8[1, 2, 3, 4, 5] + Base.GMP.MPZ.export!(bytes_to_export_to, x_to_export, order=0) + @test bytes_to_export_to == UInt8[6, 0, 0, 0, 0] end @test isqrt(big(4)) == 2 From e9d534ce8e201b15a8188bb3b973000b489fa769 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Fri, 2 May 2025 13:30:18 -0400 Subject: [PATCH 200/662] trimming: follow-ups to finalizer support --- Compiler/src/typeinfer.jl | 23 +++++++++++------------ Compiler/src/verifytrim.jl | 35 +++++++++++++++++++++++++---------- Compiler/test/verifytrim.jl | 18 ++++++++++++++++++ HISTORY.md | 7 +++++-- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 93b29a22ab7be..42da01fbd0f3e 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1611,11 +1611,14 @@ const TRIM_UNSAFE = 2 const TRIM_UNSAFE_WARN = 3 function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + + # Create an "invokelatest" queue to enable eager compilation of speculative + # invokelatest calls such as from `Core.finalizer` and `ccallable` invokelatest_queue = CompilationQueue(; interp = NativeInterpreter(get_world_counter(); inf_params) ) + codeinfos = [] - is_latest_world = true # whether this_world == world_counter() workqueue = CompilationQueue(; interp = nothing) for this_world in reverse!(sort!(worlds)) workqueue = CompilationQueue(workqueue; @@ -1623,17 +1626,13 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m ) append!(workqueue, methods) - if is_latest_world - # Provide the `invokelatest` queue so that we trigger "best-effort" code generation - # for, e.g., finalizers and cfunction. - # - # The queue is intentionally aliased, to handle e.g. a `finalizer` calling `Core.finalizer` - # (it will enqueue into itself and immediately drain) - compile!(codeinfos, workqueue; invokelatest_queue = workqueue) - else - compile!(codeinfos, workqueue) - end - is_latest_world = false + compile!(codeinfos, workqueue; invokelatest_queue) + end + + if invokelatest_queue !== nothing + # This queue is intentionally aliased, to handle e.g. a `finalizer` calling `Core.finalizer` + # (it will enqueue into itself and immediately drain) + compile!(codeinfos, invokelatest_queue; invokelatest_queue) end if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 19d8bcd623275..09a189b2ff223 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -15,9 +15,9 @@ using ..Compiler: hasintersect, haskey, in, isdispatchelem, isempty, isexpr, iterate, length, map!, max, pop!, popfirst!, push!, pushfirst!, reinterpret, reverse!, reverse, setindex!, setproperty!, similar, singleton_type, sptypes_from_meth_instance, - unsafe_pointer_to_objref, widenconst, + unsafe_pointer_to_objref, widenconst, isconcretetype, # misc - @nospecialize, C_NULL + @nospecialize, @assert, C_NULL using ..IRShow: LineInfoNode, print, show, println, append_scopes!, IOContext, IO, normalize_method_name using ..Base: Base, sourceinfo_slotnames using ..Base.StackTraces: StackFrame @@ -166,7 +166,7 @@ function may_dispatch(@nospecialize ftyp) end end -function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspected::IdSet{CodeInstance}, caches::IdDict{MethodInstance,CodeInstance}, parents::ParentMap, errors::ErrorList) +function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, codeinfo::CodeInfo, inspected::IdSet{CodeInstance}, caches::IdDict{MethodInstance,CodeInstance}, parents::ParentMap, errors::ErrorList) mi = get_ci_mi(codeinst) sptypes = sptypes_from_meth_instance(mi) src = codeinfo.code @@ -199,9 +199,9 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec if !may_dispatch(ftyp) continue end - # TODO: Make interp elsewhere - interp = NativeInterpreter(Base.get_world_counter()) - if Core._apply_iterate isa ftyp + if !isconcretetype(ftyp) + error = "unresolved call to (unknown) builtin" + elseif Core._apply_iterate isa ftyp if length(stmt.args) >= 3 # args[1] is _apply_iterate object # args[2] is invoke object @@ -233,9 +233,23 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec error = "unresolved finalizer registered" end - else - error = "unresolved call to builtin" - end + elseif Core._apply isa ftyp + error = "trim verification not yet implemented for builtin `Core._apply`" + elseif Core._call_in_world_total isa ftyp + error = "trim verification not yet implemented for builtin `Core._call_in_world_total`" + elseif Core.invoke isa ftyp + error = "trim verification not yet implemented for builtin `Core.invoke`" + elseif Core.invoke_in_world isa ftyp + error = "trim verification not yet implemented for builtin `Core.invoke_in_world`" + elseif Core.invokelatest isa ftyp + error = "trim verification not yet implemented for builtin `Core.invokelatest`" + elseif Core.modifyfield! isa ftyp + error = "trim verification not yet implemented for builtin `Core.modifyfield!`" + elseif Core.modifyglobal! isa ftyp + error = "trim verification not yet implemented for builtin `Core.modifyglobal!`" + elseif Core.memoryrefmodify! isa ftyp + error = "trim verification not yet implemented for builtin `Core.memoryrefmodify!`" + else @assert false "unexpected builtin" end end extyp = argextype(SSAValue(i), codeinfo, sptypes) if extyp === Union{} @@ -267,6 +281,7 @@ end function get_verify_typeinf_trim(codeinfos::Vector{Any}) this_world = get_world_counter() + interp = NativeInterpreter(this_world) inspected = IdSet{CodeInstance}() caches = IdDict{MethodInstance,CodeInstance}() errors = ErrorList() @@ -287,7 +302,7 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) item = codeinfos[i] if item isa CodeInstance src = codeinfos[i + 1]::CodeInfo - verify_codeinstance!(item, src, inspected, caches, parents, errors) + verify_codeinstance!(interp, item, src, inspected, caches, parents, errors) elseif item isa SimpleVector rt = item[1]::Type sig = item[2]::Type diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index ad32224be7e15..e7d57571b1db0 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -18,6 +18,24 @@ let infos = Any[] @test isempty(parents) end +finalizer(@nospecialize(f), @nospecialize(o)) = Core.finalizer(f, o) + +let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), typeof(identity), Any})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test !isempty(errors) # unresolvable finalizer + + # the only error should be a CallMissing error for the Core.finalizer builtin + (warn, desc) = only(errors) + @test !warn + @test desc isa CallMissing + @test occursin("finalizer", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test occursin( + r"""^unresolved finalizer registered from statement \(Core.finalizer\)\(f::Any, o::Any\)::Nothing + Stacktrace: + \[1\] finalizer\(f::Any, o::Any\)""", repr) +end + make_cfunction() = @cfunction(+, Float64, (Int64,Int64)) # use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel diff --git a/HISTORY.md b/HISTORY.md index d6b5b380ab364..8f4168dd7fcfb 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,8 +5,11 @@ New language features --------------------- * New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from - entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). -* Redefinition of constants is now well defined and follows world age semantics. Additional redefinitions (e.g. of structs) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). + entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). To support + Core.finalizer, inference will now opportunistically discover future invokelatest calls and compile + the required code for them. +* Redefinition of constants is now well defined and follows world age semantics. Additional redefinitions + (e.g. of structs) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). * A new keyword argument `usings::Bool` has been added to `names`, returning all names visible via `using` ([#54609]). * The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, From da6356bdf42bbd7219c20b1e08fb13b76b686fe4 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 5 May 2025 15:33:36 -0700 Subject: [PATCH 201/662] Fix completing positional arguments if a semicolon exists beyond the cursor (#58298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a stopgap solution. For now, have `find_prefix_call` tell us whether the cursor is before the `;` (:positional) or after (:kwargs), and set `exact_nargs` only when it is :kwargs. Eventually, we should remove kwargs_flag entirely and have the method completions use our precise position information. Fixes #58296, and the related issue I mention in https://github.com/JuliaLang/julia/issues/58296#issuecomment-2845310701. --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> Co-authored-by: Jameson Nash --- stdlib/REPL/src/REPLCompletions.jl | 36 ++++++++++++++++++----------- stdlib/REPL/test/replcompletions.jl | 14 +++++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 67145cdbd1714..8b8fb64b07dae 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -719,9 +719,12 @@ function _complete_methods(ex_org::Expr, context_module::Module, shift::Bool) return kwargs_flag, funct, args_ex, kwargs_ex end -function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool=false) +# cursor_pos: either :positional (complete either kwargs or positional) or :kwargs (beyond semicolon) +function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool=false, cursor_pos::Symbol=:positional) kwargs_flag, funct, args_ex, kwargs_ex = _complete_methods(ex_org, context_module, shift)::Tuple{Int, Any, Vector{Any}, Set{Symbol}} out = Completion[] + # Allow more arguments when cursor before semicolon, even if kwargs are present + cursor_pos == :positional && kwargs_flag == 1 && (kwargs_flag = 0) kwargs_flag == 2 && return out # one of the kwargs is invalid kwargs_flag == 0 && push!(args_ex, Vararg{Any}) # allow more arguments if there is no semicolon complete_methods!(out, funct, args_ex, kwargs_ex, shift ? -2 : MAX_METHOD_COMPLETIONS, kwargs_flag == 1) @@ -898,14 +901,16 @@ end end # Provide completion for keyword arguments in function calls +# Returns true if the current argument must be a keyword because the cursor is beyond the semicolon function complete_keyword_argument!(suggestions::Vector{Completion}, ex::Expr, last_word::String, - context_module::Module; shift::Bool=false) + context_module::Module, + arg_pos::Symbol; shift::Bool=false) kwargs_flag, funct, args_ex, kwargs_ex = _complete_methods(ex, context_module, true)::Tuple{Int, Any, Vector{Any}, Set{Symbol}} - kwargs_flag == 2 && false # one of the previous kwargs is invalid + kwargs_flag == 2 && return false # one of the previous kwargs is invalid methods = Completion[] - complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, shift ? -1 : MAX_METHOD_COMPLETIONS, kwargs_flag == 1) + complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, shift ? -1 : MAX_METHOD_COMPLETIONS, arg_pos == :kwargs) # TODO: use args_ex instead of Any[Vararg{Any}] and only provide kwarg completion for # method calls compatible with the current arguments. @@ -935,7 +940,7 @@ function complete_keyword_argument!(suggestions::Vector{Completion}, for kwarg in kwargs push!(suggestions, KeywordArgumentCompletion(kwarg)) end - return kwargs_flag != 0 + return kwargs_flag != 0 && arg_pos == :kwargs end function get_loading_candidates(pkgstarts::String, project_file::String) @@ -1071,7 +1076,8 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif (kind(cur) in KSet"String Comment ErrorEofMultiComment" || inside_cmdstr) && return Completion[], 1:0, false - if (n = find_prefix_call(cur_not_ws)) !== nothing + n, arg_pos = find_prefix_call(cur_not_ws) + if n !== nothing func = first(children_nt(n)) e = Expr(n) # Remove arguments past the first parse error (allows unclosed parens) @@ -1088,7 +1094,7 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif # foo(x, TAB => list of methods signatures for foo with x as first argument if kind(cur_not_ws) in KSet"( , ;" # Don't provide method completions unless the cursor is after: '(' ',' ';' - return complete_methods(e, context_module, shift), char_range(func), false + return complete_methods(e, context_module, shift, arg_pos), char_range(func), false # Keyword argument completion: # foo(ar TAB => keyword arguments like `arg1=` @@ -1096,7 +1102,7 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif r = char_range(cur) s = string[intersect(r, 1:pos)] # Return without adding more suggestions if kwargs only - complete_keyword_argument!(suggestions, e, s, context_module; shift) && + complete_keyword_argument!(suggestions, e, s, context_module, arg_pos; shift) && return sort_suggestions(), r, true end end @@ -1184,18 +1190,20 @@ function find_str(cur::CursorNode) end # Is the cursor directly inside of the arguments of a prefix call (no nested -# expressions)? +# expressions)? If so, return: +# - The call node +# - Either :positional or :kwargs, if the cursor is before or after the `;` function find_prefix_call(cur::CursorNode) n = cur.parent - n !== nothing || return nothing + n !== nothing || return nothing, nothing is_call(n) = kind(n) in KSet"call dotcall" && is_prefix_call(n) if kind(n) == K"parameters" - is_call(n.parent) || return nothing - n.parent + is_call(n.parent) || return nothing, nothing + n.parent, :kwargs else # Check that we are beyond the function name. - is_call(n) && cur.index > children_nt(n)[1].index || return nothing - n + is_call(n) && cur.index > children_nt(n)[1].index || return nothing, nothing + n, :positional end end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 5569b93640bd8..047a53edac1e5 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -2650,3 +2650,17 @@ let s = "\"example: \$(named.len" @test "len2" in c @test r == 19:21 end + +# #58296 - complete positional arguments before semicolon +let s = "string(findfi|; base=16)" + c, r = test_complete_pos(s) + @test "findfirst" in c + @test r == 8:13 +end + +# Unknown functions should not cause completions to fail +let s = "foo58296(findfi" + c, r = test_complete(s) + @test "findfirst" in c + @test r == 10:15 +end From 7c40a71dac3725b077353bd59062b8a1daddcb51 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 6 May 2025 13:03:28 +0900 Subject: [PATCH 202/662] move to a simpler versioning policy for the Compiler.jl stdlib (#57520) This commit adjusts /Compiler/Project.toml and adds a new /Compiler/README.md based on the new versioning policy for the Compiler.jl stdlib. Regarding the new versioning policy and the latest way to use the Compiler.jl stdlib, please refer to this comment[^1] and the newly added READMEs (`JuliaLang/julia/Compiler/README.md`[^2] and and `JuliaLang/BaseCompiler.jl/README.md`[^3]). [^1]: https://github.com/JuliaLang/julia/pull/57520#issuecomment-2811922086 [^2]: https://github.com/JuliaLang/julia/blob/a8f3e4d0603692ac79700890d3e8d35e904e000e/Compiler/README.md [^3]: https://github.com/JuliaLang/BaseCompiler.jl --- Compiler/Project.toml | 2 +- Compiler/README.md | 45 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 Compiler/README.md diff --git a/Compiler/Project.toml b/Compiler/Project.toml index 994634f5a8b78..7113a60318984 100644 --- a/Compiler/Project.toml +++ b/Compiler/Project.toml @@ -1,6 +1,6 @@ name = "Compiler" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.0.3" +version = "0.0.0" [compat] julia = "1.10" diff --git a/Compiler/README.md b/Compiler/README.md new file mode 100644 index 0000000000000..aa1f6a0c92827 --- /dev/null +++ b/Compiler/README.md @@ -0,0 +1,45 @@ +# The `Compiler` module + +This directory maintains the implementation of the Julia compiler. + +Through a bootstrapping process, it is bundled into the Julia runtime as `Base.Compiler`. + +You can also use this `Compiler` module as the `Compiler` standard library by following the steps below. + +## How to use + +To utilize this `Compiler.jl` standard library, you need to declare it as a dependency in +your `Project.toml` as follows: +> Project.toml +```toml +[deps] +Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" + +[compat] +Compiler = "0" +``` + +With the setup above, [the special placeholder version (v0.0.0)](https://github.com/JuliaLang/BaseCompiler.jl) +will be installed by default.[^1] + +[^1]: Currently, only version v0.0.0 is registered in the [General](https://github.com/JuliaRegistries/General) registry. + +If needed, you can switch to a custom implementation of the `Compiler` module by running +```julia-repl +pkg> dev /path/to/Compiler.jl # to use a local implementation +``` +or +```julia-repl +pkg> add https://url/of/Compiler/branch # to use a remote implementation +``` +This feature is particularly useful for developing or experimenting with alternative compiler implementations. + +> [!note] +> The Compiler.jl standard library is available starting from Julia v1.10. +> However, switching to a custom compiler implementation is supported only from +> Julia v1.12 onwards. + +> [!warning] +> When using a custom, non-`Base` version of `Compiler` implementation, it may be necessary +> to run `InteractiveUtils.@activate Compiler` to ensure proper functionality of certain +> reflection utilities. From c894e049d189fb2a5c978a43ec8eab13f4424da8 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 6 May 2025 16:43:17 +0900 Subject: [PATCH 203/662] improve robustness of `Vararg` checks in newly added `abstract_eval_xxx` (#58325) Otherwise these subroutines may raise `unhandled Vararg` errors when compiling e.g. CassetteOverlay for complex call graphs. --- Compiler/src/abstractinterpretation.jl | 179 +++++++++++++++---------- Compiler/test/inference.jl | 13 ++ 2 files changed, 122 insertions(+), 70 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 8fe30d4891e9b..058476dac0096 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2426,11 +2426,15 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s end function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 3 - return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3]) - elseif length(argtypes) == 4 - return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 5 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 3 + return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3]) + elseif length(argtypes) == 4 + return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 5 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, generic_getglobal_exct, generic_getglobal_effects, NoCallInfo()) @@ -2471,12 +2475,17 @@ end end function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, argtypes::Vector{Any}) - if length(argtypes) == 3 - return abstract_eval_get_binding_type(interp, sv, argtypes[2], argtypes[3]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 4 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 3 + return abstract_eval_get_binding_type(interp, sv, argtypes[2], argtypes[3]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 4 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + else + return CallMeta(Type, Union{TypeError, ArgumentError}, EFFECTS_THROWS, NoCallInfo()) end - return CallMeta(Type, Union{TypeError, ArgumentError}, EFFECTS_THROWS, NoCallInfo()) end const setglobal!_effects = Effects(EFFECTS_TOTAL; effect_free=ALWAYS_FALSE, nothrow=false, inaccessiblememonly=ALWAYS_FALSE) @@ -2509,11 +2518,15 @@ end const generic_setglobal!_exct = Union{ArgumentError, TypeError, ErrorException, ConcurrencyViolationError} function abstract_eval_setglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 4 - return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif length(argtypes) == 5 - return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 4 + return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + elseif length(argtypes) == 5 + return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, generic_setglobal!_exct, setglobal!_effects, NoCallInfo()) @@ -2537,11 +2550,15 @@ function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, end function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 4 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif length(argtypes) == 5 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 4 + return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + elseif length(argtypes) == 5 + return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, Union{generic_getglobal_exct,generic_setglobal!_exct}, setglobal!_effects, NoCallInfo()) @@ -2549,18 +2566,22 @@ function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, end function abstract_eval_setglobalonce!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) in (4, 5, 6) - cm = abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - if length(argtypes) >= 5 - goe = global_order_exct(argtypes[5], #=loading=#true, #=storing=#true) - cm = merge_exct(cm, goe) - end - if length(argtypes) == 6 - goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#false) - cm = merge_exct(cm, goe) - end - return CallMeta(Bool, cm.exct, cm.effects, cm.info) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) in (4, 5, 6) + cm = abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + if length(argtypes) >= 5 + goe = global_order_exct(argtypes[5], #=loading=#true, #=storing=#true) + cm = merge_exct(cm, goe) + end + if length(argtypes) == 6 + goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#false) + cm = merge_exct(cm, goe) + end + return CallMeta(Bool, cm.exct, cm.effects, cm.info) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 7 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Bool, generic_setglobal!_exct, setglobal!_effects, NoCallInfo()) @@ -2568,44 +2589,48 @@ function abstract_eval_setglobalonce!(interp::AbstractInterpreter, sv::AbsIntSta end function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) in (5, 6, 7) - (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] - T = nothing - if isa(M, Const) && isa(s, Const) - M, s = M.val, s.val - M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - gr = GlobalRef(M, s) - v′ = RefValue{Any}(v) - (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition - partition_T = nothing - partition_rte = abstract_eval_partition_load(interp, binding, partition) - if binding_kind(partition) == PARTITION_KIND_GLOBAL - partition_T = partition_restriction(partition) + if !isvarargtype(argtypes[end]) + if length(argtypes) in (5, 6, 7) + (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] + T = nothing + if isa(M, Const) && isa(s, Const) + M, s = M.val, s.val + M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) + s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) + gr = GlobalRef(M, s) + v′ = RefValue{Any}(v) + (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition + partition_T = nothing + partition_rte = abstract_eval_partition_load(interp, binding, partition) + if binding_kind(partition) == PARTITION_KIND_GLOBAL + partition_T = partition_restriction(partition) + end + partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]} + partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) + Pair{RTEffects, Any}(partition_rte, partition_T) end - partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]} - partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) - Pair{RTEffects, Any}(partition_rte, partition_T) + update_valid_age!(sv, valid_worlds) + effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) + sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) + else + sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) + end + if length(argtypes) >= 6 + goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#true) + sg = merge_exct(sg, goe) + end + if length(argtypes) == 7 + goe = global_order_exct(argtypes[7], #=loading=#true, #=storing=#false) + sg = merge_exct(sg, goe) end - update_valid_age!(sv, valid_worlds) - effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) - sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) + rt = T === nothing ? + ccall(:jl_apply_cmpswap_type, Any, (Any,), S) where S : + ccall(:jl_apply_cmpswap_type, Any, (Any,), T) + return CallMeta(rt, sg.exct, sg.effects, sg.info) else - sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) - end - if length(argtypes) >= 6 - goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#true) - sg = merge_exct(sg, goe) - end - if length(argtypes) == 7 - goe = global_order_exct(argtypes[7], #=loading=#true, #=storing=#false) - sg = merge_exct(sg, goe) + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) end - rt = T === nothing ? - ccall(:jl_apply_cmpswap_type, Any, (Any,), S) where S : - ccall(:jl_apply_cmpswap_type, Any, (Any,), T) - return CallMeta(rt, sg.exct, sg.effects, sg.info) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 8 + elseif length(argtypes) > 8 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, Union{generic_getglobal_exct,generic_setglobal!_exct}, setglobal!_effects, NoCallInfo()) @@ -2662,11 +2687,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return Future(abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], Const(true), length(argtypes) == 4 ? argtypes[4] : Const(:unordered), si.saw_latestworld, sv)) - elseif f === Core.isdefinedglobal && 3 <= length(argtypes) <= 5 - return Future(abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], - length(argtypes) >= 4 ? argtypes[4] : Const(true), - length(argtypes) >= 5 ? argtypes[5] : Const(:unordered), - si.saw_latestworld, sv)) + elseif f === Core.isdefinedglobal + return Future(abstract_eval_isdefinedglobal(interp, sv, si.saw_latestworld, argtypes)) elseif f === Core.get_binding_type return Future(abstract_eval_get_binding_type(interp, sv, argtypes)) end @@ -3347,6 +3369,23 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, @nospecializ return CallMeta(Bool, Union{exct, TypeError, UndefVarError}, generic_isdefinedglobal_effects, NoCallInfo()) end +function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) + if !isvarargtype(argtypes[end]) + if 3 <= length(argtypes) <= 5 + return abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], + length(argtypes) >= 4 ? argtypes[4] : Const(true), + length(argtypes) >= 5 ? argtypes[5] : Const(:unordered), + saw_latestworld, sv) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + else + return CallMeta(Bool, Union{ConcurrencyViolationError, TypeError, UndefVarError}, generic_isdefinedglobal_effects, NoCallInfo()) + end +end + function abstract_eval_throw_undef_if_not(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) condt = abstract_eval_value(interp, e.args[2], sstate, sv) condval = maybe_extract_const_bool(condt) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 8333b191c75c9..45f67129039f4 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6486,4 +6486,17 @@ function ss57873(a::Vector{String}, pref) end @test ss57873(["a", "b", "c"], ("",)) == String[] +@test Base.infer_return_type((Module,Symbol,Vector{Any})) do m, n, xs + getglobal(m, n, xs...) +end <: Any +@test Base.infer_return_type((Module,Symbol,Any,Vector{Any})) do m, n, v, xs + setglobal!(m, n, v, xs...) +end <: Any +@test Base.infer_return_type((Module,Symbol,Vector{Any})) do m, n, xs + isdefinedglobal(m, n, xs...) +end <: Bool +@test Base.infer_return_type((Module,Symbol,Vector{Any})) do m, n, xs + Core.get_binding_type(m, n, xs...) +end <: Type + end # module inference From 088bb9002e95631738c8ec5ba58b7b8a7b33019d Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 6 May 2025 06:08:40 -0300 Subject: [PATCH 204/662] Fix removal of globals with addrspaces in removeAddrspaces (#58322) --- src/llvm-remove-addrspaces.cpp | 4 ++-- test/llvmpasses/remove-addrspaces.ll | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/llvm-remove-addrspaces.cpp b/src/llvm-remove-addrspaces.cpp index bb492f467e74c..78ff70b12409b 100644 --- a/src/llvm-remove-addrspaces.cpp +++ b/src/llvm-remove-addrspaces.cpp @@ -256,7 +256,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) Name, (GlobalVariable *)nullptr, GV->getThreadLocalMode(), - GV->getType()->getAddressSpace()); + cast(TypeRemapper.remapType(GV->getType()))->getAddressSpace()); NGV->copyAttributesFrom(GV); VMap[GV] = NGV; } @@ -276,7 +276,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) auto *NGA = GlobalAlias::create( TypeRemapper.remapType(GA->getValueType()), - GA->getType()->getPointerAddressSpace(), + cast(TypeRemapper.remapType(GA->getType()))->getAddressSpace(), GA->getLinkage(), Name, &M); diff --git a/test/llvmpasses/remove-addrspaces.ll b/test/llvmpasses/remove-addrspaces.ll index fbd84de85a4a3..99acd92b0e03b 100644 --- a/test/llvmpasses/remove-addrspaces.ll +++ b/test/llvmpasses/remove-addrspaces.ll @@ -2,6 +2,9 @@ ; RUN: opt --load-pass-plugin=libjulia-codegen%shlibext -passes='RemoveJuliaAddrspaces' -S %s | FileCheck %s --check-prefixes=CHECK,OPAQUE +; COM: check that the addrspace of the global itself is removed +; OPAQUE: @ejl_enz_runtime_exc = external global {} +@ejl_enz_runtime_exc = external addrspace(10) global {} ; COM: check that package image fptrs work @pjlsys_BoundsError_32 = internal global {} addrspace(10)* ({}***, {} addrspace(10)*, [1 x i64] addrspace(11)*)* null @@ -111,6 +114,13 @@ define void @byval_type([1 x {} addrspace(10)*] addrspace(11)* byval([1 x {} add } +define private fastcc void @diffejulia__mapreduce_97() { +L6: +; OPAQUE: store atomic ptr @ejl_enz_runtime_exc, ptr null unordered + store atomic {} addrspace(10)* @ejl_enz_runtime_exc, {} addrspace(10)* addrspace(10)* null unordered, align 8 + unreachable +} + ; COM: check that function attributes are preserved on declarations too declare void @convergent_function() #0 attributes #0 = { convergent } From 3d48f59a138a70d3e5d48021f7a4a9a98e0843bb Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 6 May 2025 06:12:36 -0300 Subject: [PATCH 205/662] Don't use sigatomic before current task is available (#58320) --- src/staticdata.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index df38ad1fc0277..f42fc38828f65 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3593,8 +3593,6 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) jl_errorf("System image file \"%s\" not found.", fname); ios_bufmode(&f, bm_none); - JL_SIGATOMIC_BEGIN(); - ios_seek_end(&f); size_t len = ios_pos(&f); char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); @@ -3605,8 +3603,6 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) ios_close(&f); - JL_SIGATOMIC_END(); - jl_sysimage_buf = (jl_image_buf_t) { .kind = JL_IMAGE_KIND_JI, .pointers = NULL, From b43baa134f5212b1ff4ce8bbaae50f4b62170e1d Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 6 May 2025 20:05:59 +0900 Subject: [PATCH 206/662] use `src.nargs` for `validate_code!` (#58327) This is a follow-up to https://github.com/JuliaLang/julia/pull/54341. Otherwise, `validate_code!` may raise wrong errors for unmatched `nargs` information between generated code and original method. --- Compiler/src/validation.jl | 3 ++- Compiler/test/contextual.jl | 53 +++++++++++++++++++++++++++---------- Compiler/test/validation.jl | 34 +++++++----------------- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/Compiler/src/validation.jl b/Compiler/src/validation.jl index 4f9362e97b30d..d5faf51a89356 100644 --- a/Compiler/src/validation.jl +++ b/Compiler/src/validation.jl @@ -225,7 +225,7 @@ function validate_code!(errors::Vector{InvalidCodeError}, mi::Core.MethodInstanc mnargs = 0 else m = mi.def::Method - mnargs = m.nargs + mnargs = Int(m.nargs) n_sig_params = length((unwrap_unionall(m.sig)::DataType).parameters) if m.is_for_opaque_closure m.sig === Tuple || push!(errors, InvalidCodeError(INVALID_SIGNATURE_OPAQUE_CLOSURE, (m.sig, m.isva))) @@ -234,6 +234,7 @@ function validate_code!(errors::Vector{InvalidCodeError}, mi::Core.MethodInstanc end end if isa(c, CodeInfo) + mnargs = Int(c.nargs) mnargs > length(c.slotnames) && push!(errors, InvalidCodeError(SLOTNAMES_NARGS_MISMATCH)) validate_code!(errors, c, is_top_level) end diff --git a/Compiler/test/contextual.jl b/Compiler/test/contextual.jl index 4550501f5546e..941ce172d41e2 100644 --- a/Compiler/test/contextual.jl +++ b/Compiler/test/contextual.jl @@ -1,20 +1,23 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +module contextual + # N.B.: This file is also run from interpreter.jl, so needs to be standalone-executable using Test -include("setup_Compiler.jl") - # Cassette # ======== +# TODO Use CassetteBase.jl instead of this mini-cassette? + module MiniCassette # A minimal demonstration of the cassette mechanism. Doesn't support all the # fancy features, but sufficient to exercise this code path in the compiler. + using Core: SimpleVector using Core.IR - using ..Compiler - using ..Compiler: retrieve_code_info, quoted, anymap + using Base: Compiler as CC + using .CC: retrieve_code_info, quoted, anymap using Base.Meta: isexpr export Ctx, overdub @@ -22,7 +25,7 @@ module MiniCassette struct Ctx; end # A no-op cassette-like transform - function transform_expr(expr, map_slot_number, map_ssa_value, sparams::Core.SimpleVector) + function transform_expr(expr, map_slot_number, map_ssa_value, sparams::SimpleVector) @nospecialize expr transform(@nospecialize expr) = transform_expr(expr, map_slot_number, map_ssa_value, sparams) if isexpr(expr, :call) @@ -46,11 +49,11 @@ module MiniCassette end end - function transform!(mi::MethodInstance, ci::CodeInfo, nargs::Int, sparams::Core.SimpleVector) + function transform!(mi::MethodInstance, ci::CodeInfo, nargs::Int, sparams::SimpleVector) code = ci.code - di = Compiler.DebugInfoStream(mi, ci.debuginfo, length(code)) - ci.slotnames = Symbol[Symbol("#self#"), :ctx, :f, :args, ci.slotnames[nargs+1:end]...] - ci.slotflags = UInt8[(0x00 for i = 1:4)..., ci.slotflags[nargs+1:end]...] + di = CC.DebugInfoStream(mi, ci.debuginfo, length(code)) + ci.slotnames = Symbol[Symbol("#self#"), :ctx, :f, :args, ci.slotnames[nargs+2:end]...] + ci.slotflags = UInt8[(0x00 for i = 1:4)..., ci.slotflags[nargs+2:end]...] # Insert one SSAValue for every argument statement prepend!(code, Any[Expr(:call, getfield, SlotNumber(4), i) for i = 1:nargs]) prepend!(di.codelocs, fill(Int32(0), 3nargs)) @@ -77,21 +80,26 @@ module MiniCassette function overdub_generator(world::UInt, source, self, ctx, f, args) @nospecialize + argnames = Core.svec(:overdub, :ctx, :f, :args) + spnames = Core.svec() + if !Base.issingletontype(f) # (c, f, args..) -> f(args...) - ex = :(return f(args...)) - return Core.GeneratedFunctionStub(identity, Core.svec(:overdub, :ctx, :f, :args), Core.svec())(world, source, ex) + return generate_lambda_ex(world, source, argnames, spnames, :(return f(args...))) end tt = Tuple{f, args...} match = Base._which(tt; world) mi = Base.specialize_method(match) # Unsupported in this mini-cassette - @assert !mi.def.isva + !mi.def.isva || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unsupported vararg method"))) src = retrieve_code_info(mi, world) - @assert isa(src, CodeInfo) + isa(src, CodeInfo) || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unexpected code transformation"))) src = copy(src) - @assert src.edges === Core.svec() + src.edges === Core.svec() || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unexpected code transformation"))) src.edges = Any[mi] transform!(mi, src, length(args), match.sparams) # TODO: this is mandatory: code_info.min_world = max(code_info.min_world, min_world[]) @@ -99,9 +107,21 @@ module MiniCassette # Match the generator, since that's what our transform! does src.nargs = 4 src.isva = true + errors = CC.validate_code(mi, src) + if !isempty(errors) + foreach(Core.println, errors) + return generate_lambda_ex(world, source, argnames, spnames, :(error("Found errors in generated code"))) + end return src end + function generate_lambda_ex(world::UInt, source::Method, + argnames::SimpleVector, spnames::SimpleVector, + body::Expr) + stub = Core.GeneratedFunctionStub(identity, argnames, spnames) + return stub(world, source, body) + end + @inline overdub(::Ctx, f::Union{Core.Builtin, Core.IntrinsicFunction}, args...) = f(args...) @eval function overdub(ctx::Ctx, f, args...) @@ -125,3 +145,8 @@ f() = 2 foo(i) = i+bar(Val(1)) @test @inferred(overdub(Ctx(), foo, 1)) == 43 + +morethan4args(a, b, c, d, e) = (((a + b) + c) + d) + e +@test overdub(Ctx(), morethan4args, 1, 2, 3, 4, 5) == 15 + +end # module contextual diff --git a/Compiler/test/validation.jl b/Compiler/test/validation.jl index 5328516f63d36..f01ff85e4321c 100644 --- a/Compiler/test/validation.jl +++ b/Compiler/test/validation.jl @@ -20,13 +20,15 @@ function f22938(a, b, x...) return i * a end -msig = Tuple{typeof(f22938),Int,Int,Int,Int} -world = Base.get_world_counter() -match = only(Base._methods_by_ftype(msig, -1, world)) -mi = Compiler.specialize_method(match) -c0 = Compiler.retrieve_code_info(mi, world) - -@test isempty(Compiler.validate_code(mi, c0)) +const c0 = let + msig = Tuple{typeof(f22938),Int,Int,Int,Int} + world = Base.get_world_counter() + match = only(Base._methods_by_ftype(msig, -1, world)) + mi = Compiler.specialize_method(match) + c0 = Compiler.retrieve_code_info(mi, world) + @test isempty(Compiler.validate_code(mi, c0)) + c0 +end @testset "INVALID_EXPR_HEAD" begin c = copy(c0) @@ -114,15 +116,6 @@ end @test errors[1].kind === Compiler.SSAFLAGS_MISMATCH end -@testset "SIGNATURE_NARGS_MISMATCH" begin - old_sig = mi.def.sig - mi.def.sig = Tuple{1,2} - errors = Compiler.validate_code(mi, nothing) - mi.def.sig = old_sig - @test length(errors) == 1 - @test errors[1].kind === Compiler.SIGNATURE_NARGS_MISMATCH -end - @testset "NON_TOP_LEVEL_METHOD" begin c = copy(c0) c.code[1] = Expr(:method, :dummy) @@ -130,12 +123,3 @@ end @test length(errors) == 1 @test errors[1].kind === Compiler.NON_TOP_LEVEL_METHOD end - -@testset "SLOTNAMES_NARGS_MISMATCH" begin - mi.def.nargs += 20 - errors = Compiler.validate_code(mi, c0) - mi.def.nargs -= 20 - @test length(errors) == 2 - @test count(e.kind === Compiler.SLOTNAMES_NARGS_MISMATCH for e in errors) == 1 - @test count(e.kind === Compiler.SIGNATURE_NARGS_MISMATCH for e in errors) == 1 -end From 5eb515587f45b80b9650daf1f7fe11f2bbb87e34 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 6 May 2025 15:44:11 -0400 Subject: [PATCH 207/662] replace incorrect Method.deleted_world with more useful Method.dispatch_status enum (#58291) The original purpose of this field was to manage quickly detecting if a method was replaced, but that stopped being correct after #53415. It was a fairly heavy-weight description of that single bit of information. This bit of information allows quickly bypassing some method lookups from pkgimages, since it can quickly detect that the result is trivially correct (such as single-argument functions). Also fixes #58215 --- Compiler/src/typeinfer.jl | 2 +- base/errorshow.jl | 2 - base/staticdata.jl | 43 ++++++++++++++++----- src/gf.c | 78 ++++++++++++++++++++++++++------------- src/jltypes.c | 4 +- src/julia.h | 2 +- src/julia_internal.h | 4 ++ src/method.c | 4 +- src/opaque_closure.c | 1 - src/precompile_utils.c | 13 ++++--- src/staticdata.c | 15 +++----- src/staticdata_utils.c | 2 - src/typemap.c | 6 +-- test/core.jl | 2 +- test/worlds.jl | 2 + 15 files changed, 113 insertions(+), 67 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 42da01fbd0f3e..6eb4b140fd16b 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1540,7 +1540,7 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; # if this method is generally visible to the current compilation world, # and this is either the primary world, or not applicable in the primary world # then we want to compile and emit this - if item.def.primary_world <= world <= item.def.deleted_world + if item.def.primary_world <= world ci = typeinf_ext(interp, item, SOURCE_MODE_GET_SOURCE) ci isa CodeInstance && push!(workqueue, ci) end diff --git a/base/errorshow.jl b/base/errorshow.jl index 15e616fe482c5..2e798b60cf733 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -592,8 +592,6 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) end if ex.world < reinterpret(UInt, method.primary_world) print(iob, " (method too new to be called from this world context.)") - elseif ex.world > reinterpret(UInt, method.deleted_world) - print(iob, " (method deleted before this world age.)") end println(iob) diff --git a/base/staticdata.jl b/base/staticdata.jl index cb0888e9d56bb..0b65d97750194 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -288,6 +288,30 @@ end function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt) # verify that these edges intersect with the same methods as before + if n == 1 + # first, fast-path a check if the expected method simply dominates its sig anyways + # so the result of ml_matches is already simply known + let t = expecteds[i], meth, minworld, maxworld, result + if t isa Method + meth = t + else + if t isa CodeInstance + t = get_ci_mi(t) + else + t = t::MethodInstance + end + meth = t.def::Method + end + if !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + result = Any[] # result is unused + return minworld, maxworld, result + end + end + end + # next, compare the current result of ml_matches to the old result lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n minworld = Ref{UInt}(1) maxworld = Ref{UInt}(typemax(UInt)) @@ -340,24 +364,25 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n return minworld[], maxworld[], result end +# fast-path dispatch_status bit definitions (false indicates unknown) +# true indicates this method would be returned as the result from `which` when invoking `method.sig` in the current latest world +const METHOD_SIG_LATEST_WHICH = 0x1 +# true indicates this method would be returned as the only result from `methods` when calling `method.sig` in the current latest world +const METHOD_SIG_LATEST_ONLY = 0x2 + function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt) @assert invokesig isa Type local minworld::UInt, maxworld::UInt matched = nothing - if invokesig === expected.sig - # the invoke match is `expected` for `expected->sig`, unless `expected` is invalid - # TODO: this is broken since PR #53415 + if invokesig === expected.sig && !iszero(expected.dispatch_status & METHOD_SIG_LATEST_WHICH) + # the invoke match is `expected` for `expected->sig`, unless `expected` is replaced minworld = expected.primary_world - maxworld = expected.deleted_world @assert minworld ≤ world - if maxworld < world - maxworld = 0 - end - else - minworld = 1 maxworld = typemax(UInt) + else mt = get_methodtable(expected) if mt === nothing + minworld = 1 maxworld = 0 else matched, valid_worlds = Compiler._findsup(invokesig, mt, world) diff --git a/src/gf.c b/src/gf.c index a42c73e618e83..f101c3f86db10 100644 --- a/src/gf.c +++ b/src/gf.c @@ -296,7 +296,7 @@ void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) m->isva = 1; m->nargs = 2; jl_atomic_store_relaxed(&m->primary_world, 1); - jl_atomic_store_relaxed(&m->deleted_world, ~(size_t)0); + jl_atomic_store_relaxed(&m->dispatch_status, METHOD_SIG_LATEST_ONLY | METHOD_SIG_LATEST_ONLY); m->sig = (jl_value_t*)jl_anytuple_type; m->slot_syms = jl_an_empty_string; m->nospecialize = 0; @@ -307,7 +307,7 @@ void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_PUSH2(&m, &newentry); newentry = jl_typemap_alloc(jl_anytuple_type, NULL, jl_emptysvec, - (jl_value_t*)m, jl_atomic_load_relaxed(&m->primary_world), jl_atomic_load_relaxed(&m->deleted_world)); + (jl_value_t*)m, 1, ~(size_t)0); jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)jl_anytuple_type, jl_emptysvec); @@ -1908,7 +1908,7 @@ static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_ // since m2 was also a previous match over isect, // see if m was previously dominant over all m2 // or if this was already ambiguous before - if (ambig != morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { + if (ambig == morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type return 0; } @@ -2242,17 +2242,22 @@ JL_DLLEXPORT void jl_method_table_disable(jl_methtable_t *mt, jl_method_t *metho JL_LOCK(&world_counter_lock); if (!jl_atomic_load_relaxed(&allow_new_worlds)) jl_error("Method changes have been disabled via a call to disable_new_worlds."); - JL_LOCK(&mt->writelock); - // Narrow the world age on the method to make it uncallable - size_t world = jl_atomic_load_relaxed(&jl_world_counter); - assert(method == methodentry->func.method); - assert(jl_atomic_load_relaxed(&method->deleted_world) == ~(size_t)0); - jl_atomic_store_relaxed(&method->deleted_world, world); - jl_atomic_store_relaxed(&methodentry->max_world, world); - jl_method_table_invalidate(mt, method, world); - jl_atomic_store_release(&jl_world_counter, world + 1); - JL_UNLOCK(&mt->writelock); + int enabled = jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0; + if (enabled) { + JL_LOCK(&mt->writelock); + // Narrow the world age on the method to make it uncallable + size_t world = jl_atomic_load_relaxed(&jl_world_counter); + assert(method == methodentry->func.method); + jl_atomic_store_relaxed(&method->dispatch_status, 0); + assert(jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0); + jl_atomic_store_relaxed(&methodentry->max_world, world); + jl_method_table_invalidate(mt, method, world); + jl_atomic_store_release(&jl_world_counter, world + 1); + JL_UNLOCK(&mt->writelock); + } JL_UNLOCK(&world_counter_lock); + if (!enabled) + jl_errorf("Method of %s already disabled", jl_symbol_name(method->name)); } static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) @@ -2292,9 +2297,9 @@ jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, JL_LOCK(&mt->writelock); // add our new entry assert(jl_atomic_load_relaxed(&method->primary_world) == ~(size_t)0); // min-world - assert(jl_atomic_load_relaxed(&method->deleted_world) == 1); // max-world - newentry = jl_typemap_alloc((jl_tupletype_t*)method->sig, simpletype, jl_emptysvec, (jl_value_t*)method, - jl_atomic_load_relaxed(&method->primary_world), jl_atomic_load_relaxed(&method->deleted_world)); + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_ONLY) == 0); + newentry = jl_typemap_alloc((jl_tupletype_t*)method->sig, simpletype, jl_emptysvec, (jl_value_t*)method, ~(size_t)0, 1); jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); update_max_args(mt, method->sig); JL_UNLOCK(&mt->writelock); @@ -2315,7 +2320,8 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) JL_LOCK(&mt->writelock); size_t world = jl_atomic_load_relaxed(&method->primary_world); assert(world == jl_atomic_load_relaxed(&jl_world_counter) + 1); // min-world - assert(jl_atomic_load_relaxed(&method->deleted_world) == ~(size_t)0); // max-world + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_ONLY) == 0); assert(jl_atomic_load_relaxed(&newentry->min_world) == ~(size_t)0); assert(jl_atomic_load_relaxed(&newentry->max_world) == 1); jl_atomic_store_relaxed(&newentry->min_world, world); @@ -2330,12 +2336,17 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) // then check what entries we replaced oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, jl_cachearg_offset(mt), max_world); int invalidated = 0; + int only = !(jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_PRECOMPILE_MANY); // will compute if this will be currently the only result that would returned from `ml_matches` given `sig` if (replaced) { oldvalue = (jl_value_t*)replaced; + jl_method_t *m = replaced->func.method; invalidated = 1; - method_overwrite(newentry, replaced->func.method); + method_overwrite(newentry, m); // this is an optimized version of below, given we know the type-intersection is exact - jl_method_table_invalidate(mt, replaced->func.method, max_world); + jl_method_table_invalidate(mt, m, max_world); + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + jl_atomic_store_relaxed(&m->dispatch_status, 0); + only = m_dispatch & METHOD_SIG_LATEST_ONLY; } else { jl_method_t *const *d; @@ -2407,8 +2418,10 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) memset(morespec, morespec_unknown, n); for (j = 0; j < n; j++) { jl_method_t *m = d[j]; - if (morespec[j] == (char)morespec_is) + if (morespec[j] == (char)morespec_is) { + only = 0; continue; + } loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot _Atomic(jl_method_instance_t*) *data; size_t l; @@ -2438,7 +2451,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) // not actually shadowing--the existing method is still better break; if (ambig == morespec_unknown) - ambig = jl_type_morespecific(type, m->sig) ? morespec_is : morespec_isnot; + ambig = jl_type_morespecific(type, m->sig) ? morespec_isnot : morespec_is; // replacing a method--see if this really was the selected method previously // over the intersection (not ambiguous) and the new method will be selected now (morespec_is) int replaced_dispatch = is_replacing(ambig, type, m, d, n, isect, isect2, morespec); @@ -2455,6 +2468,20 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) invalidated |= invalidatedmi; } } + // now compute and store updates to METHOD_SIG_LATEST_ONLY + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + if (m_dispatch & METHOD_SIG_LATEST_ONLY) { + if (morespec[j] == (char)morespec_unknown) + morespec[j] = (char)(jl_type_morespecific(m->sig, type) ? morespec_is : morespec_isnot); + if (morespec[j] == (char)morespec_isnot) + jl_atomic_store_relaxed(&m->dispatch_status, ~METHOD_SIG_LATEST_ONLY & m_dispatch); + } + if (only) { + if (morespec[j] == (char)morespec_is || ambig == morespec_is || + (ambig == morespec_unknown && !jl_type_morespecific(type, m->sig))) { + only = 0; + } + } } if (jl_array_nrows(oldmi)) { // search mt->cache and leafcache and drop anything that might overlap with the new method @@ -2485,7 +2512,8 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) loctag = jl_cstr_to_string("jl_method_table_insert"); jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } - jl_atomic_store_relaxed(&newentry->max_world, jl_atomic_load_relaxed(&method->deleted_world)); + jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); + jl_atomic_store_relaxed(&method->dispatch_status, METHOD_SIG_LATEST_WHICH | (only ? METHOD_SIG_LATEST_ONLY : 0)); // TODO: this should be sequenced fully after the world counter store JL_UNLOCK(&mt->writelock); JL_GC_POP(); } @@ -2499,7 +2527,6 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method jl_error("Method changes have been disabled via a call to disable_new_worlds."); size_t world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_atomic_store_relaxed(&method->primary_world, world); - jl_atomic_store_relaxed(&method->deleted_world, ~(size_t)0); jl_method_table_activate(mt, newentry); jl_atomic_store_release(&jl_world_counter, world); JL_UNLOCK(&world_counter_lock); @@ -3882,6 +3909,8 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio closure->match.min_valid = max_world + 1; return 1; } + if (closure->match.max_valid > max_world) + closure->match.max_valid = max_world; jl_method_t *meth = ml->func.method; if (closure->lim >= 0 && jl_is_dispatch_tupletype(meth->sig)) { int replaced = 0; @@ -4554,12 +4583,9 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_method_t *m = matc->method; // method applicability is the same as typemapentry applicability size_t min_world = jl_atomic_load_relaxed(&m->primary_world); - size_t max_world = jl_atomic_load_relaxed(&m->deleted_world); // intersect the env valid range with method lookup's inclusive valid range if (env.match.min_valid < min_world) env.match.min_valid = min_world; - if (env.match.max_valid > max_world) - env.match.max_valid = max_world; } if (mt && cache_result && ((jl_datatype_t*)unw)->isdispatchtuple) { // cache_result parameter keeps this from being recursive if (len == 1 && !has_ambiguity) { diff --git a/src/jltypes.c b/src/jltypes.c index e5341c2621df3..a3d10a87b8091 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3545,8 +3545,8 @@ void jl_init_types(void) JL_GC_DISABLED "module", "file", "line", + "dispatch_status", // atomic "primary_world", // atomic - "deleted_world", // atomic "sig", "specializations", // !const "speckeyset", // !const @@ -3578,7 +3578,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_module_type, jl_symbol_type, jl_int32_type, - jl_ulong_type, + jl_int32_type, jl_ulong_type, jl_type_type, jl_any_type, // union(jl_simplevector_type, jl_method_instance_type), diff --git a/src/julia.h b/src/julia.h index 7f7362b9d4a9a..1ca72518c2f22 100644 --- a/src/julia.h +++ b/src/julia.h @@ -331,8 +331,8 @@ typedef struct _jl_method_t { struct _jl_module_t *module; jl_sym_t *file; int32_t line; + _Atomic(int32_t) dispatch_status; // bits defined in staticdata.jl _Atomic(size_t) primary_world; - _Atomic(size_t) deleted_world; // method's type signature. redundant with TypeMapEntry->specTypes jl_value_t *sig; diff --git a/src/julia_internal.h b/src/julia_internal.h index 573ccb3306bd0..9436e19fb26b4 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -677,6 +677,10 @@ typedef union { #define SOURCE_MODE_NOT_REQUIRED 0x0 #define SOURCE_MODE_ABI 0x1 +#define METHOD_SIG_LATEST_WHICH 0b0001 +#define METHOD_SIG_LATEST_ONLY 0b0010 +#define METHOD_SIG_PRECOMPILE_MANY 0b0100 + JL_DLLEXPORT jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner); JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src); void jl_engine_sweep(jl_ptls_t *gc_all_tls_states) JL_NOTSAFEPOINT; diff --git a/src/method.c b/src/method.c index aa4f438ede080..675293089de44 100644 --- a/src/method.c +++ b/src/method.c @@ -727,7 +727,7 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t JL_TRY { ct->ptls->in_pure_callback = 1; ct->world_age = jl_atomic_load_relaxed(&def->primary_world); - if (ct->world_age > jl_atomic_load_acquire(&jl_world_counter) || jl_atomic_load_relaxed(&def->deleted_world) < ct->world_age) + if (ct->world_age > jl_atomic_load_acquire(&jl_world_counter)) jl_error("The generator method cannot run until it is added to a method table."); // invoke code generator @@ -1006,7 +1006,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->isva = 0; m->nargs = 0; jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); - jl_atomic_store_relaxed(&m->deleted_world, 1); + jl_atomic_store_relaxed(&m->dispatch_status, 0); m->is_for_opaque_closure = 0; m->nospecializeinfer = 0; jl_atomic_store_relaxed(&m->did_scan_source, 0); diff --git a/src/opaque_closure.c b/src/opaque_closure.c index a10b5c617753c..2e39d5965b45a 100644 --- a/src/opaque_closure.c +++ b/src/opaque_closure.c @@ -159,7 +159,6 @@ JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tuplet size_t world = jl_current_task->world_age; // these are only legal in the current world since they are not in any tables jl_atomic_store_release(&meth->primary_world, world); - jl_atomic_store_release(&meth->deleted_world, world); if (isinferred) { jl_value_t *argslotty = jl_array_ptr_ref(ci->slottypes, 0); diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 84619b714b624..295f91ad31e67 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -416,20 +416,21 @@ static void jl_rebuild_methtables(arraylist_t* MIs, htable_t* mtables) ptrhash_put(mtables, old_mt, jl_new_method_table(name, m->module)); jl_methtable_t *mt = (jl_methtable_t*)ptrhash_get(mtables, old_mt); size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t * lookup = jl_methtable_lookup(mt, m->sig, world); + jl_value_t *lookup = jl_methtable_lookup(mt, m->sig, world); // Check if the method is already in the new table, if not then insert it there if (lookup == jl_nothing || (jl_method_t*)lookup != m) { //TODO: should this be a function like unsafe_insert_method? size_t min_world = jl_atomic_load_relaxed(&m->primary_world); - size_t max_world = jl_atomic_load_relaxed(&m->deleted_world); + size_t max_world = ~(size_t)0; + assert(min_world == jl_atomic_load_relaxed(&m->primary_world)); + int dispatch_status = jl_atomic_load_relaxed(&m->dispatch_status); jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); - jl_atomic_store_relaxed(&m->deleted_world, 1); + jl_atomic_store_relaxed(&m->dispatch_status, 0); jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); jl_atomic_store_relaxed(&m->primary_world, min_world); - jl_atomic_store_relaxed(&m->deleted_world, max_world); + jl_atomic_store_relaxed(&m->dispatch_status, dispatch_status); jl_atomic_store_relaxed(&newentry->min_world, min_world); - jl_atomic_store_relaxed(&newentry->max_world, max_world); + jl_atomic_store_relaxed(&newentry->max_world, max_world); // short-circuit jl_method_table_insert } } - } diff --git a/src/staticdata.c b/src/staticdata.c index f42fc38828f65..c75b9cdf94f2b 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1805,16 +1805,11 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_method_t *m = (jl_method_t*)v; jl_method_t *newm = (jl_method_t*)&f->buf[reloc_offset]; if (s->incremental) { - if (jl_atomic_load_relaxed(&newm->deleted_world) == ~(size_t)0) { - if (jl_atomic_load_relaxed(&newm->primary_world) > 1) { - jl_atomic_store_relaxed(&newm->primary_world, ~(size_t)0); // min-world - jl_atomic_store_relaxed(&newm->deleted_world, 1); // max_world - arraylist_push(&s->fixup_objs, (void*)reloc_offset); - } - } - else { - jl_atomic_store_relaxed(&newm->primary_world, 1); - jl_atomic_store_relaxed(&newm->deleted_world, 0); + if (jl_atomic_load_relaxed(&newm->primary_world) > 1) { + jl_atomic_store_relaxed(&newm->primary_world, ~(size_t)0); // min-world + int dispatch_status = jl_atomic_load_relaxed(&newm->dispatch_status); + jl_atomic_store_relaxed(&newm->dispatch_status, dispatch_status & METHOD_SIG_LATEST_ONLY ? 0 : METHOD_SIG_PRECOMPILE_MANY); + arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } else { diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index c3f6a7e98a550..65f7dc59d9397 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -706,9 +706,7 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size else if (jl_is_method(obj)) { jl_method_t *m = (jl_method_t*)obj; assert(jl_atomic_load_relaxed(&m->primary_world) == ~(size_t)0); - assert(jl_atomic_load_relaxed(&m->deleted_world) == WORLD_AGE_REVALIDATION_SENTINEL); jl_atomic_store_release(&m->primary_world, world); - jl_atomic_store_release(&m->deleted_world, ~(size_t)0); } else if (jl_is_code_instance(obj)) { jl_code_instance_t *ci = (jl_code_instance_t*)obj; diff --git a/src/typemap.c b/src/typemap.c index 6ee8f9c599f3a..8c0e585601944 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -569,10 +569,8 @@ int has_covariant_var(jl_datatype_t *ttypes, jl_tvar_t *tv) void typemap_slurp_search(jl_typemap_entry_t *ml, struct typemap_intersection_env *closure) { - // n.b. we could consider mt->max_args here too, so this optimization - // usually works even if the user forgets the `slurp...` argument, but - // there is discussion that parameter may be going away? (and it is - // already not accurately up-to-date for all tables currently anyways) + // TODO: we should consider nparams(closure->type) here too, so this optimization + // usually works even if the user forgets the `slurp...` argument if (closure->search_slurp && ml->va) { jl_value_t *sig = jl_unwrap_unionall((jl_value_t*)ml->sig); size_t nargs = jl_nparams(sig); diff --git a/test/core.jl b/test/core.jl index 15cb139601349..f24eeb23909c8 100644 --- a/test/core.jl +++ b/test/core.jl @@ -35,7 +35,7 @@ end for (T, c) in ( (Core.CodeInfo, []), (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), - (Core.Method, [:primary_world, :deleted_world]), + (Core.Method, [:primary_world, :dispatch_status]), (Core.MethodInstance, [:cache, :flags]), (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), (Core.TypeMapEntry, [:next, :min_world, :max_world]), diff --git a/test/worlds.jl b/test/worlds.jl index 42cc74a010670..542bbaa440362 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -502,6 +502,8 @@ Base.delete_method(fshadow_m2) @test Base.morespecific(fshadow_m3, fshadow_m1) @test !Base.morespecific(fshadow_m2, fshadow_m3) +@test_throws "Method of fshadow already disabled" Base.delete_method(fshadow_m2) + # Generated functions without edges must have min_world = 1. # N.B.: If changing this, move this test to precompile and make sure # that the specialization survives revalidation. From f18f392df65c6d46cce6640c22d7d3856647300c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 7 May 2025 05:01:43 +0900 Subject: [PATCH 208/662] type-stabilize `eval_import_path` (#58333) --- base/module.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/module.jl b/base/module.jl index 7e7812173060c..c21414179446e 100644 --- a/base/module.jl +++ b/base/module.jl @@ -14,6 +14,7 @@ function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, i::Int = 1 function next!() + local v i <= length(path.args) || error("invalid module path") v = path.args[i] i += 1 From 30804b6613aa21a3404646af6e25ce12cf065414 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 6 May 2025 17:01:48 -0400 Subject: [PATCH 209/662] reduce data size in threads test (#58299) Hopefully this will make the test more reliable on 32-bit. --- test/threads_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 0f54a4c6d9ee5..82f7bf97f63b2 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -1593,7 +1593,7 @@ end @sync begin for i in 1:n_tasks start_time_i = time_ns() - task_i = Threads.@spawn peakflops() + task_i = Threads.@spawn peakflops(1024) Threads.@spawn begin wait(task_i) end_time_i = time_ns() From f033630cf08fe20818ee2be7fce2839fce12ec43 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 6 May 2025 19:18:08 -0400 Subject: [PATCH 210/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=205432fdd30=20to=2080d2e7505=20(#58317)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 | 1 - .../sha512 | 1 - .../Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 | 1 + .../sha512 | 1 + stdlib/Artifacts/src/Artifacts.jl | 4 ++-- stdlib/Pkg.version | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 create mode 100644 deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 diff --git a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 deleted file mode 100644 index bcaf046283425..0000000000000 --- a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -e097e71eea7524a17bda8ba14569a983 diff --git a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 b/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 deleted file mode 100644 index 1c659df733de4..0000000000000 --- a/deps/checksums/Pkg-5432fdd30f06707ee227237224b0b72168c2c2f5.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -56f6baeda00597ab8a9d9827e6a07aa41662e7914c6ba6fe8c6c21c2008a6552db9778ad95dbaad8d0309e6caa25995916adafcd0243b0f26e1e5217ae608a01 diff --git a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 new file mode 100644 index 0000000000000..8f0c112e59907 --- /dev/null +++ b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 @@ -0,0 +1 @@ +afd0c5ac79adc7842dd414223782db56 diff --git a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 new file mode 100644 index 0000000000000..c896e2abc199d --- /dev/null +++ b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 @@ -0,0 +1 @@ +8db5ccdfadbc3acfb2b22aecc19f9466d44d00eb60bd54a78b690a4ba8ec6ad1a37917fd8af3f25f3e3c15ba82bda2c46182d28eac223464ac3ad6a38201edd3 diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl index e21db58b9445e..f240f6defa0d0 100644 --- a/stdlib/Artifacts/src/Artifacts.jl +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -562,7 +562,7 @@ function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dic meta = artifact_meta(name, artifact_dict, artifacts_toml; platform) if meta !== nothing && get(meta, "lazy", false) if LazyArtifacts isa Module && isdefined(LazyArtifacts, :ensure_artifact_installed) - if nameof(LazyArtifacts) in (:Pkg, :Artifacts) + if nameof(LazyArtifacts) in (:Pkg, :Artifacts, :PkgArtifacts) Base.depwarn("using Pkg instead of using LazyArtifacts is deprecated", :var"@artifact_str", force=true) end return jointail(LazyArtifacts.ensure_artifact_installed(string(name), meta, artifacts_toml; platform), path_tail) @@ -697,7 +697,7 @@ macro artifact_str(name, platform=nothing) # Check if the user has provided `LazyArtifacts`, and thus supports lazy artifacts # If not, check to see if `Pkg` or `Pkg.Artifacts` has been imported. LazyArtifacts = nothing - for module_name in (:LazyArtifacts, :Pkg, :Artifacts) + for module_name in (:LazyArtifacts, :Pkg, :Artifacts, :PkgArtifacts) if isdefined(__module__, module_name) LazyArtifacts = GlobalRef(__module__, module_name) break diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 8d13ca7f3e8dc..216f7ec8d13c0 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 5432fdd30f06707ee227237224b0b72168c2c2f5 +PKG_SHA1 = 80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 96e17a9cfc94d3c61ff3d938e6c42c6fa8854e8f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 7 May 2025 01:57:41 +0200 Subject: [PATCH 211/662] Drop `<<` and `>>` from sametype heuristic (#58331) These heuristics were added in #32105. However, I think `>>` and `<<` should be dropped because 1. The original analysis predates irinterp, which is significantly faster at constprop 2. These functions actuall have non-trivial bodies that do benefit (in ir size at least) from constprop (#58329). Fixes #58329 --- base/essentials.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index 9f70a90bdac7d..607ce44b39361 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1300,5 +1300,3 @@ typename(typeof(function <= end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function >= end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function < end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function > end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC -typename(typeof(function << end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC -typename(typeof(function >> end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC From 20f3afb978fa14f66cafd8c195039c3475e3ef14 Mon Sep 17 00:00:00 2001 From: Tianyi Pu <44583944+putianyi889@users.noreply.github.com> Date: Thu, 8 May 2025 00:44:26 +0800 Subject: [PATCH 212/662] Add `ispositive`, etc. (#53677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - https://github.com/JuliaLang/julia/issues/35067 It's faster for some complicated Real types. ```julia julia> f(x) = x<=zero(typeof(x)) f (generic function with 1 method) julia> g(x) = signbit(x)||iszero(x) g (generic function with 1 method) julia> @btime f(x) setup=(x=rand(BigFloat)) 99.055 ns (4 allocations: 144 bytes) true julia> @btime g(x) setup=(x=rand(BigFloat)) 10.700 ns (0 allocations: 0 bytes) false ``` If you think this is a good idea then I'll add docstrings and tests. Edit: FAQs according to discussions below - Why `ispositive(x::Bool) = x`? - Because it's fast. https://github.com/JuliaLang/julia/pull/53677#discussion_r1530731988 - Why support `ispositive(::Any)`? - #35067 --------- Co-authored-by: mikmoore <95002244+mikmoore@users.noreply.github.com> Co-authored-by: Max Horn Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- NEWS.md | 1 + base/bool.jl | 1 + base/exports.jl | 2 ++ base/gmp.jl | 40 ++++++++++++++++++++-------------------- base/int.jl | 2 ++ base/missing.jl | 2 ++ base/mpfr.jl | 7 ++++++- base/number.jl | 44 ++++++++++++++++++++++++++++++++++++++++++++ doc/src/base/base.md | 2 ++ test/numbers.jl | 22 ++++++++++++++++++++++ 10 files changed, 102 insertions(+), 21 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1bd8f16f13b76..686c3f94d702c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,6 +26,7 @@ Build system changes New library functions --------------------- +* `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). * Exporting function `fieldindex` to get the index of a struct's field ([#58119]). New library features diff --git a/base/bool.jl b/base/bool.jl index 3658318d158e5..12144756c76c8 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -156,6 +156,7 @@ abs(x::Bool) = x abs2(x::Bool) = x iszero(x::Bool) = !x isone(x::Bool) = x +ispositive(x::Bool) = x # could use fallback once #21712 is resolved <(x::Bool, y::Bool) = y&!x <=(x::Bool, y::Bool) = y|!x diff --git a/base/exports.jl b/base/exports.jl index bf65ad8d1f07f..36b9bba3d8358 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -338,7 +338,9 @@ export isinf, isinteger, isnan, + isnegative, isodd, + ispositive, ispow2, isqrt, isreal, diff --git a/base/gmp.jl b/base/gmp.jl index dcd50c6022d84..1d7c4266c331c 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -11,7 +11,7 @@ import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor, bin, oct, dec, hex, isequal, invmod, _prevpow2, _nextpow2, ndigits0zpb, widen, signed, unsafe_trunc, trunc, iszero, isone, big, flipsign, signbit, sign, isodd, iseven, digits!, hash, hash_integer, top_set_bit, - clamp, unsafe_takestring + ispositive, isnegative, clamp, unsafe_takestring import Core: Signed, Float16, Float32, Float64 @@ -383,7 +383,7 @@ function (::Type{T})(x::BigInt) where T<:Base.BitSigned else 0 <= n <= cld(sizeof(T),sizeof(Limb)) || throw(InexactError(nameof(T), T, x)) y = x % T - ispos(x) ⊻ (y > 0) && throw(InexactError(nameof(T), T, x)) # catch overflow + ispositive(x) ⊻ (y > 0) && throw(InexactError(nameof(T), T, x)) # catch overflow y end end @@ -606,7 +606,7 @@ Number of ones in the binary representation of abs(x). count_ones_abs(x::BigInt) = iszero(x) ? 0 : MPZ.mpn_popcount(x) function top_set_bit(x::BigInt) - isneg(x) && throw(DomainError(x, "top_set_bit only supports negative arguments when they have type BitSigned.")) + isnegative(x) && throw(DomainError(x, "top_set_bit only supports negative arguments when they have type BitSigned.")) iszero(x) && return 0 x.size * sizeof(Limb) << 3 - leading_zeros(GC.@preserve x unsafe_load(x.d, x.size)) end @@ -700,7 +700,7 @@ function prod(arr::AbstractArray{BigInt}) foldl(MPZ.mul!, arr; init) end -factorial(n::BigInt) = !isneg(n) ? MPZ.fac_ui(n) : throw(DomainError(n, "`n` must not be negative.")) +factorial(n::BigInt) = !isnegative(n) ? MPZ.fac_ui(n) : throw(DomainError(n, "`n` must not be negative.")) function binomial(n::BigInt, k::Integer) k < 0 && return BigInt(0) @@ -732,17 +732,17 @@ isone(x::BigInt) = x == Culong(1) <(i::Integer, x::BigInt) = cmp(x,i) > 0 <(x::BigInt, f::CdoubleMax) = isnan(f) ? false : cmp(x,f) < 0 <(f::CdoubleMax, x::BigInt) = isnan(f) ? false : cmp(x,f) > 0 -isneg(x::BigInt) = x.size < 0 -ispos(x::BigInt) = x.size > 0 +isnegative(x::BigInt) = x.size < 0 +ispositive(x::BigInt) = x.size > 0 -signbit(x::BigInt) = isneg(x) +signbit(x::BigInt) = isnegative(x) flipsign!(x::BigInt, y::Integer) = (signbit(y) && (x.size = -x.size); x) flipsign( x::BigInt, y::Integer) = signbit(y) ? -x : x flipsign( x::BigInt, y::BigInt) = signbit(y) ? -x : x # above method to resolving ambiguities with flipsign(::T, ::T) where T<:Signed function sign(x::BigInt) - isneg(x) && return -one(x) - ispos(x) && return one(x) + isnegative(x) && return -one(x) + ispositive(x) && return one(x) return x end @@ -754,12 +754,12 @@ function string(n::BigInt; base::Integer = 10, pad::Integer = 1) iszero(n) && pad < 1 && return "" nd1 = ndigits(n, base=base) nd = max(nd1, pad) - sv = Base.StringMemory(nd + isneg(n)) + sv = Base.StringMemory(nd + isnegative(n)) GC.@preserve sv MPZ.get_str!(pointer(sv) + nd - nd1, base, n) - @inbounds for i = (1:nd-nd1) .+ isneg(n) + @inbounds for i = (1:nd-nd1) .+ isnegative(n) sv[i] = '0' % UInt8 end - isneg(n) && (sv[1] = '-' % UInt8) + isnegative(n) && (sv[1] = '-' % UInt8) unsafe_takestring(sv) end @@ -769,7 +769,7 @@ function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<: # fast path using mpz_get_str via string(n; base) s = codeunits(string(n; base)) i, j = firstindex(a)-1, length(s)+1 - lasti = min(lastindex(a), firstindex(a) + length(s)-1 - isneg(n)) + lasti = min(lastindex(a), firstindex(a) + length(s)-1 - isnegative(n)) while i < lasti # base ≤ 36: 0-9, plus a-z for 10-35 # base > 36: 0-9, plus A-Z for 10-35 and a-z for 36..61 @@ -778,14 +778,14 @@ function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<: end lasti = lastindex(a) while i < lasti; a[i+=1] = zero(T); end - return isneg(n) ? map!(-,a,a) : a + return isnegative(n) ? map!(-,a,a) : a elseif a isa StridedVector{<:Base.BitInteger} && stride(a,1) == 1 && ispow2(base) && base-1 ≤ typemax(T) # fast path using mpz_export origlen = length(a) _, writelen = MPZ.export!(a, n; nails = 8sizeof(T) - trailing_zeros(base)) length(a) != origlen && resize!(a, origlen) # truncate to least-significant digits a[begin+writelen:end] .= zero(T) - return isneg(n) ? map!(-,a,a) : a + return isnegative(n) ? map!(-,a,a) : a end end return invoke(digits!, Tuple{typeof(a), Integer}, a, n; base) # slow generic fallback @@ -917,7 +917,7 @@ module MPQ # Rational{BigInt} import .Base: unsafe_rational, __throw_rational_argerror_zero -import ..GMP: BigInt, MPZ, Limb, isneg, libgmp +import ..GMP: BigInt, MPZ, Limb, libgmp gmpq(op::Symbol) = (Symbol(:__gmpq_, op), libgmp) @@ -995,7 +995,7 @@ end # define add, sub, mul, div, and their inplace versions function add!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.den) || iszero(y.den) - if iszero(x.den) && iszero(y.den) && isneg(x.num) != isneg(y.num) + if iszero(x.den) && iszero(y.den) && isnegative(x.num) != isnegative(y.num) throw(DivideError()) end return set!(z, iszero(x.den) ? x : y) @@ -1008,7 +1008,7 @@ end function sub!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.den) || iszero(y.den) - if iszero(x.den) && iszero(y.den) && isneg(x.num) == isneg(y.num) + if iszero(x.den) && iszero(y.den) && isnegative(x.num) == isnegative(y.num) throw(DivideError()) end iszero(x.den) && return set!(z, x) @@ -1025,7 +1025,7 @@ function mul!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.num) || iszero(y.num) throw(DivideError()) end - return set_si!(z, ifelse(xor(isneg(x.num), isneg(y.num)), -1, 1), 0) + return set_si!(z, ifelse(xor(isnegative(x.num), isnegative(y.num)), -1, 1), 0) end zq = _MPQ(z) ccall((:__gmpq_mul, libgmp), Cvoid, @@ -1038,7 +1038,7 @@ function div!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(y.den) throw(DivideError()) end - isneg(y.num) || return set!(z, x) + isnegative(y.num) || return set!(z, x) return set_si!(z, flipsign(-1, x.num), 0) elseif iszero(y.den) return set_si!(z, 0, 1) diff --git a/base/int.jl b/base/int.jl index c463869ec2dae..dfb4cc5d99b06 100644 --- a/base/int.jl +++ b/base/int.jl @@ -139,6 +139,8 @@ iseven(n::Real) = isinteger(n) && iszero(rem(Integer(n), 2)) signbit(x::Integer) = x < 0 signbit(x::Unsigned) = false +isnegative(x::Unsigned) = false + flipsign(x::T, y::T) where {T<:BitSigned} = flipsign_int(x, y) flipsign(x::BitSigned, y::BitSigned) = flipsign_int(promote(x, y)...) % typeof(x) diff --git a/base/missing.jl b/base/missing.jl index 6a8c09dc02aff..5fc0e6ec9e620 100644 --- a/base/missing.jl +++ b/base/missing.jl @@ -86,6 +86,8 @@ isequal(::Any, ::Missing) = false isless(::Missing, ::Missing) = false isless(::Missing, ::Any) = false isless(::Any, ::Missing) = true +ispositive(::Missing) = missing +isnegative(::Missing) = missing isapprox(::Missing, ::Missing; kwargs...) = missing isapprox(::Missing, ::Any; kwargs...) = missing isapprox(::Any, ::Missing; kwargs...) = missing diff --git a/base/mpfr.jl b/base/mpfr.jl index 7d0bd2c1b9986..25b7f0abe4b98 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -18,7 +18,8 @@ import setrounding, maxintfloat, widen, significand, frexp, tryparse, iszero, isone, big, _string_n, decompose, minmax, _precision_with_base_2, sinpi, cospi, sincospi, tanpi, sind, cosd, tand, asind, acosd, atand, - uinttype, exponent_max, exponent_min, ieee754_representation, significand_mask + uinttype, exponent_max, exponent_min, ieee754_representation, significand_mask, + ispositive, isnegative import .Core: AbstractFloat import .Base: Rational, Float16, Float32, Float64, Bool @@ -1135,6 +1136,10 @@ isfinite(x::BigFloat) = !isinf(x) && !isnan(x) iszero(x::BigFloat) = x.exp == mpfr_special_exponent_zero isone(x::BigFloat) = x == Clong(1) +# In theory, `!iszero(x) && !isnan(x)` should be the same as `x.exp > mpfr_special_exponent_nan`, but this is safer. +ispositive(x::BigFloat) = !signbit(x) && !iszero(x) && !isnan(x) +isnegative(x::BigFloat) = signbit(x) && !iszero(x) && !isnan(x) + @eval typemax(::Type{BigFloat}) = $(BigFloat(Inf)) @eval typemin(::Type{BigFloat}) = $(BigFloat(-Inf)) diff --git a/base/number.jl b/base/number.jl index 7c9ea1db19cac..8314c546147c7 100644 --- a/base/number.jl +++ b/base/number.jl @@ -136,6 +136,50 @@ true """ signbit(x::Real) = x < 0 +""" + ispositive(x) + +Test whether `x > 0`. See also [`isnegative`](@ref). + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples +```jldoctest +julia> ispositive(-4.0) +false + +julia> ispositive(99) +true + +julia> ispositive(0.0) +false +``` +""" +ispositive(x::Real) = x > 0 + +""" + isnegative(x) + +Test whether `x < 0`. See also [`ispositive`](@ref). + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples +```jldoctest +julia> isnegative(-4.0) +true + +julia> isnegative(99) +false + +julia> isnegative(-0.0) +false +``` +""" +isnegative(x::Real) = x < 0 + """ sign(x) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 695f6db46e848..2c34bcb8be9ab 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -139,6 +139,8 @@ Core.:(===) Core.isa Base.isequal Base.isless +Base.ispositive +Base.isnegative Base.isunordered Base.ifelse Core.typeassert diff --git a/test/numbers.jl b/test/numbers.jl index 702cad3f4e0f1..42eeeec4257c7 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -831,6 +831,28 @@ end @test cmp(isless, 1, NaN) == -1 @test cmp(isless, NaN, NaN) == 0 end +@testset "ispositive/isnegative" begin + for T in [Base.uniontypes(Base.BitInteger)..., Bool, Rational{Int}, BigInt, Base.uniontypes(Base.IEEEFloat)..., BigFloat, Missing] + values = T[zero(T), one(T)] + if T <: AbstractFloat + push!(values, Inf, NaN) # also check Infs and NaNs + elseif T <: Rational + push!(values, 1//0) # also check Infs + end + @testset "$T" begin + for value in values + # https://github.com/JuliaLang/julia/pull/53677#discussion_r1534044582 + # Use eval to explicitly show expressions when they fail + @eval begin + @test ispositive($value) === ($value > 0) + @test ispositive(-$value) === (-$value > 0) + @test isnegative($value) === ($value < 0) + @test isnegative(-$value) === (-$value < 0) + end + end + end + end +end @testset "Float vs Integer comparison" begin for x=-5:5, y=-5:5 @test (x==y)==(Float64(x)==Int64(y)) From d4d5b2e6aaadb4f885359fb229a63db457d0fe46 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Wed, 7 May 2025 14:59:01 -0400 Subject: [PATCH 213/662] Expose scratch kwarg via `partialsort` (#58269) `sort` defaults to a sorting method that uses scratch space and a kwarg carries it around. However, this kwarg isn't currently exposed to `partialsort`. --------- Co-authored-by: Lilith Orion Hafner --- base/sort.jl | 8 ++++---- test/sorting.jl | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/base/sort.jl b/base/sort.jl index f5709a368256c..6cc1420708a35 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -96,7 +96,7 @@ function issorted(itr; end end -function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Ordering) +function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Ordering; scratch::Union{Nothing, Vector} = nothing) # TODO move k from `alg` to `kw` # Don't perform InitialOptimizations before Bracketing. The optimizations take O(n) # time and so does the whole sort. But do perform them before recursive calls because @@ -105,7 +105,7 @@ function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Orde _sort!(v, BoolOptimization( Small{12}( # Very small inputs should go straight to insertion sort BracketedSort(k))), - o, (;)) + o, (; scratch)) maybeview(v, k) end @@ -166,8 +166,8 @@ julia> a ``` """ partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}; - lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) = - partialsort!(v, k, ord(lt,by,rev,order)) + lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward, kws...) = + partialsort!(v, k, ord(lt,by,rev,order); kws...) """ partialsort(v, k, by=identity, lt=isless, rev=false) diff --git a/test/sorting.jl b/test/sorting.jl index ab4250fef411b..da5392a8ff884 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -808,6 +808,16 @@ end end end +@testset "partialsort(x; scratch)" begin + for n in [1,10,100,1000] + v = rand(n) + scratch = [0.0] + k = n ÷ 2 + 1 + @test partialsort(v, k) == partialsort(v, k; scratch) + @test partialsort!(copy(v), k) == partialsort!(copy(v), k; scratch) + end +end + @testset "sorting preserves identity" begin a = BigInt.([2, 2, 2, 1, 1, 1]) # issue #39620 sort!(a) From 99ba3c798534c365b2eb815ed0624fd7eacbb21e Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 7 May 2025 15:01:56 -0400 Subject: [PATCH 214/662] Remove bugged and typically slower `minimum`/`maximum` method (#58267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This method intends to be a SIMD-able optimization for reductions with `min` and `max`, but it fails to achieve those goals on nearly every architecture. For example, on my Mac M1 the generic implementation is more than **6x** faster than this method. Fixes #45932, fixes #36412, fixes #36081. --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- base/reduce.jl | 57 -------------------------------------------------- test/reduce.jl | 20 +++++++++++++++++- 2 files changed, 19 insertions(+), 58 deletions(-) diff --git a/base/reduce.jl b/base/reduce.jl index f85cb285ef9ce..9041fd088ccf3 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -617,63 +617,6 @@ julia> prod(1:5; init = 1.0) prod(a; kw...) = mapreduce(identity, mul_prod, a; kw...) ## maximum, minimum, & extrema -_fast(::typeof(min),x,y) = min(x,y) -_fast(::typeof(max),x,y) = max(x,y) -function _fast(::typeof(max), x::AbstractFloat, y::AbstractFloat) - ifelse(isnan(x), - x, - ifelse(x > y, x, y)) -end - -function _fast(::typeof(min),x::AbstractFloat, y::AbstractFloat) - ifelse(isnan(x), - x, - ifelse(x < y, x, y)) -end - -isbadzero(::typeof(max), x::AbstractFloat) = (x == zero(x)) & signbit(x) -isbadzero(::typeof(min), x::AbstractFloat) = (x == zero(x)) & !signbit(x) -isbadzero(op, x) = false -isgoodzero(::typeof(max), x) = isbadzero(min, x) -isgoodzero(::typeof(min), x) = isbadzero(max, x) - -function mapreduce_impl(f, op::Union{typeof(max), typeof(min)}, - A::AbstractArrayOrBroadcasted, first::Int, last::Int) - a1 = @inbounds A[first] - v1 = mapreduce_first(f, op, a1) - v2 = v3 = v4 = v1 - chunk_len = 256 - start = first + 1 - simdstop = start + chunk_len - 4 - while simdstop <= last - 3 - for i in start:4:simdstop - v1 = _fast(op, v1, f(@inbounds(A[i+0]))) - v2 = _fast(op, v2, f(@inbounds(A[i+1]))) - v3 = _fast(op, v3, f(@inbounds(A[i+2]))) - v4 = _fast(op, v4, f(@inbounds(A[i+3]))) - end - checkbounds(A, simdstop+3) - start += chunk_len - simdstop += chunk_len - end - v = op(op(v1,v2),op(v3,v4)) - for i in start:last - @inbounds ai = A[i] - v = op(v, f(ai)) - end - - # enforce correct order of 0.0 and -0.0 - # e.g. maximum([0.0, -0.0]) === 0.0 - # should hold - if isbadzero(op, v) - for i in first:last - x = @inbounds A[i] - isgoodzero(op,x) && return x - end - end - return v -end - """ maximum(f, itr; [init]) diff --git a/test/reduce.jl b/test/reduce.jl index bb54462bb78f0..c6274711cdef4 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -299,19 +299,37 @@ end arr = zeros(N) @test minimum(arr) === 0.0 @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === -0.0 arr[i] = -0.0 @test minimum(arr) === -0.0 @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === 0.0 arr = -zeros(N) @test minimum(arr) === -0.0 @test maximum(arr) === -0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === 0.0 + @test maximum(-, arr) === 0.0 arr[i] = 0.0 @test minimum(arr) === -0.0 - @test maximum(arr) === 0.0 + @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === 0.0 end end + + @test minimum(abs, fill(-0.0, 16)) === mapreduce(abs, (x,y)->min(x,y), fill(-0.0, 16)) === 0.0 end @testset "maximum works on generic order #30320" begin From 66e155d77f5f1a3cd8b6c5f3ba6faba46be678ba Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 7 May 2025 15:17:22 -0400 Subject: [PATCH 215/662] deprecate --sysimage-native-code=no option (#58285) This option no longer serves any purpose, since we can invalidate any native code that is not usable. More importantly, removing the test for this saves >20min of CI time, since to start with that option we have to recompile everything. Thanks to @vtjnash for pointing this out. Closes #57198 --- NEWS.md | 2 ++ src/jloptions.c | 13 ++++++++++--- test/cmdlineargs.jl | 21 ++++----------------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/NEWS.md b/NEWS.md index 686c3f94d702c..211c561167122 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,6 +17,8 @@ Compiler/Runtime improvements Command-line option changes --------------------------- +* The option `--sysimage-native-code=no` has been deprecated. + Multi-threading changes ----------------------- diff --git a/src/jloptions.c b/src/jloptions.c index cd7bc4fdd4de2..21d034a666b1d 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -589,12 +589,19 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_options.use_experimental_features = JL_OPTIONS_USE_EXPERIMENTAL_FEATURES_YES; break; case opt_sysimage_native_code: - if (!strcmp(optarg,"yes")) + if (!strcmp(optarg,"yes")) { jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES; - else if (!strcmp(optarg,"no")) + } + else if (!strcmp(optarg,"no")) { jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO; - else + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_errorf("julia: --sysimage-native-code=no is deprecated"); + else if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ON) + jl_printf(JL_STDERR, "WARNING: --sysimage-native-code=no is deprecated\n"); + } + else { jl_errorf("julia: invalid argument to --sysimage-native-code={yes|no} (%s)", optarg); + } break; case opt_compiled_modules: if (!strcmp(optarg,"yes")) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 7934d9c60d54d..5fd8513849435 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -130,9 +130,6 @@ end ("--startup-file=no", false), ("--startup-file=yes", true), - # ("--sysimage-native-code=no", false), # takes a lot longer (30s) - ("--sysimage-native-code=yes", true), - ("--pkgimages=yes", true), ("--pkgimages=no", false), ) @@ -1086,22 +1083,12 @@ run(pipeline(devnull, `$(joinpath(Sys.BINDIR, Base.julia_exename())) --lisp`, de @test readchomperrors(`$(joinpath(Sys.BINDIR, Base.julia_exename())) -Cnative --lisp`) == (false, "", "ERROR: --lisp must be specified as the first argument") -# --sysimage-native-code={yes|no} -let exename = `$(Base.julia_cmd()) --startup-file=no` - @test readchomp(`$exename --sysimage-native-code=yes -E - "Bool(Base.JLOptions().use_sysimage_native_code)"`) == "true" - # TODO: Make this safe in the presence of two single-thread threadpools - # see https://github.com/JuliaLang/julia/issues/57198 - @test readchomp(`$exename --sysimage-native-code=no -t1,0 -E - "Bool(Base.JLOptions().use_sysimage_native_code)"`) == "false" -end - # backtrace contains line number info (esp. on windows #17179) -for precomp in ("yes", "no") - # TODO: Make this safe in the presence of two single-thread threadpools +let + # TODO: Make this safe in the presence of two single-thread threadpools with + # --sysimage-native-code=no, though that option is deprecated. # see https://github.com/JuliaLang/julia/issues/57198 - threads = precomp == "no" ? `-t1,0` : `` - succ, out, bt = readchomperrors(`$(Base.julia_cmd()) $threads --startup-file=no --sysimage-native-code=$precomp -E 'sqrt(-2)'`) + succ, out, bt = readchomperrors(`$(Base.julia_cmd()) --startup-file=no -E 'sqrt(-2)'`) @test !succ @test out == "" @test occursin(r"\.jl:(\d+)", bt) From 057025e0ce021ffb42941ed3dc58e526e26d50c9 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Wed, 7 May 2025 22:24:06 +0200 Subject: [PATCH 216/662] Random: move MersenneTwister in its own file (#58332) Its code was entangled with other random bits from RNGs.jl and made reading that file annoying. Also, re-arrange RNGs.jl a bit. --- stdlib/Random/src/MersenneTwister.jl | 667 ++++++++++++++++++++ stdlib/Random/src/RNGs.jl | 912 +++++---------------------- stdlib/Random/src/Random.jl | 98 +-- 3 files changed, 843 insertions(+), 834 deletions(-) create mode 100644 stdlib/Random/src/MersenneTwister.jl diff --git a/stdlib/Random/src/MersenneTwister.jl b/stdlib/Random/src/MersenneTwister.jl new file mode 100644 index 0000000000000..7caa75ddcd0a7 --- /dev/null +++ b/stdlib/Random/src/MersenneTwister.jl @@ -0,0 +1,667 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## MersenneTwister + +const MT_CACHE_F = 501 << 1 # number of Float64 in the cache +const MT_CACHE_I = 501 << 4 # number of bytes in the UInt128 cache + +@assert dsfmt_get_min_array_size() <= MT_CACHE_F + +mutable struct MersenneTwister <: AbstractRNG + seed::Any + state::DSFMT_state + vals::Vector{Float64} + ints::Vector{UInt128} + idxF::Int + idxI::Int + + # counters for show + adv::Int64 # state of advance at the DSFMT_state level + adv_jump::BigInt # number of skipped Float64 values via randjump + adv_vals::Int64 # state of advance when vals is filled-up + adv_ints::Int64 # state of advance when ints is filled-up + + function MersenneTwister(seed, state, vals, ints, idxF, idxI, + adv, adv_jump, adv_vals, adv_ints) + length(vals) == MT_CACHE_F && 0 <= idxF <= MT_CACHE_F || + throw(DomainError((length(vals), idxF), + "`length(vals)` and `idxF` must be consistent with $MT_CACHE_F")) + length(ints) == MT_CACHE_I >> 4 && 0 <= idxI <= MT_CACHE_I || + throw(DomainError((length(ints), idxI), + "`length(ints)` and `idxI` must be consistent with $MT_CACHE_I")) + new(seed, state, vals, ints, idxF, idxI, + adv, adv_jump, adv_vals, adv_ints) + end +end + +MersenneTwister(seed, state::DSFMT_state) = + MersenneTwister(seed, state, + Vector{Float64}(undef, MT_CACHE_F), + Vector{UInt128}(undef, MT_CACHE_I >> 4), + MT_CACHE_F, 0, 0, 0, -1, -1) + +""" + MersenneTwister(seed) + MersenneTwister() + +Create a `MersenneTwister` RNG object. Different RNG objects can have +their own seeds, which may be useful for generating different streams +of random numbers. +The `seed` may be an integer, a string, or a vector of `UInt32` integers. +If no seed is provided, a randomly generated one is created (using entropy from the system). +See the [`seed!`](@ref) function for reseeding an already existing `MersenneTwister` object. + +!!! compat "Julia 1.11" + Passing a negative integer seed requires at least Julia 1.11. + +# Examples +```jldoctest +julia> rng = MersenneTwister(123); + +julia> x1 = rand(rng, 2) +2-element Vector{Float64}: + 0.37453777969575874 + 0.8735343642013971 + +julia> x2 = rand(MersenneTwister(123), 2) +2-element Vector{Float64}: + 0.37453777969575874 + 0.8735343642013971 + +julia> x1 == x2 +true +``` +""" +MersenneTwister(seed=nothing) = + seed!(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) + + +function copy!(dst::MersenneTwister, src::MersenneTwister) + dst.seed = src.seed + copy!(dst.state, src.state) + copyto!(dst.vals, src.vals) + copyto!(dst.ints, src.ints) + dst.idxF = src.idxF + dst.idxI = src.idxI + dst.adv = src.adv + dst.adv_jump = src.adv_jump + dst.adv_vals = src.adv_vals + dst.adv_ints = src.adv_ints + dst +end + +copy(src::MersenneTwister) = + MersenneTwister(src.seed, copy(src.state), copy(src.vals), copy(src.ints), + src.idxF, src.idxI, src.adv, src.adv_jump, src.adv_vals, src.adv_ints) + + +==(r1::MersenneTwister, r2::MersenneTwister) = + r1.seed == r2.seed && r1.state == r2.state && + isequal(r1.vals, r2.vals) && + isequal(r1.ints, r2.ints) && + r1.idxF == r2.idxF && r1.idxI == r2.idxI + +hash(r::MersenneTwister, h::UInt) = + foldr(hash, (r.seed, r.state, r.vals, r.ints, r.idxF, r.idxI); init=h) + +function show(io::IO, rng::MersenneTwister) + # seed + if rng.adv_jump == 0 && rng.adv == 0 + return print(io, MersenneTwister, "(", repr(rng.seed), ")") + end + print(io, MersenneTwister, "(", repr(rng.seed), ", (") + # state + sep = ", " + show(io, rng.adv_jump) + print(io, sep) + show(io, rng.adv) + if rng.adv_vals != -1 || rng.adv_ints != -1 + # "(0, 0)" is nicer on the eyes than (-1, 1002) + s = rng.adv_vals != -1 + print(io, sep) + show(io, s ? rng.adv_vals : zero(rng.adv_vals)) + print(io, sep) + show(io, s ? rng.idxF : zero(rng.idxF)) + end + if rng.adv_ints != -1 + idxI = (length(rng.ints)*16 - rng.idxI) / 8 # 8 represents one Int64 + idxI = Int(idxI) # idxI should always be an integer when using public APIs + print(io, sep) + show(io, rng.adv_ints) + print(io, sep) + show(io, idxI) + end + print(io, "))") +end + +### low level API + +function reset_caches!(r::MersenneTwister) + # zeroing the caches makes comparing two MersenneTwister RNGs easier + fill!(r.vals, 0.0) + fill!(r.ints, zero(UInt128)) + mt_setempty!(r) + mt_setempty!(r, UInt128) + r.adv_vals = -1 + r.adv_ints = -1 + r +end + +#### floats + +mt_avail(r::MersenneTwister) = MT_CACHE_F - r.idxF +mt_empty(r::MersenneTwister) = r.idxF == MT_CACHE_F +mt_setfull!(r::MersenneTwister) = r.idxF = 0 +mt_setempty!(r::MersenneTwister) = r.idxF = MT_CACHE_F +mt_pop!(r::MersenneTwister) = @inbounds return r.vals[r.idxF+=1] + +@noinline function gen_rand(r::MersenneTwister) + r.adv_vals = r.adv + GC.@preserve r fill_array!(r, pointer(r.vals), length(r.vals), CloseOpen12()) + mt_setfull!(r) +end + +reserve_1(r::MersenneTwister) = (mt_empty(r) && gen_rand(r); nothing) +# `reserve` allows one to call `rand_inbounds` n times +# precondition: n <= MT_CACHE_F +reserve(r::MersenneTwister, n::Int) = (mt_avail(r) < n && gen_rand(r); nothing) + +#### ints + +logsizeof(::Type{<:Union{Bool,Int8,UInt8}}) = 0 +logsizeof(::Type{<:Union{Int16,UInt16}}) = 1 +logsizeof(::Type{<:Union{Int32,UInt32}}) = 2 +logsizeof(::Type{<:Union{Int64,UInt64}}) = 3 +logsizeof(::Type{<:Union{Int128,UInt128}}) = 4 + +idxmask(::Type{<:Union{Bool,Int8,UInt8}}) = 15 +idxmask(::Type{<:Union{Int16,UInt16}}) = 7 +idxmask(::Type{<:Union{Int32,UInt32}}) = 3 +idxmask(::Type{<:Union{Int64,UInt64}}) = 1 +idxmask(::Type{<:Union{Int128,UInt128}}) = 0 + + +mt_avail(r::MersenneTwister, ::Type{T}) where {T<:BitInteger} = + r.idxI >> logsizeof(T) + +function mt_setfull!(r::MersenneTwister, ::Type{<:BitInteger}) + r.adv_ints = r.adv + ints = r.ints + + @assert length(ints) == 501 + # dSFMT natively randomizes 52 out of 64 bits of each UInt64 words, + # i.e. 12 bits are missing; + # by generating 5 words == 5*52 == 260 bits, we can fully + # randomize 4 UInt64 = 256 bits; IOW, at the array level, we must + # randomize ceil(501*1.25) = 627 UInt128 words (with 2*52 bits each), + # which we then condense into fully randomized 501 UInt128 words + + len = 501 + 126 # 126 == ceil(501 / 4) + resize!(ints, len) + p = pointer(ints) # must be *after* resize! + GC.@preserve r fill_array!(r, Ptr{Float64}(p), len*2, CloseOpen12_64()) + + k = 501 + n = 0 + @inbounds while n != 500 + u = ints[k+=1] + ints[n+=1] ⊻= u << 48 + ints[n+=1] ⊻= u << 36 + ints[n+=1] ⊻= u << 24 + ints[n+=1] ⊻= u << 12 + end + @assert k == len - 1 + @inbounds ints[501] ⊻= ints[len] << 48 + resize!(ints, 501) + r.idxI = MT_CACHE_I +end + +mt_setempty!(r::MersenneTwister, ::Type{<:BitInteger}) = r.idxI = 0 + +function reserve1(r::MersenneTwister, ::Type{T}) where T<:BitInteger + r.idxI < sizeof(T) && mt_setfull!(r, T) + nothing +end + +function mt_pop!(r::MersenneTwister, ::Type{T}) where T<:BitInteger + reserve1(r, T) + r.idxI -= sizeof(T) + i = r.idxI + @inbounds x128 = r.ints[1 + i >> 4] + i128 = (i >> logsizeof(T)) & idxmask(T) # 0-based "indice" in x128 + (x128 >> (i128 * (sizeof(T) << 3))) % T +end + +function mt_pop!(r::MersenneTwister, ::Type{T}) where {T<:Union{Int128,UInt128}} + reserve1(r, T) + idx = r.idxI >> 4 + r.idxI = idx << 4 - 16 + @inbounds r.ints[idx] % T +end + + +#### seed!() + +function initstate!(r::MersenneTwister, data::StridedVector, seed) + # we deepcopy `seed` because the caller might mutate it, and it's useful + # to keep it constant inside `MersenneTwister`; but multiple instances + # can share the same seed without any problem (e.g. in `copy`) + r.seed = deepcopy(seed) + dsfmt_init_by_array(r.state, reinterpret(UInt32, data)) + reset_caches!(r) + r.adv = 0 + r.adv_jump = 0 + return r +end + +# When a seed is not provided, we generate one via `RandomDevice()` rather +# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, 8)` because the +# seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a +# small overhead compared to `initstate!`. +# A random seed with 128 bits is a good compromise for almost surely getting distinct +# seeds, while having them printed reasonably tersely. +seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128)) +seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice()) +seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed) + + +### generation + +# MersenneTwister produces natively Float64 +rng_native_52(::MersenneTwister) = Float64 + +#### helper functions + +# precondition: !mt_empty(r) +rand_inbounds(r::MersenneTwister, ::CloseOpen12_64) = mt_pop!(r) +rand_inbounds(r::MersenneTwister, ::CloseOpen01_64=CloseOpen01()) = + rand_inbounds(r, CloseOpen12()) - 1.0 + +rand_inbounds(r::MersenneTwister, ::UInt52Raw{T}) where {T<:BitInteger} = + reinterpret(UInt64, rand_inbounds(r, CloseOpen12())) % T + +function rand(r::MersenneTwister, x::SamplerTrivial{UInt52Raw{UInt64}}) + reserve_1(r) + rand_inbounds(r, x[]) +end + +function rand(r::MersenneTwister, ::SamplerTrivial{UInt2x52Raw{UInt128}}) + reserve(r, 2) + rand_inbounds(r, UInt52Raw(UInt128)) << 64 | rand_inbounds(r, UInt52Raw(UInt128)) +end + +function rand(r::MersenneTwister, ::SamplerTrivial{UInt104Raw{UInt128}}) + reserve(r, 2) + rand_inbounds(r, UInt52Raw(UInt128)) << 52 ⊻ rand_inbounds(r, UInt52Raw(UInt128)) +end + +#### floats + +rand(r::MersenneTwister, sp::SamplerTrivial{CloseOpen12_64}) = + (reserve_1(r); rand_inbounds(r, sp[])) + +#### integers + +rand(r::MersenneTwister, T::SamplerUnion(Int64, UInt64, Int128, UInt128)) = + mt_pop!(r, T[]) + +rand(r::MersenneTwister, T::SamplerUnion(Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32)) = + rand(r, UInt52Raw()) % T[] + +#### arrays of floats + +##### AbstractArray + +function rand!(r::MersenneTwister, A::AbstractArray{Float64}, + I::SamplerTrivial{<:FloatInterval_64}) + region = LinearIndices(A) + # what follows is equivalent to this simple loop but more efficient: + # for i=region + # @inbounds A[i] = rand(r, I[]) + # end + m = Base.checked_sub(first(region), 1) + n = last(region) + while m < n + s = mt_avail(r) + if s == 0 + gen_rand(r) + s = mt_avail(r) + end + m2 = min(n, m+s) + for i=m+1:m2 + @inbounds A[i] = rand_inbounds(r, I[]) + end + m = m2 + end + A +end + + +##### Array : internal functions + +# this is essentially equivalent to rand!(r, ::AbstractArray{Float64}, I) above, but due to +# optimizations which can't be done currently when working with pointers, we have to re-order +# manually the computation flow to get the performance +# (see https://discourse.julialang.org/t/unsafe-store-sometimes-slower-than-arrays-setindex) +function _rand_max383!(r::MersenneTwister, A::UnsafeView{Float64}, I::FloatInterval_64) + n = length(A) + @assert n <= dsfmt_get_min_array_size()+1 # == 383 + mt_avail(r) == 0 && gen_rand(r) + # from now on, at most one call to gen_rand(r) will be necessary + m = min(n, mt_avail(r)) + GC.@preserve r unsafe_copyto!(A.ptr, pointer(r.vals, r.idxF+1), m) + if m == n + r.idxF += m + else # m < n + gen_rand(r) + GC.@preserve r unsafe_copyto!(A.ptr+m*sizeof(Float64), pointer(r.vals), n-m) + r.idxF = n-m + end + if I isa CloseOpen01 + for i=1:n + A[i] -= 1.0 + end + end + A +end + +function fill_array!(rng::MersenneTwister, A::Ptr{Float64}, n::Int, I) + rng.adv += n + fill_array!(rng.state, A, n, I) +end + +fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen01_64) = + dsfmt_fill_array_close_open!(s, A, n) + +fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen12_64) = + dsfmt_fill_array_close1_open2!(s, A, n) + + +function rand!(r::MersenneTwister, A::UnsafeView{Float64}, + I::SamplerTrivial{<:FloatInterval_64}) + # depending on the alignment of A, the data written by fill_array! may have + # to be left-shifted by up to 15 bytes (cf. unsafe_copyto! below) for + # reproducibility purposes; + # so, even for well aligned arrays, fill_array! is used to generate only + # the n-2 first values (or n-3 if n is odd), and the remaining values are + # generated by the scalar version of rand + n = length(A) + n2 = (n-2) ÷ 2 * 2 + n2 < dsfmt_get_min_array_size() && return _rand_max383!(r, A, I[]) + + pA = A.ptr + align = Csize_t(pA) % 16 + if align > 0 + pA2 = pA + 16 - align + fill_array!(r, pA2, n2, I[]) # generate the data in-place, but shifted + unsafe_copyto!(pA, pA2, n2) # move the data to the beginning of the array + else + fill_array!(r, pA, n2, I[]) + end + for i=n2+1:n + A[i] = rand(r, I[]) + end + A +end + +# fills up A reinterpreted as an array of Float64 with n64 values +function _rand!(r::MersenneTwister, A::Array{T}, n64::Int, I::FloatInterval_64) where T + # n64 is the length in terms of `Float64` of the target + @assert sizeof(Float64)*n64 <= sizeof(T)*length(A) && isbitstype(T) + GC.@preserve A rand!(r, UnsafeView{Float64}(pointer(A), n64), SamplerTrivial(I)) + A +end + +##### Array: Float64, Float16, Float32 + +rand!(r::MersenneTwister, A::Array{Float64}, I::SamplerTrivial{<:FloatInterval_64}) = + _rand!(r, A, length(A), I[]) + +mask128(u::UInt128, ::Type{Float16}) = + (u & 0x03ff03ff03ff03ff03ff03ff03ff03ff) | 0x3c003c003c003c003c003c003c003c00 + +mask128(u::UInt128, ::Type{Float32}) = + (u & 0x007fffff007fffff007fffff007fffff) | 0x3f8000003f8000003f8000003f800000 + +for T in (Float16, Float32) + @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen12{$T}}) + n = length(A) + n128 = n * sizeof($T) ÷ 16 + _rand!(r, A, 2*n128, CloseOpen12()) + GC.@preserve A begin + A128 = UnsafeView{UInt128}(pointer(A), n128) + for i in 1:n128 + u = A128[i] + u ⊻= u << 26 + # at this point, the 64 low bits of u, "k" being the k-th bit of A128[i] and "+" + # the bit xor, are: + # [..., 58+32,..., 53+27, 52+26, ..., 33+7, 32+6, ..., 27+1, 26, ..., 1] + # the bits needing to be random are + # [1:10, 17:26, 33:42, 49:58] (for Float16) + # [1:23, 33:55] (for Float32) + # this is obviously satisfied on the 32 low bits side, and on the high side, + # the entropy comes from bits 33:52 of A128[i] and then from bits 27:32 + # (which are discarded on the low side) + # this is similar for the 64 high bits of u + A128[i] = mask128(u, $T) + end + end + for i in 16*n128÷sizeof($T)+1:n + @inbounds A[i] = rand(r, $T) + one($T) + end + A + end + + @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen01{$T}}) + rand!(r, A, CloseOpen12($T)) + I32 = one(Float32) + for i in eachindex(A) + @inbounds A[i] = Float32(A[i])-I32 # faster than "A[i] -= one(T)" for T==Float16 + end + A + end +end + +#### arrays of integers + +function rand!(r::MersenneTwister, A::UnsafeView{UInt128}, ::SamplerType{UInt128}) + n::Int=length(A) + i = n + while true + rand!(r, UnsafeView{Float64}(A.ptr, 2i), CloseOpen12()) + n < 5 && break + i = 0 + while n-i >= 5 + u = A[i+=1] + A[n] ⊻= u << 48 + A[n-=1] ⊻= u << 36 + A[n-=1] ⊻= u << 24 + A[n-=1] ⊻= u << 12 + n-=1 + end + end + if n > 0 + u = rand(r, UInt2x52Raw()) + for i = 1:n + A[i] ⊻= u << (12*i) + end + end + A +end + +for T in BitInteger_types + @eval function rand!(r::MersenneTwister, A::Array{$T}, sp::SamplerType{$T}) + GC.@preserve A rand!(r, UnsafeView(pointer(A), length(A)), sp) + A + end + + T == UInt128 && continue + + @eval function rand!(r::MersenneTwister, A::UnsafeView{$T}, ::SamplerType{$T}) + n = length(A) + n128 = n * sizeof($T) ÷ 16 + rand!(r, UnsafeView{UInt128}(pointer(A), n128)) + for i = 16*n128÷sizeof($T)+1:n + @inbounds A[i] = rand(r, $T) + end + A + end +end + + +#### arrays of Bool + +# similar to Array{UInt8}, but we need to mask the result so that only the LSB +# in each byte can be non-zero + +function rand!(r::MersenneTwister, A1::Array{Bool}, sp::SamplerType{Bool}) + n1 = length(A1) + n128 = n1 ÷ 16 + + if n128 == 0 + bits = rand(r, UInt52Raw()) + else + GC.@preserve A1 begin + A = UnsafeView{UInt128}(pointer(A1), n128) + rand!(r, UnsafeView{Float64}(A.ptr, 2*n128), CloseOpen12()) + # without masking, non-zero bits could be observed in other + # positions than the LSB of each byte + mask = 0x01010101010101010101010101010101 + # we need up to 15 bits of entropy in `bits` for the final loop, + # which we will extract from x = A[1] % UInt64; + # let y = x % UInt32; y contains 32 bits of entropy, but 4 + # of them will be used for A[1] itself (the first of + # each byte). To compensate, we xor with (y >> 17), + # which gets the entropy from the second bit of each byte + # of the upper-half of y, and sets it in the first bit + # of each byte of the lower half; the first two bytes + # now contain 16 usable random bits + x = A[1] % UInt64 + bits = x ⊻ x >> 17 + for i = 1:n128 + # << 5 to randomize the first bit of the 8th & 16th byte + # (i.e. we move bit 52 (resp. 52 + 64), which is unused, + # to position 57 (resp. 57 + 64)) + A[i] = (A[i] ⊻ A[i] << 5) & mask + end + end + end + for i = 16*n128+1:n1 + @inbounds A1[i] = bits % Bool + bits >>= 1 + end + A1 +end + + +### randjump + +# Old randjump methods are deprecated, the scalar version is in the Future module. + +function _randjump(r::MersenneTwister, jumppoly::DSFMT.GF2X) + adv = r.adv + adv_jump = r.adv_jump + s = MersenneTwister(r.seed, DSFMT.dsfmt_jump(r.state, jumppoly)) + reset_caches!(s) + s.adv = adv + s.adv_jump = adv_jump + s +end + +# NON-PUBLIC +function jump(r::MersenneTwister, steps::Integer) + iseven(steps) || throw(DomainError(steps, "steps must be even")) + # steps >= 0 checked in calc_jump (`steps >> 1 < 0` if `steps < 0`) + j = _randjump(r, Random.DSFMT.calc_jump(steps >> 1)) + j.adv_jump += steps + j +end + +# NON-PUBLIC +jump!(r::MersenneTwister, steps::Integer) = copy!(r, jump(r, steps)) + + +### constructors matching show (EXPERIMENTAL) + +# parameters in the tuples are: +# 1: .adv_jump (jump steps) +# 2: .adv (number of generated floats at the DSFMT_state level since seeding, besides jumps) +# 3, 4: .adv_vals, .idxF (counters to reconstruct the float cache, optional if 5-6 not shown)) +# 5, 6: .adv_ints, .idxI (counters to reconstruct the integer cache, optional) + +Random.MersenneTwister(seed, advance::NTuple{6,Integer}) = + advance!(MersenneTwister(seed), advance...) + +Random.MersenneTwister(seed, advance::NTuple{4,Integer}) = + MersenneTwister(seed, (advance..., 0, 0)) + +Random.MersenneTwister(seed, advance::NTuple{2,Integer}) = + MersenneTwister(seed, (advance..., 0, 0, 0, 0)) + +# advances raw state (per fill_array!) of r by n steps (Float64 values) +function _advance_n!(r::MersenneTwister, n::Int64, work::Vector{Float64}) + n == 0 && return + n < 0 && throw(DomainError(n, "can't advance $r to the specified state")) + ms = dsfmt_get_min_array_size() % Int64 + @assert n >= ms + lw = ms + n % ms + resize!(work, lw) + GC.@preserve work fill_array!(r, pointer(work), lw, CloseOpen12()) + c::Int64 = lw + GC.@preserve work while n > c + fill_array!(r, pointer(work), ms, CloseOpen12()) + c += ms + end + @assert n == c +end + +function _advance_to!(r::MersenneTwister, adv::Int64, work) + _advance_n!(r, adv - r.adv, work) + @assert r.adv == adv +end + +function _advance_F!(r::MersenneTwister, adv_vals, idxF, work) + _advance_to!(r, adv_vals, work) + gen_rand(r) + @assert r.adv_vals == adv_vals + r.idxF = idxF +end + +function _advance_I!(r::MersenneTwister, adv_ints, idxI, work) + _advance_to!(r, adv_ints, work) + mt_setfull!(r, Int) # sets r.adv_ints + @assert r.adv_ints == adv_ints + r.idxI = 16*length(r.ints) - 8*idxI +end + +function advance!(r::MersenneTwister, adv_jump, adv, adv_vals, idxF, adv_ints, idxI) + adv_jump = BigInt(adv_jump) + adv, adv_vals, adv_ints = Int64.((adv, adv_vals, adv_ints)) + idxF, idxI = Int.((idxF, idxI)) + + ms = dsfmt_get_min_array_size() % Int + work = sizehint!(Vector{Float64}(), 2ms) + + adv_jump != 0 && jump!(r, adv_jump) + advF = (adv_vals, idxF) != (0, 0) + advI = (adv_ints, idxI) != (0, 0) + + if advI && advF + @assert adv_vals != adv_ints + if adv_vals < adv_ints + _advance_F!(r, adv_vals, idxF, work) + _advance_I!(r, adv_ints, idxI, work) + else + _advance_I!(r, adv_ints, idxI, work) + _advance_F!(r, adv_vals, idxF, work) + end + elseif advF + _advance_F!(r, adv_vals, idxF, work) + elseif advI + _advance_I!(r, adv_ints, idxI, work) + else + @assert adv == 0 + end + _advance_to!(r, adv, work) + r +end diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index b8f71f3e2f3c8..3e8cb00f7787f 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -1,7 +1,66 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## RandomDevice +## default RNG + +""" + Random.default_rng() -> rng + +Return the default global random number generator (RNG), which is used by `rand`-related functions when +no explicit RNG is provided. + +When the `Random` module is loaded, the default RNG is _randomly_ seeded, via [`Random.seed!()`](@ref): +this means that each time a new julia session is started, the first call to `rand()` produces a different +result, unless `seed!(seed)` is called first. + +It is thread-safe: distinct threads can safely call `rand`-related functions on `default_rng()` concurrently, +e.g. `rand(default_rng())`. + +!!! note + The type of the default RNG is an implementation detail. Across different versions of + Julia, you should not expect the default RNG to always have the same type, nor that it will + produce the same stream of random numbers for a given seed. + +!!! compat "Julia 1.3" + This function was introduced in Julia 1.3. +""" +@inline default_rng() = TaskLocalRNG() +@inline default_rng(tid::Int) = TaskLocalRNG() + +# defined only for backward compatibility with pre-v1.3 code when `default_rng()` didn't exist; +# `GLOBAL_RNG` was never really documented, but was appearing in the docstring of `rand` +const GLOBAL_RNG = default_rng() + +# In v1.0, the GLOBAL_RNG was storing the seed which was used to initialize it; this seed was used to implement +# the following feature of `@testset`: +# > Before the execution of the body of a `@testset`, there is an implicit +# > call to `Random.seed!(seed)` where `seed` is the current seed of the global RNG. +# But the global RNG is now `TaskLocalRNG()` and doesn't store its seed; in order to not break `@testset`, +# in a call like `seed!(seed)` *without* an explicit RNG, we now store the state of `TaskLocalRNG()` in +# `task_local_storage()` + +# GLOBAL_SEED is used as a fall-back when no tls seed is found +# only `Random.__init__` is allowed to set it +const GLOBAL_SEED = Xoshiro(0, 0, 0, 0, 0) + +get_tls_seed() = get!(() -> copy(GLOBAL_SEED), task_local_storage(), + :__RANDOM_GLOBAL_RNG_SEED_uBlmfA8ZS__)::Xoshiro + +# seed the default RNG +function seed!(seed=nothing) + seed!(default_rng(), seed) + copy!(get_tls_seed(), default_rng()) + default_rng() +end + +function __init__() + # do not call no-arg `seed!()` to not update `task_local_storage()` unnecessarily at startup + seed!(default_rng()) + copy!(GLOBAL_SEED, TaskLocalRNG()) + ccall(:jl_gc_init_finalizer_rng_state, Cvoid, ()) +end + +## RandomDevice """ RandomDevice() @@ -43,247 +102,7 @@ end rng_native_52(::RandomDevice) = UInt64 -## MersenneTwister - -const MT_CACHE_F = 501 << 1 # number of Float64 in the cache -const MT_CACHE_I = 501 << 4 # number of bytes in the UInt128 cache - -@assert dsfmt_get_min_array_size() <= MT_CACHE_F - -mutable struct MersenneTwister <: AbstractRNG - seed::Any - state::DSFMT_state - vals::Vector{Float64} - ints::Vector{UInt128} - idxF::Int - idxI::Int - - # counters for show - adv::Int64 # state of advance at the DSFMT_state level - adv_jump::BigInt # number of skipped Float64 values via randjump - adv_vals::Int64 # state of advance when vals is filled-up - adv_ints::Int64 # state of advance when ints is filled-up - - function MersenneTwister(seed, state, vals, ints, idxF, idxI, - adv, adv_jump, adv_vals, adv_ints) - length(vals) == MT_CACHE_F && 0 <= idxF <= MT_CACHE_F || - throw(DomainError((length(vals), idxF), - "`length(vals)` and `idxF` must be consistent with $MT_CACHE_F")) - length(ints) == MT_CACHE_I >> 4 && 0 <= idxI <= MT_CACHE_I || - throw(DomainError((length(ints), idxI), - "`length(ints)` and `idxI` must be consistent with $MT_CACHE_I")) - new(seed, state, vals, ints, idxF, idxI, - adv, adv_jump, adv_vals, adv_ints) - end -end - -MersenneTwister(seed, state::DSFMT_state) = - MersenneTwister(seed, state, - Vector{Float64}(undef, MT_CACHE_F), - Vector{UInt128}(undef, MT_CACHE_I >> 4), - MT_CACHE_F, 0, 0, 0, -1, -1) - -""" - MersenneTwister(seed) - MersenneTwister() - -Create a `MersenneTwister` RNG object. Different RNG objects can have -their own seeds, which may be useful for generating different streams -of random numbers. -The `seed` may be an integer, a string, or a vector of `UInt32` integers. -If no seed is provided, a randomly generated one is created (using entropy from the system). -See the [`seed!`](@ref) function for reseeding an already existing `MersenneTwister` object. - -!!! compat "Julia 1.11" - Passing a negative integer seed requires at least Julia 1.11. - -# Examples -```jldoctest -julia> rng = MersenneTwister(123); - -julia> x1 = rand(rng, 2) -2-element Vector{Float64}: - 0.37453777969575874 - 0.8735343642013971 - -julia> x2 = rand(MersenneTwister(123), 2) -2-element Vector{Float64}: - 0.37453777969575874 - 0.8735343642013971 - -julia> x1 == x2 -true -``` -""" -MersenneTwister(seed=nothing) = - seed!(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) - - -function copy!(dst::MersenneTwister, src::MersenneTwister) - dst.seed = src.seed - copy!(dst.state, src.state) - copyto!(dst.vals, src.vals) - copyto!(dst.ints, src.ints) - dst.idxF = src.idxF - dst.idxI = src.idxI - dst.adv = src.adv - dst.adv_jump = src.adv_jump - dst.adv_vals = src.adv_vals - dst.adv_ints = src.adv_ints - dst -end - -copy(src::MersenneTwister) = - MersenneTwister(src.seed, copy(src.state), copy(src.vals), copy(src.ints), - src.idxF, src.idxI, src.adv, src.adv_jump, src.adv_vals, src.adv_ints) - - -==(r1::MersenneTwister, r2::MersenneTwister) = - r1.seed == r2.seed && r1.state == r2.state && - isequal(r1.vals, r2.vals) && - isequal(r1.ints, r2.ints) && - r1.idxF == r2.idxF && r1.idxI == r2.idxI - -hash(r::MersenneTwister, h::UInt) = - foldr(hash, (r.seed, r.state, r.vals, r.ints, r.idxF, r.idxI); init=h) - -function show(io::IO, rng::MersenneTwister) - # seed - if rng.adv_jump == 0 && rng.adv == 0 - return print(io, MersenneTwister, "(", repr(rng.seed), ")") - end - print(io, MersenneTwister, "(", repr(rng.seed), ", (") - # state - sep = ", " - show(io, rng.adv_jump) - print(io, sep) - show(io, rng.adv) - if rng.adv_vals != -1 || rng.adv_ints != -1 - # "(0, 0)" is nicer on the eyes than (-1, 1002) - s = rng.adv_vals != -1 - print(io, sep) - show(io, s ? rng.adv_vals : zero(rng.adv_vals)) - print(io, sep) - show(io, s ? rng.idxF : zero(rng.idxF)) - end - if rng.adv_ints != -1 - idxI = (length(rng.ints)*16 - rng.idxI) / 8 # 8 represents one Int64 - idxI = Int(idxI) # idxI should always be an integer when using public APIs - print(io, sep) - show(io, rng.adv_ints) - print(io, sep) - show(io, idxI) - end - print(io, "))") -end - -### low level API - -function reset_caches!(r::MersenneTwister) - # zeroing the caches makes comparing two MersenneTwister RNGs easier - fill!(r.vals, 0.0) - fill!(r.ints, zero(UInt128)) - mt_setempty!(r) - mt_setempty!(r, UInt128) - r.adv_vals = -1 - r.adv_ints = -1 - r -end - -#### floats - -mt_avail(r::MersenneTwister) = MT_CACHE_F - r.idxF -mt_empty(r::MersenneTwister) = r.idxF == MT_CACHE_F -mt_setfull!(r::MersenneTwister) = r.idxF = 0 -mt_setempty!(r::MersenneTwister) = r.idxF = MT_CACHE_F -mt_pop!(r::MersenneTwister) = @inbounds return r.vals[r.idxF+=1] - -@noinline function gen_rand(r::MersenneTwister) - r.adv_vals = r.adv - GC.@preserve r fill_array!(r, pointer(r.vals), length(r.vals), CloseOpen12()) - mt_setfull!(r) -end - -reserve_1(r::MersenneTwister) = (mt_empty(r) && gen_rand(r); nothing) -# `reserve` allows one to call `rand_inbounds` n times -# precondition: n <= MT_CACHE_F -reserve(r::MersenneTwister, n::Int) = (mt_avail(r) < n && gen_rand(r); nothing) - -#### ints - -logsizeof(::Type{<:Union{Bool,Int8,UInt8}}) = 0 -logsizeof(::Type{<:Union{Int16,UInt16}}) = 1 -logsizeof(::Type{<:Union{Int32,UInt32}}) = 2 -logsizeof(::Type{<:Union{Int64,UInt64}}) = 3 -logsizeof(::Type{<:Union{Int128,UInt128}}) = 4 - -idxmask(::Type{<:Union{Bool,Int8,UInt8}}) = 15 -idxmask(::Type{<:Union{Int16,UInt16}}) = 7 -idxmask(::Type{<:Union{Int32,UInt32}}) = 3 -idxmask(::Type{<:Union{Int64,UInt64}}) = 1 -idxmask(::Type{<:Union{Int128,UInt128}}) = 0 - - -mt_avail(r::MersenneTwister, ::Type{T}) where {T<:BitInteger} = - r.idxI >> logsizeof(T) - -function mt_setfull!(r::MersenneTwister, ::Type{<:BitInteger}) - r.adv_ints = r.adv - ints = r.ints - - @assert length(ints) == 501 - # dSFMT natively randomizes 52 out of 64 bits of each UInt64 words, - # i.e. 12 bits are missing; - # by generating 5 words == 5*52 == 260 bits, we can fully - # randomize 4 UInt64 = 256 bits; IOW, at the array level, we must - # randomize ceil(501*1.25) = 627 UInt128 words (with 2*52 bits each), - # which we then condense into fully randomized 501 UInt128 words - - len = 501 + 126 # 126 == ceil(501 / 4) - resize!(ints, len) - p = pointer(ints) # must be *after* resize! - GC.@preserve r fill_array!(r, Ptr{Float64}(p), len*2, CloseOpen12_64()) - - k = 501 - n = 0 - @inbounds while n != 500 - u = ints[k+=1] - ints[n+=1] ⊻= u << 48 - ints[n+=1] ⊻= u << 36 - ints[n+=1] ⊻= u << 24 - ints[n+=1] ⊻= u << 12 - end - @assert k == len - 1 - @inbounds ints[501] ⊻= ints[len] << 48 - resize!(ints, 501) - r.idxI = MT_CACHE_I -end - -mt_setempty!(r::MersenneTwister, ::Type{<:BitInteger}) = r.idxI = 0 - -function reserve1(r::MersenneTwister, ::Type{T}) where T<:BitInteger - r.idxI < sizeof(T) && mt_setfull!(r, T) - nothing -end - -function mt_pop!(r::MersenneTwister, ::Type{T}) where T<:BitInteger - reserve1(r, T) - r.idxI -= sizeof(T) - i = r.idxI - @inbounds x128 = r.ints[1 + i >> 4] - i128 = (i >> logsizeof(T)) & idxmask(T) # 0-based "indice" in x128 - (x128 >> (i128 * (sizeof(T) << 3))) % T -end - -function mt_pop!(r::MersenneTwister, ::Type{T}) where {T<:Union{Int128,UInt128}} - reserve1(r, T) - idx = r.idxI >> 4 - r.idxI = idx << 4 - 16 - @inbounds r.ints[idx] % T -end - - -### seeding +## SeedHasher """ Random.SeedHasher(seed=nothing) @@ -357,7 +176,94 @@ rand(rng::SeedHasher, ::SamplerType{Bool}) = rand(rng, UInt8) % Bool rng_native_52(::SeedHasher) = UInt64 -#### hash_seed() +## seeding + +""" + seed!([rng=default_rng()], seed) -> rng + seed!([rng=default_rng()]) -> rng + +Reseed the random number generator: `rng` will give a reproducible +sequence of numbers if and only if a `seed` is provided. Some RNGs +don't accept a seed, like `RandomDevice`. +After the call to `seed!`, `rng` is equivalent to a newly created +object initialized with the same seed. + +The types of accepted seeds depend on the type of `rng`, but in general, +integer seeds should work. Providing `nothing` as the seed should be +equivalent to not providing one. + +If `rng` is not specified, it defaults to seeding the state of the +shared task-local generator. + +# Examples +```julia-repl +julia> Random.seed!(1234); + +julia> x1 = rand(2) +2-element Vector{Float64}: + 0.32597672886359486 + 0.5490511363155669 + +julia> Random.seed!(1234); + +julia> x2 = rand(2) +2-element Vector{Float64}: + 0.32597672886359486 + 0.5490511363155669 + +julia> x1 == x2 +true + +julia> rng = Xoshiro(1234); rand(rng, 2) == x1 +true + +julia> Xoshiro(1) == Random.seed!(rng, 1) +true + +julia> rand(Random.seed!(rng), Bool) # not reproducible +true + +julia> rand(Random.seed!(rng), Bool) # not reproducible either +false + +julia> rand(Xoshiro(), Bool) # not reproducible either +true +``` +""" +seed! + +function seed!(rng::AbstractRNG, seed::Any=nothing) + if seed === nothing + seed!(rng, RandomDevice()) + elseif seed isa AbstractRNG + # avoid getting into an infinite recursive call from the other branches + throw(MethodError(seed!, (rng, seed))) + else + seed!(rng, SeedHasher(seed)) + end +end + + +### hash_seed() + +""" + Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8} + +Update `ctx` via `SHA.update!` with the content of `seed`. +This function is used by the [`SeedHasher`](@ref) RNG to produce +random bytes. + +`seed` can currently be of type +`Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`, +but modules can extend this function for types they own. + +`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`, +if `n != m`, then `cn` and `cm` will be distinct after calling +`hash_seed(n, cn); hash_seed(m, cm)`. +Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling +`hash_seed(n, cn); hash_seed(m, cm)`. +""" +function hash_seed end function hash_seed(seed::Integer, ctx::SHA_CTX) neg = signbit(seed) @@ -400,523 +306,3 @@ function hash_seed(str::AbstractString, ctx::SHA_CTX) end SHA.update!(ctx, (0x05,)) end - - -""" - Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8} - -Update `ctx` via `SHA.update!` with the content of `seed`. -This function is used by the [`SeedHasher`](@ref) RNG to produce -random bytes. - -`seed` can currently be of type -`Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`, -but modules can extend this function for types they own. - -`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`, -if `n != m`, then `cn` and `cm` will be distinct after calling -`hash_seed(n, cn); hash_seed(m, cm)`. -Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling -`hash_seed(n, cn); hash_seed(m, cm)`. -""" -hash_seed - - -#### seed!() - -function initstate!(r::MersenneTwister, data::StridedVector, seed) - # we deepcopy `seed` because the caller might mutate it, and it's useful - # to keep it constant inside `MersenneTwister`; but multiple instances - # can share the same seed without any problem (e.g. in `copy`) - r.seed = deepcopy(seed) - dsfmt_init_by_array(r.state, reinterpret(UInt32, data)) - reset_caches!(r) - r.adv = 0 - r.adv_jump = 0 - return r -end - -# When a seed is not provided, we generate one via `RandomDevice()` rather -# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, 8)` because the -# seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a -# small overhead compared to `initstate!`. -# A random seed with 128 bits is a good compromise for almost surely getting distinct -# seeds, while having them printed reasonably tersely. -seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128)) -seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice()) -seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed) - - -### Global RNG - -""" - Random.default_rng() -> rng - -Return the default global random number generator (RNG), which is used by `rand`-related functions when -no explicit RNG is provided. - -When the `Random` module is loaded, the default RNG is _randomly_ seeded, via [`Random.seed!()`](@ref): -this means that each time a new julia session is started, the first call to `rand()` produces a different -result, unless `seed!(seed)` is called first. - -It is thread-safe: distinct threads can safely call `rand`-related functions on `default_rng()` concurrently, -e.g. `rand(default_rng())`. - -!!! note - The type of the default RNG is an implementation detail. Across different versions of - Julia, you should not expect the default RNG to always have the same type, nor that it will - produce the same stream of random numbers for a given seed. - -!!! compat "Julia 1.3" - This function was introduced in Julia 1.3. -""" -@inline default_rng() = TaskLocalRNG() -@inline default_rng(tid::Int) = TaskLocalRNG() - -# defined only for backward compatibility with pre-v1.3 code when `default_rng()` didn't exist; -# `GLOBAL_RNG` was never really documented, but was appearing in the docstring of `rand` -const GLOBAL_RNG = default_rng() - -# In v1.0, the GLOBAL_RNG was storing the seed which was used to initialize it; this seed was used to implement -# the following feature of `@testset`: -# > Before the execution of the body of a `@testset`, there is an implicit -# > call to `Random.seed!(seed)` where `seed` is the current seed of the global RNG. -# But the global RNG is now `TaskLocalRNG()` and doesn't store its seed; in order to not break `@testset`, -# in a call like `seed!(seed)` *without* an explicit RNG, we now store the state of `TaskLocalRNG()` in -# `task_local_storage()` - -# GLOBAL_SEED is used as a fall-back when no tls seed is found -# only `Random.__init__` is allowed to set it -const GLOBAL_SEED = Xoshiro(0, 0, 0, 0, 0) - -get_tls_seed() = get!(() -> copy(GLOBAL_SEED), task_local_storage(), - :__RANDOM_GLOBAL_RNG_SEED_uBlmfA8ZS__)::Xoshiro - -# seed the default RNG -function seed!(seed=nothing) - seed!(default_rng(), seed) - copy!(get_tls_seed(), default_rng()) - default_rng() -end - -function __init__() - # do not call no-arg `seed!()` to not update `task_local_storage()` unnecessarily at startup - seed!(default_rng()) - copy!(GLOBAL_SEED, TaskLocalRNG()) - ccall(:jl_gc_init_finalizer_rng_state, Cvoid, ()) -end - - -### generation - -# MersenneTwister produces natively Float64 -rng_native_52(::MersenneTwister) = Float64 - -#### helper functions - -# precondition: !mt_empty(r) -rand_inbounds(r::MersenneTwister, ::CloseOpen12_64) = mt_pop!(r) -rand_inbounds(r::MersenneTwister, ::CloseOpen01_64=CloseOpen01()) = - rand_inbounds(r, CloseOpen12()) - 1.0 - -rand_inbounds(r::MersenneTwister, ::UInt52Raw{T}) where {T<:BitInteger} = - reinterpret(UInt64, rand_inbounds(r, CloseOpen12())) % T - -function rand(r::MersenneTwister, x::SamplerTrivial{UInt52Raw{UInt64}}) - reserve_1(r) - rand_inbounds(r, x[]) -end - -function rand(r::MersenneTwister, ::SamplerTrivial{UInt2x52Raw{UInt128}}) - reserve(r, 2) - rand_inbounds(r, UInt52Raw(UInt128)) << 64 | rand_inbounds(r, UInt52Raw(UInt128)) -end - -function rand(r::MersenneTwister, ::SamplerTrivial{UInt104Raw{UInt128}}) - reserve(r, 2) - rand_inbounds(r, UInt52Raw(UInt128)) << 52 ⊻ rand_inbounds(r, UInt52Raw(UInt128)) -end - -#### floats - -rand(r::MersenneTwister, sp::SamplerTrivial{CloseOpen12_64}) = - (reserve_1(r); rand_inbounds(r, sp[])) - -#### integers - -rand(r::MersenneTwister, T::SamplerUnion(Int64, UInt64, Int128, UInt128)) = - mt_pop!(r, T[]) - -rand(r::MersenneTwister, T::SamplerUnion(Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32)) = - rand(r, UInt52Raw()) % T[] - -#### arrays of floats - -##### AbstractArray - -function rand!(r::MersenneTwister, A::AbstractArray{Float64}, - I::SamplerTrivial{<:FloatInterval_64}) - region = LinearIndices(A) - # what follows is equivalent to this simple loop but more efficient: - # for i=region - # @inbounds A[i] = rand(r, I[]) - # end - m = Base.checked_sub(first(region), 1) - n = last(region) - while m < n - s = mt_avail(r) - if s == 0 - gen_rand(r) - s = mt_avail(r) - end - m2 = min(n, m+s) - for i=m+1:m2 - @inbounds A[i] = rand_inbounds(r, I[]) - end - m = m2 - end - A -end - - -##### Array : internal functions - -# internal array-like type to circumvent the lack of flexibility with reinterpret -struct UnsafeView{T} <: DenseArray{T,1} - ptr::Ptr{T} - len::Int -end - -Base.length(a::UnsafeView) = a.len -Base.getindex(a::UnsafeView, i::Int) = unsafe_load(a.ptr, i) -Base.setindex!(a::UnsafeView, x, i::Int) = unsafe_store!(a.ptr, x, i) -Base.pointer(a::UnsafeView) = a.ptr -Base.size(a::UnsafeView) = (a.len,) -Base.elsize(::Type{UnsafeView{T}}) where {T} = sizeof(T) - -# this is essentially equivalent to rand!(r, ::AbstractArray{Float64}, I) above, but due to -# optimizations which can't be done currently when working with pointers, we have to re-order -# manually the computation flow to get the performance -# (see https://discourse.julialang.org/t/unsafe-store-sometimes-slower-than-arrays-setindex) -function _rand_max383!(r::MersenneTwister, A::UnsafeView{Float64}, I::FloatInterval_64) - n = length(A) - @assert n <= dsfmt_get_min_array_size()+1 # == 383 - mt_avail(r) == 0 && gen_rand(r) - # from now on, at most one call to gen_rand(r) will be necessary - m = min(n, mt_avail(r)) - GC.@preserve r unsafe_copyto!(A.ptr, pointer(r.vals, r.idxF+1), m) - if m == n - r.idxF += m - else # m < n - gen_rand(r) - GC.@preserve r unsafe_copyto!(A.ptr+m*sizeof(Float64), pointer(r.vals), n-m) - r.idxF = n-m - end - if I isa CloseOpen01 - for i=1:n - A[i] -= 1.0 - end - end - A -end - -function fill_array!(rng::MersenneTwister, A::Ptr{Float64}, n::Int, I) - rng.adv += n - fill_array!(rng.state, A, n, I) -end - -fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen01_64) = - dsfmt_fill_array_close_open!(s, A, n) - -fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen12_64) = - dsfmt_fill_array_close1_open2!(s, A, n) - - -function rand!(r::MersenneTwister, A::UnsafeView{Float64}, - I::SamplerTrivial{<:FloatInterval_64}) - # depending on the alignment of A, the data written by fill_array! may have - # to be left-shifted by up to 15 bytes (cf. unsafe_copyto! below) for - # reproducibility purposes; - # so, even for well aligned arrays, fill_array! is used to generate only - # the n-2 first values (or n-3 if n is odd), and the remaining values are - # generated by the scalar version of rand - n = length(A) - n2 = (n-2) ÷ 2 * 2 - n2 < dsfmt_get_min_array_size() && return _rand_max383!(r, A, I[]) - - pA = A.ptr - align = Csize_t(pA) % 16 - if align > 0 - pA2 = pA + 16 - align - fill_array!(r, pA2, n2, I[]) # generate the data in-place, but shifted - unsafe_copyto!(pA, pA2, n2) # move the data to the beginning of the array - else - fill_array!(r, pA, n2, I[]) - end - for i=n2+1:n - A[i] = rand(r, I[]) - end - A -end - -# fills up A reinterpreted as an array of Float64 with n64 values -function _rand!(r::MersenneTwister, A::Array{T}, n64::Int, I::FloatInterval_64) where T - # n64 is the length in terms of `Float64` of the target - @assert sizeof(Float64)*n64 <= sizeof(T)*length(A) && isbitstype(T) - GC.@preserve A rand!(r, UnsafeView{Float64}(pointer(A), n64), SamplerTrivial(I)) - A -end - -##### Array: Float64, Float16, Float32 - -rand!(r::MersenneTwister, A::Array{Float64}, I::SamplerTrivial{<:FloatInterval_64}) = - _rand!(r, A, length(A), I[]) - -mask128(u::UInt128, ::Type{Float16}) = - (u & 0x03ff03ff03ff03ff03ff03ff03ff03ff) | 0x3c003c003c003c003c003c003c003c00 - -mask128(u::UInt128, ::Type{Float32}) = - (u & 0x007fffff007fffff007fffff007fffff) | 0x3f8000003f8000003f8000003f800000 - -for T in (Float16, Float32) - @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen12{$T}}) - n = length(A) - n128 = n * sizeof($T) ÷ 16 - _rand!(r, A, 2*n128, CloseOpen12()) - GC.@preserve A begin - A128 = UnsafeView{UInt128}(pointer(A), n128) - for i in 1:n128 - u = A128[i] - u ⊻= u << 26 - # at this point, the 64 low bits of u, "k" being the k-th bit of A128[i] and "+" - # the bit xor, are: - # [..., 58+32,..., 53+27, 52+26, ..., 33+7, 32+6, ..., 27+1, 26, ..., 1] - # the bits needing to be random are - # [1:10, 17:26, 33:42, 49:58] (for Float16) - # [1:23, 33:55] (for Float32) - # this is obviously satisfied on the 32 low bits side, and on the high side, - # the entropy comes from bits 33:52 of A128[i] and then from bits 27:32 - # (which are discarded on the low side) - # this is similar for the 64 high bits of u - A128[i] = mask128(u, $T) - end - end - for i in 16*n128÷sizeof($T)+1:n - @inbounds A[i] = rand(r, $T) + one($T) - end - A - end - - @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen01{$T}}) - rand!(r, A, CloseOpen12($T)) - I32 = one(Float32) - for i in eachindex(A) - @inbounds A[i] = Float32(A[i])-I32 # faster than "A[i] -= one(T)" for T==Float16 - end - A - end -end - -#### arrays of integers - -function rand!(r::MersenneTwister, A::UnsafeView{UInt128}, ::SamplerType{UInt128}) - n::Int=length(A) - i = n - while true - rand!(r, UnsafeView{Float64}(A.ptr, 2i), CloseOpen12()) - n < 5 && break - i = 0 - while n-i >= 5 - u = A[i+=1] - A[n] ⊻= u << 48 - A[n-=1] ⊻= u << 36 - A[n-=1] ⊻= u << 24 - A[n-=1] ⊻= u << 12 - n-=1 - end - end - if n > 0 - u = rand(r, UInt2x52Raw()) - for i = 1:n - A[i] ⊻= u << (12*i) - end - end - A -end - -for T in BitInteger_types - @eval function rand!(r::MersenneTwister, A::Array{$T}, sp::SamplerType{$T}) - GC.@preserve A rand!(r, UnsafeView(pointer(A), length(A)), sp) - A - end - - T == UInt128 && continue - - @eval function rand!(r::MersenneTwister, A::UnsafeView{$T}, ::SamplerType{$T}) - n = length(A) - n128 = n * sizeof($T) ÷ 16 - rand!(r, UnsafeView{UInt128}(pointer(A), n128)) - for i = 16*n128÷sizeof($T)+1:n - @inbounds A[i] = rand(r, $T) - end - A - end -end - - -#### arrays of Bool - -# similar to Array{UInt8}, but we need to mask the result so that only the LSB -# in each byte can be non-zero - -function rand!(r::MersenneTwister, A1::Array{Bool}, sp::SamplerType{Bool}) - n1 = length(A1) - n128 = n1 ÷ 16 - - if n128 == 0 - bits = rand(r, UInt52Raw()) - else - GC.@preserve A1 begin - A = UnsafeView{UInt128}(pointer(A1), n128) - rand!(r, UnsafeView{Float64}(A.ptr, 2*n128), CloseOpen12()) - # without masking, non-zero bits could be observed in other - # positions than the LSB of each byte - mask = 0x01010101010101010101010101010101 - # we need up to 15 bits of entropy in `bits` for the final loop, - # which we will extract from x = A[1] % UInt64; - # let y = x % UInt32; y contains 32 bits of entropy, but 4 - # of them will be used for A[1] itself (the first of - # each byte). To compensate, we xor with (y >> 17), - # which gets the entropy from the second bit of each byte - # of the upper-half of y, and sets it in the first bit - # of each byte of the lower half; the first two bytes - # now contain 16 usable random bits - x = A[1] % UInt64 - bits = x ⊻ x >> 17 - for i = 1:n128 - # << 5 to randomize the first bit of the 8th & 16th byte - # (i.e. we move bit 52 (resp. 52 + 64), which is unused, - # to position 57 (resp. 57 + 64)) - A[i] = (A[i] ⊻ A[i] << 5) & mask - end - end - end - for i = 16*n128+1:n1 - @inbounds A1[i] = bits % Bool - bits >>= 1 - end - A1 -end - - -### randjump - -# Old randjump methods are deprecated, the scalar version is in the Future module. - -function _randjump(r::MersenneTwister, jumppoly::DSFMT.GF2X) - adv = r.adv - adv_jump = r.adv_jump - s = MersenneTwister(r.seed, DSFMT.dsfmt_jump(r.state, jumppoly)) - reset_caches!(s) - s.adv = adv - s.adv_jump = adv_jump - s -end - -# NON-PUBLIC -function jump(r::MersenneTwister, steps::Integer) - iseven(steps) || throw(DomainError(steps, "steps must be even")) - # steps >= 0 checked in calc_jump (`steps >> 1 < 0` if `steps < 0`) - j = _randjump(r, Random.DSFMT.calc_jump(steps >> 1)) - j.adv_jump += steps - j -end - -# NON-PUBLIC -jump!(r::MersenneTwister, steps::Integer) = copy!(r, jump(r, steps)) - - -### constructors matching show (EXPERIMENTAL) - -# parameters in the tuples are: -# 1: .adv_jump (jump steps) -# 2: .adv (number of generated floats at the DSFMT_state level since seeding, besides jumps) -# 3, 4: .adv_vals, .idxF (counters to reconstruct the float cache, optional if 5-6 not shown)) -# 5, 6: .adv_ints, .idxI (counters to reconstruct the integer cache, optional) - -Random.MersenneTwister(seed, advance::NTuple{6,Integer}) = - advance!(MersenneTwister(seed), advance...) - -Random.MersenneTwister(seed, advance::NTuple{4,Integer}) = - MersenneTwister(seed, (advance..., 0, 0)) - -Random.MersenneTwister(seed, advance::NTuple{2,Integer}) = - MersenneTwister(seed, (advance..., 0, 0, 0, 0)) - -# advances raw state (per fill_array!) of r by n steps (Float64 values) -function _advance_n!(r::MersenneTwister, n::Int64, work::Vector{Float64}) - n == 0 && return - n < 0 && throw(DomainError(n, "can't advance $r to the specified state")) - ms = dsfmt_get_min_array_size() % Int64 - @assert n >= ms - lw = ms + n % ms - resize!(work, lw) - GC.@preserve work fill_array!(r, pointer(work), lw, CloseOpen12()) - c::Int64 = lw - GC.@preserve work while n > c - fill_array!(r, pointer(work), ms, CloseOpen12()) - c += ms - end - @assert n == c -end - -function _advance_to!(r::MersenneTwister, adv::Int64, work) - _advance_n!(r, adv - r.adv, work) - @assert r.adv == adv -end - -function _advance_F!(r::MersenneTwister, adv_vals, idxF, work) - _advance_to!(r, adv_vals, work) - gen_rand(r) - @assert r.adv_vals == adv_vals - r.idxF = idxF -end - -function _advance_I!(r::MersenneTwister, adv_ints, idxI, work) - _advance_to!(r, adv_ints, work) - mt_setfull!(r, Int) # sets r.adv_ints - @assert r.adv_ints == adv_ints - r.idxI = 16*length(r.ints) - 8*idxI -end - -function advance!(r::MersenneTwister, adv_jump, adv, adv_vals, idxF, adv_ints, idxI) - adv_jump = BigInt(adv_jump) - adv, adv_vals, adv_ints = Int64.((adv, adv_vals, adv_ints)) - idxF, idxI = Int.((idxF, idxI)) - - ms = dsfmt_get_min_array_size() % Int - work = sizehint!(Vector{Float64}(), 2ms) - - adv_jump != 0 && jump!(r, adv_jump) - advF = (adv_vals, idxF) != (0, 0) - advI = (adv_ints, idxI) != (0, 0) - - if advI && advF - @assert adv_vals != adv_ints - if adv_vals < adv_ints - _advance_F!(r, adv_vals, idxF, work) - _advance_I!(r, adv_ints, idxI, work) - else - _advance_I!(r, adv_ints, idxI, work) - _advance_F!(r, adv_vals, idxF, work) - end - elseif advF - _advance_F!(r, adv_vals, idxF, work) - elseif advI - _advance_I!(r, adv_ints, idxI, work) - else - @assert adv == 0 - end - _advance_to!(r, adv, work) - r -end diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 796b3edfe6321..809a98dfa27e8 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -165,6 +165,11 @@ Sampler(::Type{<:AbstractRNG}, ::Type{T}, ::Repetition) where {T} = SamplerType{ Base.getindex(::SamplerType{T}) where {T} = T +# SamplerUnion(X, Y, ...}) == Union{SamplerType{X}, SamplerType{Y}, ...} +SamplerUnion(U...) = Union{Any[SamplerType{T} for T in U]...} +const SamplerBoolBitInteger = SamplerUnion(Bool, BitInteger_types...) + + struct SamplerTrivial{T,E} <: Sampler{E} self::T end @@ -293,19 +298,24 @@ rand( ::Type{X}, dims::Dims) where {X} = rand(default_rng(), X, d rand(r::AbstractRNG, ::Type{X}, d::Integer, dims::Integer...) where {X} = rand(r, X, Dims((d, dims...))) rand( ::Type{X}, d::Integer, dims::Integer...) where {X} = rand(X, Dims((d, dims...))) -# SamplerUnion(X, Y, ...}) == Union{SamplerType{X}, SamplerType{Y}, ...} -SamplerUnion(U...) = Union{Any[SamplerType{T} for T in U]...} -const SamplerBoolBitInteger = SamplerUnion(Bool, BitInteger_types...) +### UnsafeView +# internal array-like type to circumvent the lack of flexibility with reinterpret -include("Xoshiro.jl") -include("RNGs.jl") -include("generation.jl") -include("normal.jl") -include("misc.jl") -include("XoshiroSimd.jl") +struct UnsafeView{T} <: DenseArray{T,1} + ptr::Ptr{T} + len::Int +end + +Base.length(a::UnsafeView) = a.len +Base.getindex(a::UnsafeView, i::Int) = unsafe_load(a.ptr, i) +Base.setindex!(a::UnsafeView, x, i::Int) = unsafe_store!(a.ptr, x, i) +Base.pointer(a::UnsafeView) = a.ptr +Base.size(a::UnsafeView) = (a.len,) +Base.elsize(::Type{UnsafeView{T}}) where {T} = sizeof(T) -## rand & rand! & seed! docstrings + +## rand & rand! docstrings """ rand([rng=default_rng()], [S], [dims...]) @@ -405,67 +415,13 @@ julia> rand!(Xoshiro(123), zeros(5)) """ rand! -""" - seed!([rng=default_rng()], seed) -> rng - seed!([rng=default_rng()]) -> rng - -Reseed the random number generator: `rng` will give a reproducible -sequence of numbers if and only if a `seed` is provided. Some RNGs -don't accept a seed, like `RandomDevice`. -After the call to `seed!`, `rng` is equivalent to a newly created -object initialized with the same seed. - -The types of accepted seeds depend on the type of `rng`, but in general, -integer seeds should work. Providing `nothing` as the seed should be -equivalent to not providing one. - -If `rng` is not specified, it defaults to seeding the state of the -shared task-local generator. -# Examples -```julia-repl -julia> Random.seed!(1234); - -julia> x1 = rand(2) -2-element Vector{Float64}: - 0.32597672886359486 - 0.5490511363155669 - -julia> Random.seed!(1234); - -julia> x2 = rand(2) -2-element Vector{Float64}: - 0.32597672886359486 - 0.5490511363155669 - -julia> x1 == x2 -true - -julia> rng = Xoshiro(1234); rand(rng, 2) == x1 -true - -julia> Xoshiro(1) == Random.seed!(rng, 1) -true - -julia> rand(Random.seed!(rng), Bool) # not reproducible -true - -julia> rand(Random.seed!(rng), Bool) # not reproducible either -false - -julia> rand(Xoshiro(), Bool) # not reproducible either -true -``` -""" -function seed!(rng::AbstractRNG, seed::Any=nothing) - if seed === nothing - seed!(rng, RandomDevice()) - elseif seed isa AbstractRNG - # avoid getting into an infinite recursive call from the other branches - throw(MethodError(seed!, (rng, seed))) - else - seed!(rng, SeedHasher(seed)) - end -end +include("Xoshiro.jl") +include("RNGs.jl") +include("MersenneTwister.jl") +include("generation.jl") +include("normal.jl") +include("misc.jl") +include("XoshiroSimd.jl") end # module From d1ec7d57000604c9335531197798f2267d511021 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 May 2025 16:52:23 -0400 Subject: [PATCH 217/662] extend Method.dispatch_status optimization to ml_matches_visitor also (#58335) This extends the use of the optimization in #58291 to also apply to some uses of ml_matches also. --- src/gf.c | 55 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/gf.c b/src/gf.c index f101c3f86db10..b1552fa50d0d9 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1682,7 +1682,6 @@ static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_in assert(jl_atomic_load_relaxed(&oldentry->min_world) <= jl_atomic_load_relaxed(&closure->newentry->min_world) && "old method cannot be newer than new method"); assert(jl_atomic_load_relaxed(&oldentry->max_world) != jl_atomic_load_relaxed(&closure->newentry->min_world) && "method cannot be added at the same time as method deleted"); // don't need to consider other similar methods if this oldentry will always fully intersect with them and dominates all of them - typemap_slurp_search(oldentry, &closure->match); jl_method_t *oldmethod = oldentry->func.method; if (closure->match.issubty // e.g. jl_subtype(closure->newentry.sig, oldentry->sig) && jl_subtype(oldmethod->sig, (jl_value_t*)closure->newentry->sig)) { // e.g. jl_type_equal(closure->newentry->sig, oldentry->sig) @@ -1691,7 +1690,18 @@ static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_in } if (closure->shadowed == NULL) closure->shadowed = (jl_value_t*)jl_alloc_vec_any(0); + if (closure->match.issubty) { // this should be rarely true (in fact, get_intersect_visitor should be rarely true), but might as well skip the rest of the scan fast anyways since we can + int only = jl_atomic_load_relaxed(&oldmethod->dispatch_status) & METHOD_SIG_LATEST_ONLY; + if (only) { + size_t len = jl_array_nrows(closure->shadowed); + if (len > 0) + jl_array_del_end((jl_array_t*)closure->shadowed, len); + jl_array_ptr_1d_push((jl_array_t*)closure->shadowed, (jl_value_t*)oldmethod); + return 0; + } + } jl_array_ptr_1d_push((jl_array_t*)closure->shadowed, (jl_value_t*)oldmethod); + typemap_slurp_search(oldentry, &closure->match); return 1; } @@ -3912,30 +3922,26 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio if (closure->match.max_valid > max_world) closure->match.max_valid = max_world; jl_method_t *meth = ml->func.method; - if (closure->lim >= 0 && jl_is_dispatch_tupletype(meth->sig)) { - int replaced = 0; - // check if this is replaced, in which case we need to avoid double-counting it against the limit - // (although it will figure out later which one to keep and return) - size_t len = jl_array_nrows(closure->t); - for (int i = 0; i < len; i++) { - if (jl_types_equal(((jl_method_match_t*)jl_array_ptr_ref(closure->t, i))->method->sig, meth->sig)) { - replaced = 1; - break; - } - } - if (!replaced) { - if (closure->lim == 0) - return 0; - closure->lim--; + int only = jl_atomic_load_relaxed(&meth->dispatch_status) & METHOD_SIG_LATEST_ONLY; + if (closure->lim >= 0 && only) { + if (closure->lim == 0) { + closure->t = jl_an_empty_vec_any; + return 0; } + closure->lim--; } - // don't need to consider other similar methods if this ml will always fully intersect with them and dominates all of them - if (!closure->include_ambiguous || closure->lim != -1) - typemap_slurp_search(ml, &closure->match); closure->matc = make_method_match((jl_tupletype_t*)closure->match.ti, closure->match.env, meth, closure->match.issubty ? FULLY_COVERS : NOT_FULLY_COVERS); size_t len = jl_array_nrows(closure->t); + if (closure->match.issubty && only) { + if (len == 0) + closure->t = (jl_value_t*)jl_alloc_vec_any(1); + else if (len > 1) + jl_array_del_end((jl_array_t*)closure->t, len - 1); + jl_array_ptr_set(closure->t, 0, (jl_value_t*)closure->matc); + return 0; + } if (len == 0) { closure->t = (jl_value_t*)jl_alloc_vec_any(1); jl_array_ptr_set(closure->t, 0, (jl_value_t*)closure->matc); @@ -3943,6 +3949,9 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio else { jl_array_ptr_1d_push((jl_array_t*)closure->t, (jl_value_t*)closure->matc); } + // don't need to consider other similar methods if this ml will always fully intersect with them and dominates all of them + if (!closure->include_ambiguous || closure->lim != -1) + typemap_slurp_search(ml, &closure->match); return 1; } @@ -4323,9 +4332,9 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, return env.t; } } - if (!ml_mtable_visitor(mt, &env.match)) { + if (!ml_mtable_visitor(mt, &env.match) && env.t == jl_an_empty_vec_any) { JL_GC_POP(); - // if we return early, set only the min/max valid collected from matching + // if we return early without returning methods, set only the min/max valid collected from matching *min_valid = env.match.min_valid; *max_valid = env.match.max_valid; return jl_nothing; @@ -4333,9 +4342,9 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, } else { // else: scan everything - if (!jl_foreach_reachable_mtable(ml_mtable_visitor, &env.match)) { + if (!jl_foreach_reachable_mtable(ml_mtable_visitor, &env.match) && env.t == jl_an_empty_vec_any) { JL_GC_POP(); - // if we return early, set only the min/max valid collected from matching + // if we return early without returning methods, set only the min/max valid collected from matching *min_valid = env.match.min_valid; *max_valid = env.match.max_valid; return jl_nothing; From 6165395a13af363d2d82b864de511b8f0ac7b705 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 May 2025 17:21:25 -0400 Subject: [PATCH 218/662] atomics: avoid iterator invalidation in C++ Observed causing segfaults in CI. --- src/codegen.cpp | 93 +++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 07cbf7423c614..97f2b8b98f2db 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -9874,56 +9874,57 @@ void linkFunctionBody(Function &Dst, Function &Src) void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { - jl_workqueue_t &edges = params.workqueue; - bool always_inline = false; - for (auto &it : edges) { - if (it.second.private_linkage) - always_inline = true; - } - if (!always_inline) - return; - jl_task_t *ct = jl_current_task; - int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) - jl_code_info_t *src = nullptr; - params.safepoint_on_entry = false; - params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); - JL_GC_PUSH2(¶ms.temporary_roots, &src); - for (auto &it : edges) { - jl_code_instance_t *codeinst = it.first; - auto &proto = it.second; - if (!proto.private_linkage) - continue; - if (proto.decl->isDeclaration()) { - src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - jl_method_t *def = mi->def.method; - if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) - src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); - if (src && jl_is_code_info(src) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) { - jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints - if (!result_m) - break; - // TODO: jl_optimize_roots(params, mi, *result_m.getModuleUnlocked()); // contains safepoints - Module &M = *result_m.getModuleUnlocked(); - if (decls.functionObject != "jl_fptr_args" && - decls.functionObject != "jl_fptr_sparam" && - decls.functionObject != "jl_f_opaque_closure_call") { - Function *F = M.getFunction(decls.functionObject); - F->eraseFromParent(); - } - if (!decls.specFunctionObject.empty()) { - Function *specF = M.getFunction(decls.specFunctionObject); - linkFunctionBody(*proto.decl, *specF); - proto.decl->addFnAttr(Attribute::InlineHint); - proto.decl->setLinkage(proto.external_linkage ? GlobalValue::AvailableExternallyLinkage : GlobalValue::PrivateLinkage); - specF->eraseFromParent(); + while (true) { + SmallVector always_inline; + for (auto &it : params.workqueue) { + if (it.second.private_linkage && it.second.decl->isDeclaration()) + always_inline.push_back(it); + it.second.private_linkage = false; + } + if (always_inline.empty()) + return; + jl_task_t *ct = jl_current_task; + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) + jl_code_info_t *src = nullptr; + params.safepoint_on_entry = false; + params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); + JL_GC_PUSH2(¶ms.temporary_roots, &src); + for (auto &it : always_inline) { + jl_code_instance_t *codeinst = it.first; + auto &proto = it.second; + Function *decl = proto.decl; + if (decl->isDeclaration()) { + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + jl_method_t *def = mi->def.method; + if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); + if (src && jl_is_code_info(src) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) { + jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints + if (!result_m) + break; + // TODO: jl_optimize_roots(params, mi, *result_m.getModuleUnlocked()); // contains safepoints + Module &M = *result_m.getModuleUnlocked(); + if (decls.functionObject != "jl_fptr_args" && + decls.functionObject != "jl_fptr_sparam" && + decls.functionObject != "jl_f_opaque_closure_call") { + Function *F = M.getFunction(decls.functionObject); + F->eraseFromParent(); + } + if (!decls.specFunctionObject.empty()) { + Function *specF = M.getFunction(decls.specFunctionObject); + linkFunctionBody(*decl, *specF); + decl->addFnAttr(Attribute::InlineHint); + decl->setLinkage(proto.external_linkage ? GlobalValue::AvailableExternallyLinkage : GlobalValue::PrivateLinkage); + specF->eraseFromParent(); + } } } } + params.temporary_roots = nullptr; + JL_GC_POP(); + jl_gc_unsafe_leave(ct->ptls, gc_state); } - params.temporary_roots = nullptr; - JL_GC_POP(); - jl_gc_unsafe_leave(ct->ptls, gc_state); } // --- initialization --- From 555d3baec5a4a6b3e83c97f4300722e931a5cf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesper=20=C3=96qvist?= Date: Thu, 8 May 2025 01:36:02 +0200 Subject: [PATCH 219/662] Remove obsolete comment (#57698) The `bufmode_t` enum was added to `src/support/ios.h` in d316c93ab3cca456b4cd37f5859ff472550ea2d3 by Jeff Bezanson with the comment ```c // this flag controls when data actually moves out to the underlying I/O // channel. memory streams are a special case of this where the data // never moves out. typedef enum { bm_none, bm_line, bm_block, bm_mem } bufmode_t; ``` In 30aed865b713a546d2d2eb44b88b43aa1c27e1ed Keno Fischer added a new comment and modified the enum: ```diff // this flag controls when data actually moves out to the underlying I/O // channel. memory streams are a special case of this where the data // never moves out. -typedef enum { bm_none, bm_line, bm_block, bm_mem } bufmode_t; + +//make it compatible with UV Handles +typedef enum { bm_none=UV_HANDLE_TYPE_MAX+1, bm_line, bm_block, bm_mem } bufmode_t; ``` In c12aca890a8ae387d11db0b54351f8b61305c00b Jameson Nash removed `uv.h` and updated the enum definition to no longer use `UV_HANDLE_TYPE_MAX`: ```diff -typedef enum { bm_none=UV_HANDLE_TYPE_MAX+1, bm_line, bm_block, bm_mem } bufmode_t; +typedef enum { bm_none=1000, bm_line, bm_block, bm_mem } bufmode_t; ``` This last change made the comment by Keno Fischer seemingly obsolete, yet the comment remains. This pull request simply removes the two lines remaining from Keno Fischer's commit. Perhaps it is better to update the comment to explain why `bm_none` now has the value `1000` but I did not dig deep enough into the changes to figure out if this value is significant in some way that should be documented. --- src/support/ios.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/support/ios.h b/src/support/ios.h index 6eab9e21c45b6..bd26ee8d28181 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -14,8 +14,6 @@ extern "C" { // this flag controls when data actually moves out to the underlying I/O // channel. memory streams are a special case of this where the data // never moves out. - -//make it compatible with UV Handles typedef enum { bm_none=1000, bm_line, bm_block, bm_mem } bufmode_t; typedef enum { bst_none, bst_rd, bst_wr } bufstate_t; From 2dc5928c035a29ce0eaced609364fa7d2779b3cc Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Thu, 8 May 2025 01:39:33 +0200 Subject: [PATCH 220/662] More efficient chomp for String (#58192) We can avoid all the computation of valid indices for this simple function, since it only strips CR (0xD) and LF (0xA), which are known to be single bytes. This is approximately four times faster. --------- Co-authored-by: Dilum Aluthge --- base/strings/util.jl | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/base/strings/util.jl b/base/strings/util.jl index 184fdf34a2eac..9c63daad9de90 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -319,7 +319,7 @@ end """ chomp(s::AbstractString)::SubString -Remove a single trailing newline from a string. +Remove a single trailing newline (i.e. "\\r\\n" or "\\n") from a string. See also [`chop`](@ref). @@ -327,6 +327,12 @@ See also [`chop`](@ref). ```jldoctest julia> chomp("Hello\\n") "Hello" + +julia> chomp("World\\r\\n") +"World" + +julia> chomp("Julia\\r\\n\\n") +"Julia\\r\\n" ``` """ function chomp(s::AbstractString) @@ -336,17 +342,22 @@ function chomp(s::AbstractString) (j < 1 || s[j] != '\r') && (return SubString(s, 1, j)) return SubString(s, 1, prevind(s,j)) end -function chomp(s::String) - i = lastindex(s) - if i < 1 || codeunit(s,i) != 0x0a - return @inbounds SubString(s, 1, i) - elseif i < 2 || codeunit(s,i-1) != 0x0d - return @inbounds SubString(s, 1, prevind(s, i)) + +@assume_effects :removable :foldable function chomp(s::Union{String, SubString{String}}) + cu = codeunits(s) + ncu = length(cu) + len = if iszero(ncu) + 0 else - return @inbounds SubString(s, 1, prevind(s, i-1)) + has_lf = @inbounds(cu[ncu]) == 0x0a + two_bytes = ncu > 1 + has_cr = has_lf & two_bytes & (@inbounds(cu[ncu - two_bytes]) == 0x0d) + ncu - (has_lf + has_cr) end + off = s isa String ? 0 : s.offset + par = s isa String ? s : s.string + @inbounds @inline SubString{String}(par, off, len, Val{:noshift}()) end - """ lstrip([pred=isspace,] str::AbstractString)::SubString lstrip(str::AbstractString, chars)::SubString From a330d546217537eb45987bf2ea79d093284baaf9 Mon Sep 17 00:00:00 2001 From: matthias314 <56549971+matthias314@users.noreply.github.com> Date: Wed, 7 May 2025 20:02:46 -0400 Subject: [PATCH 221/662] treat empty `AbstractVector` like empty `Vector` in `print_array` (#57898) I don't see why an empty `Vector` should be displayed differently from any other empty `AbstractVector`. Here is an example using views: `Vector` (both master and this PR): ``` julia> [collect(Int8, 1:k) for k in (0,2)] 2-element Vector{Vector{Int8}}: [] [1, 2] julia> Any[collect(Int8, 1:k) for k in (0,2)] 2-element Vector{Any}: Int8[] Int8[1, 2] ``` `AbstractVector` in master: ``` julia> v = Int8[1,2]; [view(v, 1:k) for k in (0,2)] 2-element Vector{SubArray{Int8, 1, Vector{Int8}, Tuple{UnitRange{Int64}}, true}}: 0-element view(::Vector{Int8}, 1:0) with eltype Int8 [1, 2] julia> v = Int8[1,2]; Any[view(v, 1:k) for k in (0,2)] 2-element Vector{Any}: 0-element view(::Vector{Int8}, 1:0) with eltype Int8 Int8[1, 2] ``` `AbstractVector` with this PR: ``` julia> v = Int8[1,2]; [view(v, 1:k) for k in (0,2)] 2-element Vector{SubArray{Int8, 1, Vector{Int8}, Tuple{UnitRange{Int64}}, true}}: [] [1, 2] julia> v = Int8[1,2]; Any[view(v, 1:k) for k in (0,2)] 2-element Vector{Any}: Int8[] Int8[1, 2] ``` --- base/arrayshow.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/arrayshow.jl b/base/arrayshow.jl index 31e95e6659657..f792d26d8e9b5 100644 --- a/base/arrayshow.jl +++ b/base/arrayshow.jl @@ -361,7 +361,7 @@ print_array(io::IO, X::AbstractArray) = show_nd(io, X, print_matrix, true) # typeinfo aware # implements: show(io::IO, ::MIME"text/plain", X::AbstractArray) function show(io::IO, ::MIME"text/plain", X::AbstractArray) - if isempty(X) && (get(io, :compact, false)::Bool || X isa Vector) + if isempty(X) && (get(io, :compact, false)::Bool || X isa AbstractVector) return show(io, X) end # 1) show summary before setting :compact From 9c8710b135b1e3fb22f68c79643b77809c213213 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 8 May 2025 02:20:59 +0200 Subject: [PATCH 222/662] delete redundant `IteratorSize` method for `Slices` (#58050) Already covered as an `AbstractArray` subtype. --- base/slicearray.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/slicearray.jl b/base/slicearray.jl index 7318181c1e826..65d3b6c4d75fd 100644 --- a/base/slicearray.jl +++ b/base/slicearray.jl @@ -225,7 +225,6 @@ constructed by [`eachcol`](@ref). const ColumnSlices{P<:AbstractMatrix,AX,S<:AbstractVector} = Slices{P,Tuple{Colon,Int},AX,S,1} -IteratorSize(::Type{Slices{P,SM,AX,S,N}}) where {P,SM,AX,S,N} = HasShape{N}() axes(s::Slices) = s.axes size(s::Slices) = map(length, s.axes) From 963eaa744c211e5a5500c860f6fc745a3cf6c098 Mon Sep 17 00:00:00 2001 From: GHTaarn <62629455+GHTaarn@users.noreply.github.com> Date: Thu, 8 May 2025 02:28:37 +0200 Subject: [PATCH 223/662] Base.get_extension & Dates.format made public (#58108) `Base.get_extension` and `Dates.format` both appear in the manual and should therefore be `public` symbols according to [51335](https://github.com/JuliaLang/julia/issues/51335#issuecomment-1743091402). They appear in the manual [here](https://docs.julialang.org/en/v1/base/base/#Base.get_extension) and [here](https://docs.julialang.org/en/v1/stdlib/Dates/#Dates.format-Tuple%7BTimeType,%20AbstractString%7D). Please also consider back porting this to version 1.12 --- base/public.jl | 1 + stdlib/Dates/src/Dates.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/public.jl b/base/public.jl index ca2cf0b146e0f..380eea344a3a8 100644 --- a/base/public.jl +++ b/base/public.jl @@ -54,6 +54,7 @@ public active_project, # Reflection and introspection + get_extension, isambiguous, isexpr, isidentifier, diff --git a/stdlib/Dates/src/Dates.jl b/stdlib/Dates/src/Dates.jl index a4600a5f82043..0e6d0d0ef6986 100644 --- a/stdlib/Dates/src/Dates.jl +++ b/stdlib/Dates/src/Dates.jl @@ -81,4 +81,6 @@ export Period, DatePeriod, TimePeriod, # io.jl ISODateTimeFormat, ISODateFormat, ISOTimeFormat, DateFormat, RFC1123Format, @dateformat_str +public format + end # module From 998fff5b4fd528b44d99edde24dc48c0c8f9751d Mon Sep 17 00:00:00 2001 From: Arno Strouwen Date: Thu, 8 May 2025 02:35:54 +0200 Subject: [PATCH 224/662] Add default for inline command line switch (#58230) Fixes small discrepancy between man page and documentation. --- doc/src/manual/command-line-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 5a4c710faa22f..6cbb215a2ded5 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -195,7 +195,7 @@ The following is a complete list of command-line switches available when launchi |`-O`, `--optimize={0\|1\|2*\|3}` |Set the optimization level (level is 3 if `-O` is used without a level) ($)| |`--min-optlevel={0*\|1\|2\|3}` |Set the lower bound on per-module optimization| |`-g`, `--debug-info={0\|1*\|2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level) ($)| -|`--inline={yes\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| +|`--inline={yes*\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| |`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations ($)| |`--math-mode={ieee\|user*}` |Always follow `ieee` floating point semantics or respect `@fastmath` declarations| |`--polly={yes*\|no}` |Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)| From 9b63fd91b8ca5ff0c96ee99782a7c1a0fd448987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Mart=C3=ADnez?= <31046348+lmiq@users.noreply.github.com> Date: Thu, 8 May 2025 06:26:20 -0300 Subject: [PATCH 225/662] improve docs of replace(::String) with multiple replacements (#57683) addresses: https://github.com/JuliaLang/julia/issues/57669#issuecomment-2706554204 --- base/strings/util.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/strings/util.jl b/base/strings/util.jl index 9c63daad9de90..b8a2f7882c252 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -1073,8 +1073,11 @@ is supplied, the transformed string is instead written to `io` (returning `io`). (For example, this can be used in conjunction with an [`IOBuffer`](@ref) to re-use a pre-allocated buffer array in-place.) -Multiple patterns can be specified, and they will be applied left-to-right -simultaneously, so only one pattern will be applied to any character, and the +Multiple patterns can be specified: The input string will be scanned only once +from start (left) to end (right), and the first matching replacement +will be applied to each substring. Replacements are applied in the order of +the arguments provided if they match substrings starting at the same +input string position. Thus, only one pattern will be applied to any character, and the patterns will only be applied to the input text, not the replacements. !!! compat "Julia 1.7" From 492d10ae456b883640211f721d265726603afe84 Mon Sep 17 00:00:00 2001 From: Alex Pan <101936321+alexp616@users.noreply.github.com> Date: Thu, 8 May 2025 06:36:16 -0700 Subject: [PATCH 226/662] Fix incorrect result for Base.invmod(<:Signed, <:Unsigned) (#58010) MWE of bug: ```julia julia> Int(invmod(1024, UInt(12289))) 5652 julia> invmod(1024, 12289) 12277 ``` --------- Co-authored-by: Max Horn Co-authored-by: Lilith Orion Hafner --- base/intfuncs.jl | 2 +- test/intfuncs.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 1c73698658aa9..4116f1631b724 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -286,7 +286,7 @@ function invmod(n::Integer, m::Integer) g, x, y = gcdx(n, m) g != 1 && throw(DomainError((n, m), LazyString("Greatest common divisor is ", g, "."))) # Note that m might be negative here. - if n isa Unsigned && hastypemax(typeof(n)) && x > typemax(n)>>1 + if x isa Unsigned && hastypemax(typeof(x)) && x > typemax(x)>>1 # x might have wrapped if it would have been negative # adding back m forces a correction x += m diff --git a/test/intfuncs.jl b/test/intfuncs.jl index a62092ad6b849..405d6f4644bb0 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -258,6 +258,13 @@ end @test invmod(T(3), T(124))::T == 83 end + for T in (Int8, Int16, Int32, Int64, Int128) + @test invmod(T(3), unsigned(T)(124)) == 83 + end + + # Verify issue described in PR 58010 is fixed + @test invmod(UInt8(3), UInt16(50000)) === 0x411b + for T in (Int8, UInt8) for x in typemin(T):typemax(T) for m in typemin(T):typemax(T) From 3f96528b0f0bf875f5bafb4e16cb292ee97c9a27 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 8 May 2025 13:34:35 -0400 Subject: [PATCH 227/662] CODEOWNERS: Set `@JuliaLang/committers-available-to-review` as the global (default) codeowners for the `JuliaLang/julia` repo (#58315) --- .github/CODEOWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bf1380f5a07bc..13f0180b3fa1a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,13 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. +# Unless a later match takes precedence, +# a member of the `@JuliaLang/committers-available-to-review` team +# will be requested for review when someone opens a pull request. +* @JuliaLang/committers-available-to-review + CODEOWNERS @JuliaLang/github-actions /.github/ @JuliaLang/github-actions /.buildkite/ @JuliaLang/github-actions From 366a2246b243c5b831bdcded3a046a86d1559f8f Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Mon, 14 Apr 2025 17:34:28 -0400 Subject: [PATCH 228/662] typeinfer: De-duplicate backedges as they are stored Since the caller CodeInstance is always part of the identity that we are de-duplicating on, this makes the linear scan much faster than it is in `gf.c` --- Compiler/src/typeinfer.jl | 82 ++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 6eb4b140fd16b..0fa924e46c21d 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -687,12 +687,17 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: nothing end -# record the backedges -function store_backedges(caller::CodeInstance, edges::SimpleVector) - isa(caller.def.def, Method) || return # don't add backedges to toplevel method instance - i = 1 - while true - i > length(edges) && return nothing +# Iterate a series of back-edges that need registering, based on the provided forward edge list. +# Back-edges are returned as (invokesig, item), where the item is a Binding, MethodInstance, or +# MethodTable. +struct ForwardToBackedgeIterator + forward_edges::SimpleVector +end + +function Base.iterate(it::ForwardToBackedgeIterator, i::Int = 1) + edges = it.forward_edges + i > length(edges) && return nothing + while i ≤ length(edges) item = edges[i] if item isa Int i += 2 @@ -702,34 +707,55 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) i += 1 continue elseif isa(item, Core.Binding) - i += 1 - maybe_add_binding_backedge!(item, caller) - continue + return ((nothing, item), i + 1) end if isa(item, CodeInstance) - item = item.def - end - if isa(item, MethodInstance) # regular dispatch - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, nothing, caller) - i += 1 + item = get_ci_mi(item) + return ((nothing, item), i + 1) + elseif isa(item, MethodInstance) # regular dispatch + return ((nothing, item), i + 1) else + invokesig = item callee = edges[i+1] - if isa(callee, MethodTable) # abstract dispatch (legacy style edges) - ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) - i += 2 - continue - elseif isa(callee, Method) - # ignore `Method`-edges (from e.g. failed `abstract_call_method`) - i += 2 - continue - # `invoke` edge - elseif isa(callee, CodeInstance) - callee = get_ci_mi(callee) + isa(callee, Method) && (i += 2; continue) # ignore `Method`-edges (from e.g. failed `abstract_call_method`) + if isa(callee, MethodTable) + # abstract dispatch (legacy style edges) + return ((invokesig, callee), i + 2) else - callee = callee::MethodInstance + # `invoke` edge + callee = isa(callee, CodeInstance) ? get_ci_mi(callee) : callee::MethodInstance + return ((invokesig, callee), i + 2) + end + end + end + return nothing +end + +# record the backedges +function store_backedges(caller::CodeInstance, edges::SimpleVector) + isa(caller.def.def, Method) || return # don't add backedges to toplevel method instance + + backedges = ForwardToBackedgeIterator(edges) + for (i, (invokesig, item)) in enumerate(backedges) + # check for any duplicate edges we've already registered + duplicate_found = false + for (i′, (invokesig′, item′)) in enumerate(backedges) + i == i′ && break + if item′ === item && invokesig′ == invokesig + duplicate_found = true + break + end + end + + if !duplicate_found + if item isa Core.Binding + maybe_add_binding_backedge!(item, caller) + elseif item isa MethodTable + ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) + else + item::MethodInstance + ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) end - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) - i += 2 end end nothing From 5c66152aec65f0a33f105be220675290f0ded383 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Mon, 14 Apr 2025 17:37:23 -0400 Subject: [PATCH 229/662] staticdata: do not serialize backedges for `MethodInstance` / `MethodTable` These are restored in their entirety by staticdata.jl, so there's no need to serialize them. Dropping them has the additional advantage of making it unnecessary to de-duplicate edges in `gf.c` --- src/staticdata.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/staticdata.c b/src/staticdata.c index c75b9cdf94f2b..211ba09ac72d7 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -785,6 +785,12 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } + if (s->incremental && jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t *)v; + // Any back-edges will be re-validated and added by staticdata.jl, so + // drop them from the image here + record_field_change((jl_value_t**)&mt->backedges, NULL); + } if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; if (s->incremental) { @@ -800,12 +806,14 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ // we only need 3 specific fields of this (the rest are restored afterward, if valid) // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, // so must not be present here - record_field_change((jl_value_t**)&mi->backedges, NULL); record_field_change((jl_value_t**)&mi->cache, NULL); } else { assert(!needs_recaching(v, s->query_cache)); } + // Any back-edges will be re-validated and added by staticdata.jl, so + // drop them from the image here + record_field_change((jl_value_t**)&mi->backedges, NULL); // n.b. opaque closures cannot be inspected and relied upon like a // normal method since they can get improperly introduced by generated // functions, so if they appeared at all, we will probably serialize From bf725f1fb487ebbbda259b04da282ad04b151031 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Mon, 14 Apr 2025 17:39:20 -0400 Subject: [PATCH 230/662] gf.c: make edge de-duplication a caller responsibility On my system, this saves ~500 ms when loading CairoMakie (and all dependent packages) --- src/gf.c | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/gf.c b/src/gf.c index b1552fa50d0d9..483a18d13682c 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1997,7 +1997,6 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, assert(invokesig == NULL || jl_is_type(invokesig)); JL_LOCK(&callee->def.method->writelock); if (jl_atomic_load_relaxed(&allow_new_worlds)) { - int found = 0; jl_array_t *backedges = jl_mi_get_backedges(callee); // TODO: use jl_cache_type_(invokesig) like cache_method does to save memory if (!backedges) { @@ -2006,25 +2005,7 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, callee->backedges = backedges; jl_gc_wb(callee, backedges); } - else { - size_t i = 0, l = jl_array_nrows(backedges); - for (i = 0; i < l; i++) { - // optimized version of while (i < l) i = get_next_edge(callee->backedges, i, &invokeTypes, &mi); - jl_value_t *ciedge = jl_array_ptr_ref(backedges, i); - if (ciedge != (jl_value_t*)caller) - continue; - jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(backedges, i - 1) : NULL; - if (invokeTypes && jl_is_code_instance(invokeTypes)) - invokeTypes = NULL; - if ((invokesig == NULL && invokeTypes == NULL) || - (invokesig && invokeTypes && jl_types_equal(invokesig, invokeTypes))) { - found = 1; - break; - } - } - } - if (!found) - push_edge(backedges, invokesig, caller); + push_edge(backedges, invokesig, caller); } JL_UNLOCK(&callee->def.method->writelock); } @@ -2047,14 +2028,6 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t else { // check if the edge is already present and avoid adding a duplicate size_t i, l = jl_array_nrows(mt->backedges); - for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) == (jl_value_t*)caller) { - if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { - JL_UNLOCK(&mt->writelock); - return; - } - } - } // reuse an already cached instance of this type, if possible // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? for (i = 1; i < l; i += 2) { From 75255683192e98b07d22cf0cd4e7242401ba84fb Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 8 May 2025 11:27:40 -0400 Subject: [PATCH 231/662] Update back-edge de-duplication test --- Compiler/test/invalidation.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index c0925694d332e..b51411db1da05 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -177,25 +177,24 @@ begin end # Verify that adding the backedge again does not actually add a new backedge - let mi1 = Base.method_instance(deduped_caller1, (Int,)), - mi2 = Base.method_instance(deduped_caller2, (Int,)), - ci1 = mi1.cache - ci2 = mi2.cache + let mi = Base.method_instance(deduped_caller1, (Int,)), + ci = mi.cache callee_mi = Base.method_instance(deduped_callee, (Int,)) # Inference should have added the callers to the callee's backedges - @test ci1 in callee_mi.backedges - @test ci2 in callee_mi.backedges + @test ci in callee_mi.backedges + # In practice, inference will never end up calling `store_backedges` + # twice on the same CodeInstance like this - we only need to check + # that de-duplication works for a single invocation N = length(callee_mi.backedges) - Core.Compiler.store_backedges(ci1, Core.svec(callee_mi)) - Core.Compiler.store_backedges(ci2, Core.svec(callee_mi)) + Core.Compiler.store_backedges(ci, Core.svec(callee_mi, callee_mi)) N′ = length(callee_mi.backedges) - # The number of backedges should not be affected by an additional store, - # since de-duplication should have noticed the edge is already tracked - @test N == N′ + # A single `store_backedges` invocation should de-duplicate any of the + # edges it is adding. + @test N′ - N == 1 end end From 6ded6aa1ec10c3396459049930e98950c7cc7713 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Thu, 8 May 2025 19:21:48 +0100 Subject: [PATCH 232/662] Link against libstdc++ from CSL on windows (#58151) Fix #57021. Split out from #58142, not sure if there is a better way to fix this. --------- Co-authored-by: Jameson Nash --- deps/csl.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/csl.mk b/deps/csl.mk index fef950aa41621..86eb50966fa4f 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -120,6 +120,7 @@ ifeq ($(OS),WINNT) GCC_VERSION = 14 install-csl: mkdir -p $(build_private_libdir)/ + cp -a $(build_shlibdir)/$(call versioned_libname,libstdc++,6) $(build_shlibdir)/libstdc++.$(SHLIB_EXT) cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc_s.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmsvcrt.a $(build_private_libdir)/ @@ -130,6 +131,7 @@ endif ifeq ($(OS),WINNT) uninstall-csl: uninstall-gcc-libraries uninstall-gcc-libraries: + -rm -f $(build_shlibdir)/libstdc++.$(SHLIB_EXT) -rm -f $(build_private_libdir)/libgcc_s.a -rm -f $(build_private_libdir)/libgcc.a -rm -f $(build_private_libdir)/libmsvcrt.a From fa6f79866e69c869b6376e66a6516edbd4d6f69d Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Thu, 8 May 2025 13:25:07 -0500 Subject: [PATCH 233/662] Revert "CODEOWNERS: Set `@JuliaLang/committers-available-to-review` as the global (default) codeowners for the `JuliaLang/julia` repo" (#58358) --- .github/CODEOWNERS | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 13f0180b3fa1a..bf1380f5a07bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,13 +1,3 @@ -# This is a comment. -# Each line is a file pattern followed by one or more owners. - -# These owners will be the default owners for everything in -# the repo. -# Unless a later match takes precedence, -# a member of the `@JuliaLang/committers-available-to-review` team -# will be requested for review when someone opens a pull request. -* @JuliaLang/committers-available-to-review - CODEOWNERS @JuliaLang/github-actions /.github/ @JuliaLang/github-actions /.buildkite/ @JuliaLang/github-actions From 9986d97b879f77941f32f39be6fe785c2acd9c95 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 8 May 2025 15:27:52 -0300 Subject: [PATCH 234/662] Fix late gc lowering pass for vector intrinsics (#55864) Fixes #55844 --- src/llvm-late-gc-lowering.cpp | 50 +++++++++++++++++++++++++++++--- test/llvmpasses/image-codegen.jl | 2 +- test/llvmpasses/late-lower-gc.ll | 15 ++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 378466ad36ef1..4ee7e4b30e61c 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1,6 +1,8 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "llvm-gc-interface-passes.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Casting.h" #define DEBUG_TYPE "late_lower_gcroot" @@ -171,12 +173,12 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac (void)LI; break; } - else if (auto II = dyn_cast(CurrentV)) { - // Some intrinsics behave like LoadInst followed by a SelectInst - // This should never happen in a derived addrspace (since those cannot be stored to memory) - // so we don't need to lift these operations, but we do need to check if it's loaded and continue walking the base pointer + else if (auto *II = dyn_cast(CurrentV)) { if (II->getIntrinsicID() == Intrinsic::masked_load || II->getIntrinsicID() == Intrinsic::masked_gather) { + // Some intrinsics behave like LoadInst followed by a SelectInst + // This should never happen in a derived addrspace (since those cannot be stored to memory) + // so we don't need to lift these operations, but we do need to check if it's loaded and continue walking the base pointer if (auto VTy = dyn_cast(II->getType())) { if (hasLoadedTy(VTy->getElementType())) { Value *Mask = II->getOperand(2); @@ -205,6 +207,24 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac // In general a load terminates a walk break; } + else if (II->getIntrinsicID() == Intrinsic::vector_extract) { + if (auto VTy = dyn_cast(II->getType())) { + if (hasLoadedTy(VTy->getElementType())) { + Value *Idx = II->getOperand(1); + if (!isa(Idx)) { + assert(isa(Idx) && "unimplemented"); + (void)Idx; + } + CurrentV = II->getOperand(0); + fld_idx = -1; + continue; + } + } + break; + } else { + // Unknown Intrinsic + break; + } } else if (auto CI = dyn_cast(CurrentV)) { auto callee = CI->getCalledFunction(); @@ -212,9 +232,11 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac CurrentV = CI->getArgOperand(0); continue; } + // Unknown Call break; } else { + // Unknown Instruction break; } } @@ -530,6 +552,22 @@ SmallVector LateLowerGCFrame::NumberAllBase(State &S, Value *CurrentV) { Numbers = NumberAll(S, IEI->getOperand(0)); int ElNumber = Number(S, IEI->getOperand(1)); Numbers[idx] = ElNumber; + // C++17 + // } else if (auto *II = dyn_cast(CurrentV); II && II->getIntrinsicID() == Intrinsic::vector_insert) { + } else if (isa(CurrentV) && cast(CurrentV)->getIntrinsicID() == Intrinsic::vector_insert) { + auto *II = dyn_cast(CurrentV); + // Vector insert is a bit like a shuffle so use the same approach + SmallVector Numbers1 = NumberAll(S, II->getOperand(0)); + SmallVector Numbers2 = NumberAll(S, II->getOperand(1)); + unsigned first_idx = cast(II->getOperand(2))->getZExtValue(); + for (unsigned i = 0; i < Numbers1.size(); ++i) { + if (i < first_idx) + Numbers.push_back(Numbers1[i]); + else if (i - first_idx < Numbers2.size()) + Numbers.push_back(Numbers2[i - first_idx]); + else + Numbers.push_back(Numbers1[i]); + } } else if (auto *IVI = dyn_cast(CurrentV)) { Numbers = NumberAll(S, IVI->getAggregateOperand()); auto Tracked = TrackCompositeType(IVI->getType()); @@ -1206,6 +1244,10 @@ State LateLowerGCFrame::LocalScan(Function &F) { } } } + if (II->getIntrinsicID() == Intrinsic::vector_extract || II->getIntrinsicID() == Intrinsic::vector_insert) { + // These are not real defs + continue; + } } auto callee = CI->getCalledFunction(); if (callee && callee == typeof_func) { diff --git a/test/llvmpasses/image-codegen.jl b/test/llvmpasses/image-codegen.jl index 35e5add2de601..d594c02a4392e 100644 --- a/test/llvmpasses/image-codegen.jl +++ b/test/llvmpasses/image-codegen.jl @@ -2,7 +2,7 @@ # RUN: export JULIA_LLVM_ARGS="--print-before=loop-vectorize --print-module-scope" # RUN: rm -rf %t # RUN: mkdir %t -# RUN: julia --image-codegen --startup-file=no %s 2> %t/output.txt +# RUN: julia --image-codegen -t1,0 --startup-file=no %s 2> %t/output.txt # RUN: FileCheck %s < %t/output.txt # COM: checks that global variables compiled in imaging codegen diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index 4739fa186ffc7..af71d3c8d2c75 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -164,6 +164,21 @@ define {} addrspace(10)* @gclift_switch({} addrspace(13)* addrspace(10)* %input, ret {} addrspace(10)* %ret } +; Shouldn't hang +define void @vector_insert(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) { +top: + %pgcstack = call {}*** @julia.get_pgcstack() + %2 = call <4 x {} addrspace(10)*> @llvm.vector.insert.v4p10.v2p10(<4 x {} addrspace(10)*> %0, <2 x {} addrspace(10)*> %1, i64 2) + ret void +} + +define void @vector_extract(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) { +top: + %pgcstack = call {}*** @julia.get_pgcstack() + %2 = call <2 x {} addrspace(10)*> @llvm.vector.extract.v2p10.v4p10(<4 x {} addrspace(10)* > %0, i64 2) + ret void +} + define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { %v2 = call {}*** @julia.get_pgcstack() %e0 = extractvalue [2 x {} addrspace(10)* addrspace(11)*] %ar, 0 From 62d33712da05dfcfaba0cb2e59dcb48e068d0df2 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Thu, 8 May 2025 21:15:32 -0700 Subject: [PATCH 235/662] [REPL] Fix ends_with_semicolon on adjoint operator with comment (#58364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes yet another ad-hoc lexer from the REPL and replaces it with JuliaSyntax. Fixes #46189, with tests taken from (and superseding) #57974. Co-authored-by: Páll Haraldsson --- stdlib/REPL/src/REPL.jl | 50 ++++++++-------------------------------- stdlib/REPL/test/repl.jl | 7 ++++++ 2 files changed, 16 insertions(+), 41 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 3d0135b200641..81272ac971d40 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -42,6 +42,7 @@ using Base.Meta, Sockets, StyledStrings using JuliaSyntaxHighlighting import InteractiveUtils import FileWatching +import Base.JuliaSyntax: kind, @K_str, @KSet_str, Tokenize.tokenize export AbstractREPL, @@ -1700,49 +1701,16 @@ answer_color(r::StreamREPL) = r.answer_color input_color(r::LineEditREPL) = r.envcolors ? Base.input_color() : r.input_color input_color(r::StreamREPL) = r.input_color -let matchend = Dict("\"" => r"\"", "\"\"\"" => r"\"\"\"", "'" => r"'", - "`" => r"`", "```" => r"```", "#" => r"$"m, "#=" => r"=#|#=") - global _rm_strings_and_comments - function _rm_strings_and_comments(code::Union{String,SubString{String}}) - buf = IOBuffer(sizehint = sizeof(code)) - pos = 1 - while true - i = findnext(r"\"(?!\"\")|\"\"\"|'|`(?!``)|```|#(?!=)|#=", code, pos) - isnothing(i) && break - match = SubString(code, i) - j = findnext(matchend[match]::Regex, code, nextind(code, last(i))) - if match == "#=" # possibly nested - nested = 1 - while j !== nothing - nested += SubString(code, j) == "#=" ? +1 : -1 - iszero(nested) && break - j = findnext(r"=#|#=", code, nextind(code, last(j))) - end - elseif match[1] != '#' # quote match: check non-escaped - while j !== nothing - notbackslash = findprev(!=('\\'), code, prevind(code, first(j)))::Int - isodd(first(j) - notbackslash) && break # not escaped - j = findnext(matchend[match]::Regex, code, nextind(code, first(j))) - end - end - isnothing(j) && break - if match[1] == '#' - print(buf, SubString(code, pos, prevind(code, first(i)))) - else - print(buf, SubString(code, pos, last(i)), ' ', SubString(code, j)) - end - pos = nextind(code, last(j)) - end - print(buf, SubString(code, pos, lastindex(code))) - return String(take!(buf)) - end -end - # heuristic function to decide if the presence of a semicolon # at the end of the expression was intended for suppressing output -ends_with_semicolon(code::AbstractString) = ends_with_semicolon(String(code)) -ends_with_semicolon(code::Union{String,SubString{String}}) = - contains(_rm_strings_and_comments(code), r";\s*$") +function ends_with_semicolon(code) + semi = false + for tok in tokenize(code) + kind(tok) in KSet"Whitespace NewlineWs Comment EndMarker" && continue + semi = kind(tok) == K";" + end + return semi +end function banner(io::IO = stdout; short = false) if Base.GIT_VERSION_INFO.tagged_commit diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 8dab038b01c50..b2ddbefd0c25a 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -983,6 +983,13 @@ let ends_with_semicolon = REPL.ends_with_semicolon @test ends_with_semicolon("f()= 1;") # the next result does not matter because this is not legal syntax @test_nowarn ends_with_semicolon("1; #=# 2") + + # #46189 - adjoint operator with comment + @test ends_with_semicolon("W';") == true + @test ends_with_semicolon("W'; # comment") + @test !ends_with_semicolon("W'") + @test !ends_with_semicolon("x'") + @test !ends_with_semicolon("'a'") end # PR #20794, TTYTerminal with other kinds of streams From 5af63ffaa3b3ea1169f206e5bbec379a1a095a07 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Fri, 9 May 2025 12:31:47 +0200 Subject: [PATCH 236/662] faster `rand(TaskLocalRNG(), 1:n)` by outlining `throw` (#58306) In #58089, this method took a small performance hit in some contexts. It turns out that by outlining the unlikely branch which throws on empty ranges, this hit can be recovered. In https://github.com/JuliaLang/julia/pull/50509#issuecomment-2798850590, a graph of the performance improvement of the "speed-up randperm by using our current rand(1:n)" was posted, but I realized it was only true when calls to `rand(1:n)` were prefixed by `@inline`; without `@inline` it was overall slower for `TaskLocalRNG()` for very big arrays (but still faster otherwise). An alternative to these `@inline` annotation is to outline `throw` like here, for equivalent benefits as `@inline` in that `randperm` PR. Assuming that PR is merged, this PR improves roughly performance by 2x for `TaskLocalRNG()` (no change for other RNGs): ![new-shuffle-outlinethrow](https://github.com/user-attachments/assets/8c0d4740-3bb4-4bcf-a49d-9af09426bec7) While at it, I outlined a bunch of other unliky throwing branches. After that, #50509 can probably be merged, finally! --- stdlib/Random/src/Random.jl | 4 +++- stdlib/Random/src/Xoshiro.jl | 2 +- stdlib/Random/src/generation.jl | 17 ++++++++++------- stdlib/Random/src/misc.jl | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 809a98dfa27e8..9e05475b48c20 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -15,7 +15,9 @@ using Base.GMP.MPZ using Base.GMP: Limb using SHA: SHA, SHA2_256_CTX, SHA2_512_CTX, SHA_CTX -using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing +using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing, + _throw_argerror + import Base: copymutable, copy, copy!, ==, hash, convert, rand, randn, show diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 63a027046c545..7d059261b69f5 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -232,7 +232,7 @@ rng_native_52(::TaskLocalRNG) = UInt64 # this variant of setstate! initializes the internal splitmix state, a.k.a. `s4` @inline function initstate!(x::Union{TaskLocalRNG, Xoshiro}, state) length(state) == 4 && eltype(state) == UInt64 || - throw(ArgumentError("initstate! expects a list of 4 `UInt64` values")) + _throw_argerror("initstate! expects a list of 4 `UInt64` values") s0, s1, s2, s3 = state setstate!(x, (s0, s1, s2, s3, 1s0 + 3s1 + 5s2 + 7s3)) end diff --git a/stdlib/Random/src/generation.jl b/stdlib/Random/src/generation.jl index 61d0dd74eaa5c..1b09c624193f8 100644 --- a/stdlib/Random/src/generation.jl +++ b/stdlib/Random/src/generation.jl @@ -57,7 +57,7 @@ Sampler(::Type{<:AbstractRNG}, I::FloatInterval{BigFloat}, ::Repetition) = SamplerBigFloat{typeof(I)}(precision(BigFloat)) function _rand!(rng::AbstractRNG, z::BigFloat, sp::SamplerBigFloat) - precision(z) == sp.prec || throw(ArgumentError("incompatible BigFloat precision")) + precision(z) == sp.prec || _throw_argerror("incompatible BigFloat precision") limbs = sp.limbs rand!(rng, limbs) @inbounds begin @@ -229,6 +229,9 @@ uint_sup(::Type{<:Base.BitInteger32}) = UInt32 uint_sup(::Type{<:Union{Int64,UInt64}}) = UInt64 uint_sup(::Type{<:Union{Int128,UInt128}}) = UInt128 +@noinline empty_collection_error() = throw(ArgumentError("collection must be non-empty")) + + #### Fast struct SamplerRangeFast{U<:BitUnsigned,T<:BitInteger} <: Sampler{T} @@ -242,7 +245,7 @@ SamplerRangeFast(r::AbstractUnitRange{T}) where T<:BitInteger = SamplerRangeFast(r, uint_sup(T)) function SamplerRangeFast(r::AbstractUnitRange{T}, ::Type{U}) where {T,U} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() m = (last(r) - first(r)) % unsigned(T) % U # % unsigned(T) to not propagate sign bit bw = (Base.top_set_bit(m)) % UInt # bit-width mask = ((1 % U) << bw) - (1 % U) @@ -316,7 +319,7 @@ SamplerRangeInt(r::AbstractUnitRange{T}) where T<:BitInteger = SamplerRangeInt(r, uint_sup(T)) function SamplerRangeInt(r::AbstractUnitRange{T}, ::Type{U}) where {T,U} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() a = first(r) m = (last(r) - first(r)) % unsigned(T) % U k = m + one(U) @@ -362,7 +365,7 @@ struct SamplerRangeNDL{U<:Unsigned,T} <: Sampler{T} end function SamplerRangeNDL(r::AbstractUnitRange{T}) where {T} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() a = first(r) U = uint_sup(T) s = (last(r) - first(r)) % unsigned(T) % U + one(U) # overflow ok @@ -405,7 +408,7 @@ end function SamplerBigInt(::Type{RNG}, r::AbstractUnitRange{BigInt}, N::Repetition=Val(Inf) ) where {RNG<:AbstractRNG} m = last(r) - first(r) - m.size < 0 && throw(ArgumentError("collection must be non-empty")) + m.size < 0 && empty_collection_error() nlimbs = Int(m.size) hm = nlimbs == 0 ? Limb(0) : GC.@preserve m unsafe_load(m.d, nlimbs) highsp = Sampler(RNG, Limb(0):hm, N) @@ -461,7 +464,7 @@ rand(rng::AbstractRNG, sp::SamplerSimple{<:AbstractArray,<:Sampler}) = ## random values from Dict function Sampler(::Type{RNG}, t::Dict, ::Repetition) where RNG<:AbstractRNG - isempty(t) && throw(ArgumentError("collection must be non-empty")) + isempty(t) && empty_collection_error() # we use Val(Inf) below as rand is called repeatedly internally # even for generating only one random value from t SamplerSimple(t, Sampler(RNG, LinearIndices(t.slots), Val(Inf))) @@ -490,7 +493,7 @@ rand(rng::AbstractRNG, sp::SamplerTag{<:Set,<:Sampler}) = rand(rng, sp.data).fir ## random values from BitSet function Sampler(RNG::Type{<:AbstractRNG}, t::BitSet, n::Repetition) - isempty(t) && throw(ArgumentError("collection must be non-empty")) + isempty(t) && empty_collection_error() SamplerSimple(t, Sampler(RNG, minimum(t):maximum(t), Val(Inf))) end diff --git a/stdlib/Random/src/misc.jl b/stdlib/Random/src/misc.jl index 908776383d45f..16c1962f235d6 100644 --- a/stdlib/Random/src/misc.jl +++ b/stdlib/Random/src/misc.jl @@ -97,7 +97,7 @@ end # size-m subset of A where m is fixed!) function randsubseq!(r::AbstractRNG, S::AbstractArray, A::AbstractArray, p::Real) require_one_based_indexing(S, A) - 0 <= p <= 1 || throw(ArgumentError("probability $p not in [0,1]")) + 0 <= p <= 1 || _throw_argerror(LazyString("probability ", p, " not in [0,1]")) n = length(A) p == 1 && return copyto!(resize!(S, n), A) empty!(S) From 0cb1adbccfecaa18eabd8ddf546f247130e8c340 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 9 May 2025 11:56:12 -0400 Subject: [PATCH 237/662] Use getsplit interface for InvokeCallInfo (#58328) This makes invoke inlining work for CallInfo wrappers, e.g. - https://github.com/CedarEDA/DAECompiler.jl/issues/23 - https://github.com/JuliaDebug/Cthulhu.jl/pull/640 I think this is probably the cleanest way to do it. Other ways might be to refactor InvokeCallInfo to wrap around a generic call info, but we already have this interface for callinfos, so might as well try to use it. --- Compiler/src/ssair/inlining.jl | 12 ++++++++---- Compiler/src/stmtinfo.jl | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index c7e052ed17218..ec3879e27357f 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -1159,14 +1159,18 @@ function is_builtin(𝕃ₒ::AbstractLattice, s::Signature) end function handle_invoke_call!(todo::Vector{Pair{Int,Any}}, - ir::IRCode, idx::Int, stmt::Expr, info::InvokeCallInfo, flag::UInt32, + ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info), flag::UInt32, sig::Signature, state::InliningState) - match = info.match + nspl = nsplit(info) + nspl == 0 && return nothing # e.g. InvokeCICallInfo + @assert nspl == 1 + mresult = getsplit(info, 1) + match = mresult.matches[1] if !match.fully_covers # TODO: We could union split out the signature check and continue on return nothing end - result = info.result + result = getresult(info, 1) if isa(result, ConcreteResult) item = concrete_result_item(result, info, state) elseif isa(result, SemiConcreteResult) @@ -1648,7 +1652,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, ModifyOpInfo) handle_modifyop!_call!(ir, idx, stmt, info, state) - elseif isa(info, InvokeCallInfo) + elseif sig.f === Core.invoke handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) handle_finalizer_call!(ir, idx, stmt, info, state) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 6a85bc6605d3f..d108c671301b9 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -278,6 +278,7 @@ struct InvokeCICallInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::InvokeCICallInfo) = add_inlining_edge!(edges, info.edge) +nsplit_impl(info::InvokeCICallInfo) = 0 """ info::InvokeCallInfo @@ -390,6 +391,11 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) nothing end +nsplit_impl(info::InvokeCallInfo) = 1 +getsplit_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; MethodLookupResult(Core.MethodMatch[info.match], + WorldRange(typemin(UInt), typemax(UInt)), false)) +getresult_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; info.result) + """ info::OpaqueClosureCallInfo From 24d2f4af851e6911fe2ea242a82cf2338d46afc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 9 May 2025 18:37:44 +0200 Subject: [PATCH 238/662] Defer global caching of `CodeInstance` to post-optimization step (#58343) This PR extracts the caching improvements from https://github.com/JuliaLang/julia/pull/56687, implemented by @aviatesk. It essentially defers global caching to the post-optimization step, giving a temporary cache to the optimizer instead of relying on the global cache. The issue with caching globally before optimization is that any errors occurring within the optimizer may leave a partially initialized `CodeInstance` in the cache, which was meant to be updated post-optimization. Exceptions should not be thrown in the native compilation pipeline, but abstract interpreters extending optimization routines will frequently encounter some during an iterative development (see https://github.com/CedarEDA/DAECompiler.jl/issues/25). An extra benefit from deferring global caching is that the optimizer can then more safely update `CodeInstance`s with new inference information, as is the intent of https://github.com/JuliaLang/julia/pull/56687. --------- Co-authored-by: Shuhei Kadowaki Co-authored-by: Cody Tapscott <84105208+topolarity@users.noreply.github.com> --- Compiler/src/optimize.jl | 50 +++++++++++++++++++++++----- Compiler/src/typeinfer.jl | 70 ++++++++++++++++++++++++--------------- 2 files changed, 84 insertions(+), 36 deletions(-) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index fdf97c447559d..3532b2043d76f 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -147,16 +147,46 @@ struct InliningState{Interp<:AbstractInterpreter} edges::Vector{Any} world::UInt interp::Interp + opt_cache::IdDict{MethodInstance,CodeInstance} end -function InliningState(sv::InferenceState, interp::AbstractInterpreter) - return InliningState(sv.edges, frame_world(sv), interp) +function InliningState(sv::InferenceState, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + return InliningState(sv.edges, frame_world(sv), interp, opt_cache) end -function InliningState(interp::AbstractInterpreter) - return InliningState(Any[], get_inference_world(interp), interp) +function InliningState(interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + return InliningState(Any[], get_inference_world(interp), interp, opt_cache) +end + +struct OptimizerCache{CodeCache} + wvc::WorldView{CodeCache} + owner + opt_cache::IdDict{MethodInstance,CodeInstance} + function OptimizerCache( + wvc::WorldView{CodeCache}, + @nospecialize(owner), + opt_cache::IdDict{MethodInstance,CodeInstance}) where CodeCache + new{CodeCache}(wvc, owner, opt_cache) + end +end +function get((; wvc, owner, opt_cache)::OptimizerCache, mi::MethodInstance, default) + if haskey(opt_cache, mi) + codeinst = opt_cache[mi] + @assert codeinst.min_world ≤ wvc.worlds.min_world && + wvc.worlds.max_world ≤ codeinst.max_world && + codeinst.owner === owner + @assert isdefined(codeinst, :inferred) && codeinst.inferred === nothing + return codeinst + end + return get(wvc, mi, default) end # get `code_cache(::AbstractInterpreter)` from `state::InliningState` -code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.world) +function code_cache(state::InliningState) + cache = WorldView(code_cache(state.interp), state.world) + owner = cache_owner(state.interp) + return OptimizerCache(cache, owner, state.opt_cache) +end mutable struct OptimizationResult ir::IRCode @@ -183,13 +213,15 @@ mutable struct OptimizationState{Interp<:AbstractInterpreter} bb_vartables::Vector{Union{Nothing,VarTable}} insert_coverage::Bool end -function OptimizationState(sv::InferenceState, interp::AbstractInterpreter) - inlining = InliningState(sv, interp) +function OptimizationState(sv::InferenceState, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + inlining = InliningState(sv, interp, opt_cache) return OptimizationState(sv.linfo, sv.src, nothing, sv.stmt_info, sv.mod, sv.sptypes, sv.slottypes, inlining, sv.cfg, sv.unreachable, sv.bb_vartables, sv.insert_coverage) end -function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractInterpreter) +function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) # prepare src for running optimization passes if it isn't already nssavalues = src.ssavaluetypes if nssavalues isa Int @@ -209,7 +241,7 @@ function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractIn mod = isa(def, Method) ? def.module : def # Allow using the global MI cache, but don't track edges. # This method is mostly used for unit testing the optimizer - inlining = InliningState(interp) + inlining = InliningState(interp, opt_cache) cfg = compute_basic_blocks(src.code) unreachable = BitSet() bb_vartables = Union{VarTable,Nothing}[] diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 0fa924e46c21d..9970235e5babc 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -110,6 +110,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation elseif isdefined(result, :ci) edges = result_edges(interp, caller) ci = result.ci + mi = result.linfo # if we aren't cached, we don't need this edge # but our caller might, so let's just make it anyways if last(result.valid_worlds) >= validation_world @@ -143,7 +144,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation resize!(inferred_result.slottypes::Vector{Any}, nslots) resize!(inferred_result.slotnames, nslots) end - inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result) + inferred_result = maybe_compress_codeinfo(interp, mi, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing # The global cache can only handle objects that codegen understands @@ -151,14 +152,19 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation end end if debuginfo === nothing - debuginfo = DebugInfo(result.linfo) + debuginfo = DebugInfo(mi) end + min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) + ipo_effects = encode_effects(result.ipo_effects) time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), - ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), + ci, inferred_result, const_flag, min_world, max_world, ipo_effects, result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) + if is_cached(caller) # CACHE_MODE_GLOBAL + cache_result!(interp, result, ci) + end engine_reject(interp, ci) codegen = codegen_cache(interp) if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) @@ -171,7 +177,6 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation # This is necessary to get decent bootstrapping performance # when compiling the compiler to inject everything eagerly # where codegen can start finding and using it right away - mi = result.linfo if mi.def isa Method && isa_compileable_sig(mi) && is_cached(caller) ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, uncompressed) end @@ -181,6 +186,11 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation return nothing end +function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) + mi = result.linfo + code_cache(interp)[mi] = ci +end + function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstance, src::CodeInfo) user_edges = src.edges edges = user_edges isa SimpleVector ? user_edges : user_edges === nothing ? Core.svec() : Core.svec(user_edges...) @@ -215,11 +225,13 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan end function finish_nocycle(::AbstractInterpreter, frame::InferenceState, time_before::UInt64) - finishinfer!(frame, frame.interp, frame.cycleid) + opt_cache = IdDict{MethodInstance,CodeInstance}() + finishinfer!(frame, frame.interp, frame.cycleid, opt_cache) opt = frame.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(frame.interp, opt, frame.result) end + empty!(opt_cache) validation_world = get_world_counter() finish!(frame.interp, frame, validation_world, time_before) if isdefined(frame.result, :ci) @@ -249,10 +261,11 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei cycle_valid_worlds = intersect(cycle_valid_worlds, caller.world.valid_worlds) cycle_valid_effects = merge_effects(cycle_valid_effects, caller.ipo_effects) end + opt_cache = IdDict{MethodInstance,CodeInstance}() for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) - finishinfer!(caller, caller.interp, cycleid) + finishinfer!(caller, caller.interp, cycleid, opt_cache) time_now = _time_ns() caller.time_self_ns += (time_now - time_before) time_before = time_now @@ -273,6 +286,7 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller.time_paused = UInt64(0) caller.time_caches = 0.0 end + empty!(opt_cache) cycletop = frames[cycleid]::InferenceState time_start = cycletop.time_start validation_world = get_world_counter() @@ -434,22 +448,6 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance return ci end -function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) - @assert isdefined(ci, :inferred) - # check if the existing linfo metadata is also sufficient to describe the current inference result - # to decide if it is worth caching this right now - mi = result.linfo - cache = WorldView(code_cache(interp), result.valid_worlds) - if haskey(cache, mi) - ci = cache[mi] - # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later - @assert isdefined(ci, :inferred) - return false - end - code_cache(interp)[mi] = ci - return true -end - function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) if typ isa LimitedAccuracy frames = sv.callstack::Vector{AbsIntState} @@ -579,7 +577,8 @@ const empty_edges = Core.svec() # inference completed on `me` # update the MethodInstance -function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid::Int) +function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid::Int, + opt_cache::IdDict{MethodInstance, CodeInstance}) # prepare to run optimization passes on fulltree @assert isempty(me.ip) # inspect whether our inference had a limited result accuracy, @@ -635,7 +634,7 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: # disable optimization if we've already obtained very accurate result !result_is_constabi(interp, result) if doopt - result.src = OptimizationState(me, interp) + result.src = OptimizationState(me, interp, opt_cache) else result.src = me.src # for reflection etc. end @@ -670,23 +669,40 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: rettype_const = nothing const_flags = 0x0 end + di = nothing edges = empty_edges # `edges` will be updated within `finish!` ci = result.ci + min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, - first(result.valid_worlds), last(result.valid_worlds), + min_world, max_world, encode_effects(result.ipo_effects), result.analysis_results, di, edges) if is_cached(me) # CACHE_MODE_GLOBAL - cached_result = cache_result!(me.interp, result, ci) - if !cached_result + already_cached = is_already_cached(me.interp, result, ci) + if already_cached me.cache_mode = CACHE_MODE_VOLATILE + else + opt_cache[result.linfo] = ci end end end nothing end +function is_already_cached(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) + # check if the existing linfo metadata is also sufficient to describe the current inference result + # to decide if it is worth caching this right now + mi = result.linfo + cache = WorldView(code_cache(interp), result.valid_worlds) + if haskey(cache, mi) + # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later + @assert isdefined(cache[mi], :inferred) + return true + end + return false +end + # Iterate a series of back-edges that need registering, based on the provided forward edge list. # Back-edges are returned as (invokesig, item), where the item is a Binding, MethodInstance, or # MethodTable. From edf76576b7bd75fa2262047c8ccce0ad72066bac Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Fri, 9 May 2025 14:39:32 -0300 Subject: [PATCH 239/662] Apply debug_compile_units hack also in remove-addrspaces (#58365) This fixes the issues seen in the LLVM 20 PR Co-authored-by: Zentrik --- src/llvm-remove-addrspaces.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/llvm-remove-addrspaces.cpp b/src/llvm-remove-addrspaces.cpp index 78ff70b12409b..9e99cd3fcb25f 100644 --- a/src/llvm-remove-addrspaces.cpp +++ b/src/llvm-remove-addrspaces.cpp @@ -334,7 +334,11 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) GV->setInitializer(nullptr); } - + // Same workaround as in CloneCtx::prepare_vmap to avoid LLVM bug when cloning + auto &MD = VMap.MD(); + for (auto cu: M.debug_compile_units()) { + MD[cu].reset(cu); + } // Similarly, copy over and rewrite function bodies for (Function *F : Functions) { Function *NF = cast(VMap[F]); @@ -414,6 +418,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) } } + return true; } From 1b16a39648f78b75bf942d6bc72449296a0bd432 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Fri, 9 May 2025 19:11:54 +0100 Subject: [PATCH 240/662] Fix some issues with llvm source build (#57792) - The build fails if any runtime is enabled as `LLVM_ENABLE_RUNTIMES` will start with `;` which it seems to interpret as a runtime whose name is the empty string. - When llvm links against libLLVM, this fails as some of the compilers on CI set a sysroot causing zlib to not be found, see https://github.com/llvm/llvm-project/issues/131119. - Also, fix freebsd build by compiling statically linked code with `-fPIC`. --- deps/llvm.mk | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 09dd4f187d611..8afa0580b4bf6 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -60,6 +60,10 @@ ifeq ($(BUILD_LLD), 1) LLVM_ENABLE_PROJECTS := $(LLVM_ENABLE_PROJECTS);lld endif +# Remove ; if it's the first character +ifneq ($(LLVM_ENABLE_RUNTIMES),) + LLVM_ENABLE_RUNTIMES := $(patsubst ;%,%,$(LLVM_ENABLE_RUNTIMES)) +endif LLVM_LIB_FILE := libLLVMCodeGen.a @@ -70,7 +74,7 @@ LLVM_EXPERIMENTAL_TARGETS := LLVM_CFLAGS := LLVM_CXXFLAGS := LLVM_CPPFLAGS := -LLVM_LDFLAGS := +LLVM_LDFLAGS := "-L$(build_shlibdir)" # hacky way to force zlib to be found when linking against libLLVM and sysroot is set LLVM_CMAKE := LLVM_CMAKE += -DLLVM_ENABLE_PROJECTS="$(LLVM_ENABLE_PROJECTS)" @@ -183,6 +187,11 @@ endif ifeq ($(fPIC),) LLVM_CMAKE += -DLLVM_ENABLE_PIC=OFF +else +ifeq ($(OS),FreeBSD) + # On FreeBSD, we must force even statically-linked code to have -fPIC + LLVM_CMAKE += -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE +endif endif LLVM_CMAKE += -DCMAKE_C_FLAGS="$(LLVM_CPPFLAGS) $(LLVM_CFLAGS)" \ From f07565fc65f3a0339d0fedbb2fbc2801d6cc853c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 9 May 2025 15:05:33 -0400 Subject: [PATCH 241/662] codegen: remove readonly from abstract type calling convention (#58356) Refs: #58070 --- Compiler/test/codegen.jl | 4 ++++ src/cgutils.cpp | 2 +- src/codegen.cpp | 39 ++++++++++++++++++++++----------------- src/julia.h | 2 +- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 45db9e73d5a3f..3ff9790e29cf8 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1033,3 +1033,7 @@ end const x57872 = "Hello" f57872() = (Core.isdefinedglobal(@__MODULE__, Base.compilerbarrier(:const, :x57872)), x57872) # Extra globalref here to force world age bounds @test f57872() == (true, "Hello") + +@noinline f_mutateany(@nospecialize x) = x[] = 1 +g_mutateany() = (y = Ref(0); f_mutateany(y); y[]) +@test g_mutateany() === 1 diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 5b6d3ab963e55..359aaf0772d49 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1479,6 +1479,7 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) return dyn_size; } } +*/ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) { @@ -1493,7 +1494,6 @@ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) mutabl = ctx.builder.CreateLShr(mutabl, 1); return ctx.builder.CreateTrunc(mutabl, getInt1Ty(ctx.builder.getContext())); } -*/ static Value *emit_datatype_isprimitivetype(jl_codectx_t &ctx, Value *typ) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 97f2b8b98f2db..28d3281584b85 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1600,7 +1600,7 @@ static MDNode *best_tbaa(jl_tbaacache_t &tbaa_cache, jl_value_t *jt) { // note that this includes jl_isbits, although codegen should work regardless static bool jl_is_concrete_immutable(jl_value_t* t) { - return jl_is_immutable_datatype(t) && ((jl_datatype_t*)t)->isconcretetype; + return jl_may_be_immutable_datatype(t) && ((jl_datatype_t*)t)->isconcretetype; } static bool jl_is_pointerfree(jl_value_t* t) @@ -7385,8 +7385,8 @@ static Function *gen_cfun_wrapper( inputarg = mark_julia_type(ctx, val, false, jargty); } } - else if (static_at || (!jl_is_typevar(jargty) && !jl_is_immutable_datatype(jargty))) { - // must be a jl_value_t* (because it's mutable or contains gc roots) + else if (static_at || (!jl_is_typevar(jargty) && (!jl_is_datatype(jargty) || jl_is_abstracttype(jargty) || jl_is_mutable_datatype(jargty)))) { + // must be a jl_value_t* (because it is mutable or abstract) inputarg = mark_julia_type(ctx, maybe_decay_untracked(ctx, val), true, jargty_proper); } else { @@ -7400,31 +7400,36 @@ static Function *gen_cfun_wrapper( emit_ptrgep(ctx, nestPtr, jl_array_nrows(*closure_types) * ctx.types().sizeof_ptr), Align(sizeof(void*))); BasicBlock *boxedBB = BasicBlock::Create(ctx.builder.getContext(), "isboxed", cw); - BasicBlock *loadBB = BasicBlock::Create(ctx.builder.getContext(), "need-load", cw); + BasicBlock *notanyBB = BasicBlock::Create(ctx.builder.getContext(), "not-any", cw); BasicBlock *unboxedBB = BasicBlock::Create(ctx.builder.getContext(), "maybe-unboxed", cw); BasicBlock *isanyBB = BasicBlock::Create(ctx.builder.getContext(), "any", cw); BasicBlock *afterBB = BasicBlock::Create(ctx.builder.getContext(), "after", cw); - Value *isrtboxed = ctx.builder.CreateIsNull(val); // XXX: this is the wrong condition and should be inspecting runtime_dt instead - ctx.builder.CreateCondBr(isrtboxed, boxedBB, loadBB); - ctx.builder.SetInsertPoint(boxedBB); - Value *p1 = val; - p1 = track_pjlvalue(ctx, p1); - ctx.builder.CreateBr(afterBB); - ctx.builder.SetInsertPoint(loadBB); Value *isrtany = ctx.builder.CreateICmpEQ( - literal_pointer_val(ctx, (jl_value_t*)jl_any_type), val); - ctx.builder.CreateCondBr(isrtany, isanyBB, unboxedBB); + track_pjlvalue(ctx,literal_pointer_val(ctx, (jl_value_t*)jl_any_type)), runtime_dt); + ctx.builder.CreateCondBr(isrtany, isanyBB, notanyBB); ctx.builder.SetInsertPoint(isanyBB); - Value *p2 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))); + Value *p1 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))); + ctx.builder.CreateBr(afterBB); + isanyBB = ctx.builder.GetInsertBlock(); // could have changed + ctx.builder.SetInsertPoint(notanyBB); + jl_cgval_t runtime_dt_val = mark_julia_type(ctx, runtime_dt, true, jl_any_type); + Value *isrtboxed = // (!jl_is_datatype(runtime_dt) || !jl_is_concrete_datatype(runtime_dt) || jl_is_mutable_datatype(runtime_dt)) + emit_guarded_test(ctx, emit_exactly_isa(ctx, runtime_dt_val, jl_datatype_type), true, [&] { + return ctx.builder.CreateOr(ctx.builder.CreateNot(emit_isconcrete(ctx, runtime_dt)), emit_datatype_mutabl(ctx, runtime_dt)); + }); + ctx.builder.CreateCondBr(isrtboxed, boxedBB, unboxedBB); + ctx.builder.SetInsertPoint(boxedBB); + Value *p2 = track_pjlvalue(ctx, val); ctx.builder.CreateBr(afterBB); + boxedBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(unboxedBB); Value *p3 = emit_new_bits(ctx, runtime_dt, val); unboxedBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(afterBB); PHINode *p = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 3); - p->addIncoming(p1, boxedBB); - p->addIncoming(p2, isanyBB); + p->addIncoming(p1, isanyBB); + p->addIncoming(p2, boxedBB); p->addIncoming(p3, unboxedBB); inputarg = mark_julia_type(ctx, p, true, jargty_proper); } @@ -7980,7 +7985,7 @@ static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module param.addAttribute(Attribute::ReadOnly); ty = PointerType::get(M->getContext(), AddressSpace::Derived); } - else if (isboxed && jl_is_immutable_datatype(jt)) { + else if (isboxed && jl_may_be_immutable_datatype(jt) && !jl_is_abstracttype(jt)) { param.addAttribute(Attribute::ReadOnly); } else if (jl_is_primitivetype(jt) && ty->isIntegerTy()) { diff --git a/src/julia.h b/src/julia.h index 95241aa6f48a8..08ae5d6eaed62 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1617,7 +1617,7 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_mutable(t) (((jl_datatype_t*)t)->name->mutabl) #define jl_is_mutable_datatype(t) (jl_is_datatype(t) && (((jl_datatype_t*)t)->name->mutabl)) #define jl_is_immutable(t) (!((jl_datatype_t*)t)->name->mutabl) -#define jl_is_immutable_datatype(t) (jl_is_datatype(t) && (!((jl_datatype_t*)t)->name->mutabl)) +#define jl_may_be_immutable_datatype(t) (jl_is_datatype(t) && (!((jl_datatype_t*)t)->name->mutabl)) #define jl_is_uniontype(v) jl_typetagis(v,jl_uniontype_tag<<4) #define jl_is_typevar(v) jl_typetagis(v,jl_tvar_tag<<4) #define jl_is_unionall(v) jl_typetagis(v,jl_unionall_tag<<4) From 75382d65926fbdf59ae5046888198fd16edcf287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Sat, 10 May 2025 00:19:03 +0200 Subject: [PATCH 242/662] Attach source edges to `:uninferred` generator `CodeInstance` (#58324) I am not sure when we may be in the situation where `edges` is `null`, but if we do, we crash. I triggered such a crash in https://github.com/CedarEDA/DAECompiler.jl/pull/24, which manually adds edges to handcrafted code instances. I may be doing something wrong, but in any case, it doesn't hurt to be more robust (and perhaps handle this situation more gracefully). --------- Co-authored-by: Jameson Nash --- src/toplevel.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/toplevel.c b/src/toplevel.c index 7fd35f15702a9..826b9195da059 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -517,11 +517,15 @@ int jl_needs_lowering(jl_value_t *e) JL_NOTSAFEPOINT JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_for_uninferred(jl_method_instance_t *mi, jl_code_info_t *src) { + jl_svec_t *edges = jl_emptysvec; + if (src->edges && jl_is_svec(src->edges)) + edges = (jl_svec_t*)src->edges; + // Do not compress this, we expect it to be shortlived. jl_code_instance_t *ci = jl_new_codeinst(mi, (jl_value_t*)jl_uninferred_sym, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, (jl_value_t*)src, 0, src->min_world, src->max_world, - 0, NULL, NULL, NULL); + 0, NULL, NULL, edges); return ci; } From 4a3d736de068a97bc57bf4a073f9d1c5d8c542c4 Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Sat, 10 May 2025 08:49:52 -0400 Subject: [PATCH 243/662] use rapidhash (#57509) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes https://github.com/JuliaLang/julia/issues/57235 todos: * are the test changes acceptable? * default seed going from `0` --> `0xbdd89aa982704029` * lots and lots of benchmarking * I don't really understand the effects stuff (for the `String` hash) * how configurable should the secret be? to address https://github.com/JuliaLang/julia/issues/37166 should the seed/secret be an env var? * proper attribution to creators of `rapidhash` and also to reference implementations * I changed some `h + seed` to `h ⊻ seed` only because I don't really get why `+` was used in the first place * should instances of `hash(x) - 3h` instead be `hash(x, h)` ? --------- Co-authored-by: Jeff Bezanson Co-authored-by: Oscar Smith Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> Co-authored-by: Ian Butterworth --- NEWS.md | 2 + base/abstractarray.jl | 2 +- base/binaryplatforms.jl | 2 +- base/char.jl | 2 +- base/gmp.jl | 4 +- base/hashing.jl | 193 +++++++++++++++++++++++++------------- base/multidimensional.jl | 2 +- base/pkgid.jl | 2 +- base/regex.jl | 2 +- base/stacktraces.jl | 2 +- base/strings/substring.jl | 6 +- base/tuple.jl | 4 +- base/version.jl | 2 +- stdlib/TOML/test/print.jl | 27 ++++-- test/arrayops.jl | 2 +- test/hashing.jl | 11 ++- test/show.jl | 6 +- test/tuple.jl | 8 +- 18 files changed, 178 insertions(+), 101 deletions(-) diff --git a/NEWS.md b/NEWS.md index 211c561167122..f74d00bb1fb7f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,6 +11,8 @@ Language changes * `mod(x::AbstractFloat, -Inf)` now returns `x` (as long as `x` is finite), this aligns with C standard and is considered a bug fix ([#47102]) + - The `hash` algorithm and its values have changed. Most `hash` specializations will remain correct and require no action. Types that reimplement the core hashing logic independently, such as some third-party string packages do, may require a migration to the new algorithm. ([#57509]) + Compiler/Runtime improvements ----------------------------- diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 1c1e7ebb5c3ee..bf2a6ebabecba 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3563,7 +3563,7 @@ sizehint!(a::AbstractVector, _) = a const hash_abstractarray_seed = UInt === UInt64 ? 0x7e2d6fb6448beb77 : 0xd4514ce5 function hash(A::AbstractArray, h::UInt) - h += hash_abstractarray_seed + h ⊻= hash_abstractarray_seed # Axes are themselves AbstractArrays, so hashing them directly would stack overflow # Instead hash the tuple of firsts and lasts along each dimension h = hash(map(first, axes(A)), h) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 51c65a6b310a6..86fd9118b6738 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -157,7 +157,7 @@ end # Hash definition to ensure that it's stable function Base.hash(p::Platform, h::UInt) - h += 0x506c6174666f726d % UInt + h ⊻= 0x506c6174666f726d % UInt h = hash(p.tags, h) h = hash(p.compare_strategies, h) return h diff --git a/base/char.jl b/base/char.jl index fc173bb3c3a44..22ffa977ca6ed 100644 --- a/base/char.jl +++ b/base/char.jl @@ -222,7 +222,7 @@ in(x::AbstractChar, y::AbstractChar) = x == y ==(x::Char, y::Char) = bitcast(UInt32, x) == bitcast(UInt32, y) isless(x::Char, y::Char) = bitcast(UInt32, x) < bitcast(UInt32, y) hash(x::Char, h::UInt) = - hash_uint64(((bitcast(UInt32, x) + UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) + hash_finalizer(((bitcast(UInt32, x) + UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) % UInt # fallbacks: isless(x::AbstractChar, y::AbstractChar) = isless(Char(x), Char(y)) diff --git a/base/gmp.jl b/base/gmp.jl index 1d7c4266c331c..a91226cebd737 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -892,7 +892,7 @@ if Limb === UInt64 === UInt return hash(ldexp(flipsign(Float64(limb), sz), pow), h) end h = hash_integer(pow, h) - h ⊻= hash_uint(flipsign(limb, sz) ⊻ h) + h ⊻= hash_finalizer(flipsign(limb, sz) ⊻ h) for idx = idx+1:asz if shift == 0 limb = unsafe_load(ptr, idx) @@ -906,7 +906,7 @@ if Limb === UInt64 === UInt limb = limb2 << upshift | limb1 >> shift end end - h ⊻= hash_uint(limb ⊻ h) + h ⊻= hash_finalizer(limb ⊻ h) end return h end diff --git a/base/hashing.jl b/base/hashing.jl index a2b00844ecaf5..c409d3ae7940f 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -1,6 +1,11 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## hashing a single value ## +const HASH_SEED = UInt == UInt64 ? 0xbdd89aa982704029 : 0xeabe9406 +const HASH_SECRET = tuple( + 0x2d358dccaa6c78a5, + 0x8bb84b93962eacc9, + 0x4b33a62ed433d4a3, +) """ hash(x[, h::UInt])::UInt @@ -17,75 +22,52 @@ The hash value may change when a new Julia process is started. ```jldoctest; filter = r"0x[0-9a-f]{16}" julia> a = hash(10) -0x95ea2955abd45275 +0x759d18cc5346a65f julia> hash(10, a) # only use the output of another hash function as the second argument -0xd42bad54a8575b16 +0x03158cd61b1b0bd1 ``` See also: [`objectid`](@ref), [`Dict`](@ref), [`Set`](@ref). """ -hash(x::Any) = hash(x, zero(UInt)) +hash(data::Any) = hash(data, HASH_SEED) hash(w::WeakRef, h::UInt) = hash(w.value, h) # Types can't be deleted, so marking as total allows the compiler to look up the hash -hash(T::Type, h::UInt) = hash_uint(3h - @assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T)) +hash(T::Type, h::UInt) = + hash((@assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T)), h) +hash(@nospecialize(data), h::UInt) = hash(objectid(data), h) -## hashing general objects ## - -hash(@nospecialize(x), h::UInt) = hash_uint(3h - objectid(x)) - -hash(x::Symbol) = objectid(x) - -## core data hashing functions ## - -function hash_64_64(n::UInt64) - a::UInt64 = n - a = ~a + a << 21 - a = a ⊻ a >> 24 - a = a + a << 3 + a << 8 - a = a ⊻ a >> 14 - a = a + a << 2 + a << 4 - a = a ⊻ a >> 28 - a = a + a << 31 - return a +function mul_parts(a::UInt64, b::UInt64) + p = widemul(a, b) + return (p >> 64) % UInt64, p % UInt64 end - -function hash_64_32(n::UInt64) - a::UInt64 = n - a = ~a + a << 18 - a = a ⊻ a >> 31 - a = a * 21 - a = a ⊻ a >> 11 - a = a + a << 6 - a = a ⊻ a >> 22 - return a % UInt32 +hash_mix(a::UInt64, b::UInt64) = ⊻(mul_parts(a, b)...) + +# faster-but-weaker than hash_mix intended for small keys +hash_mix_linear(x::UInt64, h::UInt) = 3h - x +function hash_finalizer(x::UInt64) + x ⊻= (x >> 32) + x *= 0x63652a4cd374b267 + x ⊻= (x >> 33) + return x end -function hash_32_32(n::UInt32) - a::UInt32 = n - a = a + 0x7ed55d16 + a << 12 - a = a ⊻ 0xc761c23c ⊻ a >> 19 - a = a + 0x165667b1 + a << 5 - a = a + 0xd3a2646c ⊻ a << 9 - a = a + 0xfd7046c5 + a << 3 - a = a ⊻ 0xb55a4f09 ⊻ a >> 16 - return a -end +hash_64_64(data::UInt64) = hash_finalizer(data) +hash_64_32(data::UInt64) = hash_64_64(data) % UInt32 +hash_32_32(data::UInt32) = hash_64_32(UInt64(data)) if UInt === UInt64 - hash_uint64(x::UInt64) = hash_64_64(x) - hash_uint(x::UInt) = hash_64_64(x) + const hash_uint64 = hash_64_64 + const hash_uint = hash_64_64 else - hash_uint64(x::UInt64) = hash_64_32(x) - hash_uint(x::UInt) = hash_32_32(x) + const hash_uint64 = hash_64_32 + const hash_uint = hash_32_32 end -## efficient value-based hashing of integers ## - -hash(x::Int64, h::UInt) = hash_uint64(bitcast(UInt64, x)) - 3h -hash(x::UInt64, h::UInt) = hash_uint64(x) - 3h -hash(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32}, h::UInt) = hash(Int64(x), h) +hash(x::UInt64, h::UInt) = hash_uint64(hash_mix_linear(x, h)) +hash(x::Int64, h::UInt) = hash(bitcast(UInt64, x), h) +hash(x::Union{Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32}, h::UInt) = hash(Int64(x), h) function hash_integer(n::Integer, h::UInt) h ⊻= hash_uint((n % UInt) ⊻ h) @@ -100,7 +82,7 @@ end ## efficient value-based hashing of floats ## -const hx_NaN = hash_uint64(reinterpret(UInt64, NaN)) +const hx_NaN = hash(reinterpret(UInt64, NaN)) function hash(x::Float64, h::UInt) # see comments on trunc and hash(Real, UInt) if typemin(Int64) <= x < typemax(Int64) @@ -116,7 +98,7 @@ function hash(x::Float64, h::UInt) elseif isnan(x) return hx_NaN ⊻ h # NaN does not have a stable bit pattern end - return hash_uint64(bitcast(UInt64, x)) - 3h + return hash(bitcast(UInt64, x), h) end hash(x::Float32, h::UInt) = hash(Float64(x), h) @@ -131,7 +113,7 @@ function hash(x::Float16, h::UInt) elseif isnan(x) return hx_NaN ⊻ h # NaN does not have a stable bit pattern end - return hash_uint64(bitcast(UInt64, Float64(x))) - 3h + return hash(bitcast(UInt64, Float64(x)), h) end ## generic hashing for rational values ## @@ -180,21 +162,100 @@ end ## symbol & expression hashing ## - if UInt === UInt64 - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h + 0x83c7900696d26dc6)) - hash(x::QuoteNode, h::UInt) = hash(x.value, h + 0x2c97bf8b3de87020) + hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x83c7900696d26dc6)) + hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x2c97bf8b3de87020) else - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h + 0x96d26dc6)) - hash(x::QuoteNode, h::UInt) = hash(x.value, h + 0x469d72af) + hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x469d72af)) + hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x469d72af) end -## hashing strings ## +hash(x::Symbol) = objectid(x) -const memhash = UInt === UInt64 ? :memhash_seed : :memhash32_seed -const memhash_seed = UInt === UInt64 ? 0x71e729fd56419c81 : 0x56419c81 -@assume_effects :total function hash(s::String, h::UInt) - h += memhash_seed - ccall(memhash, UInt, (Ptr{UInt8}, Csize_t, UInt32), s, sizeof(s), h % UInt32) + h +load_le(::Type{T}, ptr::Ptr{UInt8}, i) where {T <: Union{UInt32, UInt64}} = + unsafe_load(convert(Ptr{T}, ptr + i - 1)) + +function read_small(ptr::Ptr{UInt8}, n::Int) + return (UInt64(unsafe_load(ptr)) << 56) | + (UInt64(unsafe_load(ptr, div(n, 2) + 1)) << 32) | + UInt64(unsafe_load(ptr, n)) end + +@assume_effects :terminates_globally function hash_bytes( + ptr::Ptr{UInt8}, + n::Int, + seed::UInt64, + secret::NTuple{3, UInt64} + ) + # Adapted with gratitude from [rapidhash](https://github.com/Nicoshev/rapidhash) + buflen = UInt64(n) + seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + + a = zero(UInt64) + b = zero(UInt64) + + if buflen ≤ 16 + if buflen ≥ 4 + a = (UInt64(load_le(UInt32, ptr, 1)) << 32) | + UInt64(load_le(UInt32, ptr, n - 3)) + + delta = (buflen & 24) >>> (buflen >>> 3) + b = (UInt64(load_le(UInt32, ptr, delta + 1)) << 32) | + UInt64(load_le(UInt32, ptr, n - 3 - delta)) + elseif buflen > 0 + a = read_small(ptr, n) + end + else + pos = 1 + i = buflen + while i ≥ 48 + see1 = seed + see2 = seed + while i ≥ 48 + seed = hash_mix( + load_le(UInt64, ptr, pos) ⊻ secret[1], + load_le(UInt64, ptr, pos + 8) ⊻ seed + ) + see1 = hash_mix( + load_le(UInt64, ptr, pos + 16) ⊻ secret[2], + load_le(UInt64, ptr, pos + 24) ⊻ see1 + ) + see2 = hash_mix( + load_le(UInt64, ptr, pos + 32) ⊻ secret[3], + load_le(UInt64, ptr, pos + 40) ⊻ see2 + ) + pos += 48 + i -= 48 + end + seed = seed ⊻ see1 ⊻ see2 + end + if i > 16 + seed = hash_mix( + load_le(UInt64, ptr, pos) ⊻ secret[3], + load_le(UInt64, ptr, pos + 8) ⊻ seed ⊻ secret[2] + ) + if i > 32 + seed = hash_mix( + load_le(UInt64, ptr, pos + 16) ⊻ secret[3], + load_le(UInt64, ptr, pos + 24) ⊻ seed + ) + end + end + + a = load_le(UInt64, ptr, n - 15) + b = load_le(UInt64, ptr, n - 7) + end + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) +end + +@assume_effects :total hash(data::String, h::UInt) = + GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt + +# no longer used in Base, but a lot of packages access these internals +const memhash = UInt === UInt64 ? :memhash_seed : :memhash32_seed +const memhash_seed = UInt === UInt64 ? 0x71e729fd56419c81 : 0x56419c81 diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 1b2dd748bda97..7add7b9e74205 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -148,7 +148,7 @@ module IteratorsMD # hashing const cartindexhash_seed = UInt == UInt64 ? 0xd60ca92f8284b8b0 : 0xf2ea7c2e function Base.hash(ci::CartesianIndex, h::UInt) - h += cartindexhash_seed + h ⊻= cartindexhash_seed for i in ci.I h = hash(i, h) end diff --git a/base/pkgid.jl b/base/pkgid.jl index 8c776d79a69cb..577529bbe7f63 100644 --- a/base/pkgid.jl +++ b/base/pkgid.jl @@ -17,7 +17,7 @@ end ==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name function hash(pkg::PkgId, h::UInt) - h += 0xc9f248583a0ca36c % UInt + h ⊻= 0xc9f248583a0ca36c % UInt h = hash(pkg.uuid, h) h = hash(pkg.name, h) return h diff --git a/base/regex.jl b/base/regex.jl index baef3f9fdd197..691dbc94c5563 100644 --- a/base/regex.jl +++ b/base/regex.jl @@ -802,7 +802,7 @@ end ## hash ## const hashre_seed = UInt === UInt64 ? 0x67e195eb8555e72d : 0xe32373e4 function hash(r::Regex, h::UInt) - h += hashre_seed + h ⊻= hashre_seed h = hash(r.pattern, h) h = hash(r.compile_options, h) h = hash(r.match_options, h) diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 7ba23f6e715dc..9e454e430e2b6 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -90,7 +90,7 @@ function ==(a::StackFrame, b::StackFrame) end function hash(frame::StackFrame, h::UInt) - h += 0xf4fbda67fe20ce88 % UInt + h ⊻= 0xf4fbda67fe20ce88 % UInt h = hash(frame.line, h) h = hash(frame.file, h) h = hash(frame.func, h) diff --git a/base/strings/substring.jl b/base/strings/substring.jl index 148c860705dd3..860895207f444 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -135,10 +135,8 @@ end pointer(x::SubString{String}) = pointer(x.string) + x.offset pointer(x::SubString{String}, i::Integer) = pointer(x.string) + x.offset + (i-1) -function hash(s::SubString{String}, h::UInt) - h += memhash_seed - ccall(memhash, UInt, (Ptr{UInt8}, Csize_t, UInt32), s, sizeof(s), h % UInt32) + h -end +hash(data::SubString{String}, h::UInt) = + GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt _isannotated(::SubString{T}) where {T} = _isannotated(T) diff --git a/base/tuple.jl b/base/tuple.jl index 4982ef1b23eb0..5255f8ba07539 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -576,10 +576,10 @@ function _eq(t1::Any32, t2::Any32) end const tuplehash_seed = UInt === UInt64 ? 0x77cfa1eef01bca90 : 0xf01bca90 -hash(::Tuple{}, h::UInt) = h + tuplehash_seed +hash(::Tuple{}, h::UInt) = h ⊻ tuplehash_seed hash(t::Tuple, h::UInt) = hash(t[1], hash(tail(t), h)) function hash(t::Any32, h::UInt) - out = h + tuplehash_seed + out = h ⊻ tuplehash_seed for i = length(t):-1:1 out = hash(t[i], out) end diff --git a/base/version.jl b/base/version.jl index b362daa78f04f..71192916a5b22 100644 --- a/base/version.jl +++ b/base/version.jl @@ -218,7 +218,7 @@ function isless(a::VersionNumber, b::VersionNumber) end function hash(v::VersionNumber, h::UInt) - h += 0x8ff4ffdb75f9fede % UInt + h ⊻= 0x8ff4ffdb75f9fede % UInt h = hash(v.major, h) h = hash(v.minor, h) h = hash(v.patch, h) diff --git a/stdlib/TOML/test/print.jl b/stdlib/TOML/test/print.jl index e8a6431cb34a7..9734d96b3c8c1 100644 --- a/stdlib/TOML/test/print.jl +++ b/stdlib/TOML/test/print.jl @@ -58,7 +58,7 @@ end [option] """ d = TOML.parse(s) - @test toml_str(d) == "user = \"me\"\n\n[julia]\n\n[option]\n" + @test toml_str(d; sorted=true) == "user = \"me\"\n\n[julia]\n\n[option]\n" end @testset "special characters" begin @@ -83,17 +83,28 @@ loaders = ["gzip", { driver = "csv", args = {delim = "\t"}}] @testset "vec with dicts and non-dicts" begin # https://github.com/JuliaLang/julia/issues/45340 d = Dict("b" => Any[111, Dict("a" => 222, "d" => 333)]) - @test toml_str(d) == "b = [111, {a = 222, d = 333}]\n" + @test toml_str(d) == (sizeof(Int) == 8 ? + "b = [111, {a = 222, d = 333}]\n" : + "b = [111, {d = 333, a = 222}]\n") + d = Dict("b" => Any[Dict("a" => 222, "d" => 333), 111]) - @test toml_str(d) == "b = [{a = 222, d = 333}, 111]\n" + @test toml_str(d) == (sizeof(Int) == 8 ? + "b = [{a = 222, d = 333}, 111]\n" : + "b = [{d = 333, a = 222}, 111]\n") d = Dict("b" => Any[Dict("a" => 222, "d" => 333)]) - @test toml_str(d) == """ - [[b]] - a = 222 - d = 333 - """ + @test toml_str(d) == (sizeof(Int) == 8 ? + """ + [[b]] + a = 222 + d = 333 + """ : + """ + [[b]] + d = 333 + a = 222 + """) # https://github.com/JuliaLang/julia/pull/57584 d = Dict("b" => [MyStruct(1), MyStruct(2)]) diff --git a/test/arrayops.jl b/test/arrayops.jl index d5fba79c47017..7e2454eabd93c 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -2203,7 +2203,7 @@ end # All we really care about is that we have an optimized # implementation, but the seed is a useful way to check that. -@test hash(CartesianIndex()) == Base.IteratorsMD.cartindexhash_seed +@test hash(CartesianIndex()) == Base.IteratorsMD.cartindexhash_seed ⊻ Base.HASH_SEED @test hash(CartesianIndex(1, 2)) != hash((1, 2)) @testset "itr, iterate" begin diff --git a/test/hashing.jl b/test/hashing.jl index 173a31d10a6a9..41a1d525961cc 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -88,6 +88,7 @@ vals = Any[ Dict(42 => 101, 77 => 93), Dict{Any,Any}(42 => 101, 77 => 93), (1,2,3,4), (1.0,2.0,3.0,4.0), (1,3,2,4), ("a","b"), (SubString("a",1,1), SubString("b",1,1)), + join('c':'s'), SubString(join('a':'z'), 3, 19), # issue #6900 Dict(x => x for x in 1:10), Dict(7=>7,9=>9,4=>4,10=>10,2=>2,3=>3,8=>8,5=>5,6=>6,1=>1), @@ -108,7 +109,7 @@ vals = Any[ ["a", "b", 1, 2], ["a", 1, 2], ["a", "b", 2, 2], ["a", "a", 1, 2], ["a", "b", 2, 3] ] -for a in vals, b in vals +for (i, a) in enumerate(vals), b in vals[i:end] @test isequal(a,b) == (hash(a)==hash(b)) end @@ -249,7 +250,9 @@ end ) for a in vals, b in vals - @test isequal(a, b) == (Base.hash_64_32(a) == Base.hash_64_32(b)) + ha = Base.hash_64_32(a) + hb = Base.hash_64_32(b) + @test isequal(a, b) == (ha == hb) end end @@ -260,7 +263,9 @@ end ) for a in vals, b in vals - @test isequal(a, b) == (Base.hash_32_32(a) == Base.hash_32_32(b)) + ha = Base.hash_32_32(a) + hb = Base.hash_32_32(b) + @test isequal(a, b) == (ha == hb) end end end diff --git a/test/show.jl b/test/show.jl index 17f0e2bfbf5e3..fa5989d6cd91d 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1957,9 +1957,9 @@ end @test replstr(view(A, [1], :)) == "1×1 view(::Matrix{Float64}, [1], :) with eltype Float64:\n 0.0" # issue #27680 - @test showstr(Set([(1.0,1.0), (2.0,2.0), (3.0, 3.0)])) == (sizeof(Int) == 8 ? - "Set([(1.0, 1.0), (3.0, 3.0), (2.0, 2.0)])" : - "Set([(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)])") + @test showstr(Set([(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)])) == (sizeof(Int) == 8 ? + "Set([(2.0, 2.0), (1.0, 1.0), (3.0, 3.0)])" : + "Set([(2.0, 2.0), (1.0, 1.0), (3.0, 3.0)])") # issue #27747 let t = (x = Integer[1, 2],) diff --git a/test/tuple.jl b/test/tuple.jl index 13af5ac992434..560a1425c6bb6 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -369,9 +369,9 @@ end @test !isless((1,2), (1,2)) @test !isless((2,1), (1,2)) - @test hash(()) === Base.tuplehash_seed - @test hash((1,)) === hash(1, Base.tuplehash_seed) - @test hash((1,2)) === hash(1, hash(2, Base.tuplehash_seed)) + @test hash(()) === Base.tuplehash_seed ⊻ Base.HASH_SEED + @test hash((1,)) === hash(1, Base.tuplehash_seed ⊻ Base.HASH_SEED) + @test hash((1,2)) === hash(1, hash(2, Base.tuplehash_seed ⊻ Base.HASH_SEED)) # Test Any32 methods t = ntuple(identity, 32) @@ -393,7 +393,7 @@ end @test !isless((t...,1,2), (t...,1,2)) @test !isless((t...,2,1), (t...,1,2)) - @test hash(t) === foldr(hash, [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,(),UInt(0)]) + @test hash(t) === foldr(hash, vcat(1:32, (), Base.HASH_SEED)) end @testset "functions" begin From c13ea2b4f0f9733dfb6521c64ed1c62ec311b351 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 10 May 2025 09:30:55 -0500 Subject: [PATCH 244/662] bpart: failed "edge" invalidation (#58336) This adds a test for an invalidation failure when `consts` are redefined. The failure occurs only in "edge" mode, i.e., due to the separation of precompilation from the user's running session. --- test/precompile.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/precompile.jl b/test/precompile.jl index 18e66e3172d2d..f92a2cc62ef0d 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -956,6 +956,17 @@ precompile_test_harness("code caching") do dir use_stale(c) = stale(c[1]) + not_stale("hello") build_stale(x) = use_stale(Any[x]) + # bindings + struct InvalidatedBinding + x::Int + end + struct Wrapper + ib::InvalidatedBinding + end + makewib(x) = Wrapper(InvalidatedBinding(x)) + const gib = makewib(1) + fib() = gib.ib.x + # force precompilation build_stale(37) stale('c') @@ -985,6 +996,7 @@ precompile_test_harness("code caching") do dir Base.Experimental.@force_compile useA2() end + precompile($StaleA.fib, ()) ## Reporting tests call_nbits(x::Integer) = $StaleA.nbits(x) @@ -1014,6 +1026,15 @@ precompile_test_harness("code caching") do dir @eval using $StaleA MA = invokelatest(getfield, @__MODULE__, StaleA) Base.eval(MA, :(nbits(::UInt8) = 8)) + Base.eval(MA, quote + struct InvalidatedBinding + x::Float64 + end + struct Wrapper + ib::InvalidatedBinding + end + const gib = makewib(2.0) + end) @eval using $StaleC invalidations = Base.StaticData.debug_method_invalidation(true) @eval using $StaleB @@ -1044,6 +1065,11 @@ precompile_test_harness("code caching") do dir m = only(methods(MC.call_buildstale)) mi = m.specializations::Core.MethodInstance @test hasvalid(mi, world) # was compiled with the new method + m = only(methods(MA.fib)) + mi = m.specializations::Core.MethodInstance + @test isdefined(mi, :cache) # it was precompiled by StaleB + @test_broken !hasvalid(mi, world) # invalidated by redefining `gib` before loading StaleB + @test_broken MA.fib() === 2.0 # Reporting test (ensure SnoopCompile works) @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) From 97a2403ca18acb57d169dca974463bc0a389cbcf Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Sat, 10 May 2025 10:37:59 -0500 Subject: [PATCH 245/662] CODEOWNERS: Add some specific people to files (#58357) --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bf1380f5a07bc..6a152517b78c1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,3 +4,10 @@ CODEOWNERS @JuliaLang/github-actions /.github/workflows/rerun_failed.yml @DilumAluthge /.github/workflows/statuses.yml @DilumAluthge +/base/special/ @oscardssmith +/base/sort.jl @LilithHafner +/test/sorting.jl @LilithHafner +/stdlib/*_jll @giordano +/base/binaryplatforms.jl @giordano +/src/julia_gcext.h @fingolfin +/test/gcext/gcext.c @fingolfin From 4635377e11f91ab34c0db12e0055556ff829e316 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Sat, 10 May 2025 15:29:37 -0400 Subject: [PATCH 246/662] make `WeakKeyDict` more type stable and add `const` to some fields (#57877) --- base/weakkeydict.jl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index 1a98bf1ee4333..191cca485ab3c 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -2,6 +2,12 @@ # weak key dictionaries +mutable struct WeakKeyDictFinalizer{T} + const d::T +end +(d::WeakKeyDictFinalizer)(k) = d.d.dirty = true + + """ WeakKeyDict([itr]) @@ -16,15 +22,15 @@ object was unreferenced anywhere before insertion. See also [`WeakRef`](@ref). """ mutable struct WeakKeyDict{K,V} <: AbstractDict{K,V} - ht::Dict{WeakRef,V} - lock::ReentrantLock - finalizer::Function + const ht::Dict{WeakRef,V} + const lock::ReentrantLock dirty::Bool + finalizer::WeakKeyDictFinalizer # Constructors mirror Dict's - function WeakKeyDict{K,V}() where V where K - t = new(Dict{WeakRef,V}(), ReentrantLock(), identity, 0) - t.finalizer = k -> t.dirty = true + function WeakKeyDict{K,V}() where {K, V} + t = new{K,V}(Dict{WeakRef,V}(), ReentrantLock(), false) + t.finalizer = WeakKeyDictFinalizer(t) return t end end From d53422ceff19ad1381992292c520142564651b69 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Sat, 10 May 2025 16:57:45 -0300 Subject: [PATCH 247/662] Add set to temporary roots to avoid O(N) check (#57961) --- src/aotcompile.cpp | 1 + src/codegen.cpp | 7 +++---- src/jitlayers.cpp | 1 + src/jitlayers.h | 2 ++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 1e78882901d04..144ad6fc804d7 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -824,6 +824,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm generate_cfunc_thunks(params, compiled_functions); aot_optimize_roots(params, method_roots, compiled_functions); params.temporary_roots = nullptr; + params.temporary_roots_set.clear(); JL_GC_POP(); // process the globals array, before jl_merge_module destroys them diff --git a/src/codegen.cpp b/src/codegen.cpp index 28d3281584b85..e5600115b3a7d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3170,11 +3170,10 @@ static void jl_temporary_root(jl_codegen_params_t &ctx, jl_value_t *val) { if (!jl_is_globally_rooted(val)) { jl_array_t *roots = ctx.temporary_roots; - for (size_t i = 0; i < jl_array_dim0(roots); i++) { - if (jl_array_ptr_ref(roots, i) == val) - return; - } + if (ctx.temporary_roots_set.find(val) != ctx.temporary_roots_set.end()) + return; jl_array_ptr_1d_push(roots, val); + ctx.temporary_roots_set.insert(val); } } static void jl_temporary_root(jl_codectx_t &ctx, jl_value_t *val) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index f0cd6e643e0a5..676b5aeab84a9 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -788,6 +788,7 @@ void jl_emit_codeinst_to_jit_impl( } jl_optimize_roots(params, jl_get_ci_mi(codeinst), *result_m.getModuleUnlocked()); // contains safepoints params.temporary_roots = nullptr; + params.temporary_roots_set.clear(); JL_GC_POP(); { // drop lock before acquiring engine_lock auto release = std::move(params.tsctx_lock); diff --git a/src/jitlayers.h b/src/jitlayers.h index 619e5f3757642..49030fbe8cbf0 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -1,5 +1,6 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license +#include "llvm/ADT/SmallSet.h" #include #include #include @@ -245,6 +246,7 @@ struct jl_codegen_params_t { SmallVector cfuncs; std::map global_targets; jl_array_t *temporary_roots = nullptr; + SmallSet temporary_roots_set; std::map, GlobalVariable*> external_fns; std::map ditypes; std::map llvmtypes; From bbb058295a6ed57dd33b6e35be18dbdfb7447ac4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 10 May 2025 18:27:55 -0400 Subject: [PATCH 248/662] fix called-argument analysis for calls with splat (#58070) --- src/julia-syntax.scm | 5 +++++ test/syntax.jl | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 0268306f147f0..8f0bcd55ac194 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3513,6 +3513,11 @@ (let ((vi (get tab (cadr e) #f))) (if vi (vinfo:set-called! vi #t)) + ;; calls f(x...) go through `_apply_iterate` + (if (and (length> e 3) (equal? (cadr e) '(core _apply_iterate))) + (let ((vi2 (get tab (cadddr e) #f))) + (if vi2 + (vinfo:set-called! vi2 #t)))) ;; calls to functions with keyword args have head of `kwcall` first (if (and (length> e 3) (equal? (cadr e) '(core kwcall))) (let ((vi2 (get tab (cadddr e) #f))) diff --git a/test/syntax.jl b/test/syntax.jl index 7a7ca63f797da..27abea4c57a66 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3573,6 +3573,8 @@ end # issue #45162 f45162(f) = f(x=1) @test first(methods(f45162)).called != 0 +f45162_2(f) = f([]...) +@test first(methods(f45162_2)).called != 0 # issue #45024 @test_parseerror "const x" "expected assignment after \"const\"" From 6a796e069340a98fd0f8288fcacd49d256c282d0 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 11 May 2025 00:28:10 +0200 Subject: [PATCH 249/662] Make trimming tests work in an out-of-tree build. (#58340) --- test/Makefile | 2 +- test/trimming/Makefile | 27 +++++++++++++++------------ test/trimming/trimming.jl | 7 +++++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/test/Makefile b/test/Makefile index 9b151cd213274..69b7ad1451d0f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -24,7 +24,7 @@ EMBEDDING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/embedding" "CC=$(CC GCEXT_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/gcext" "CC=$(CC)" -TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(JULIAHOME)/usr/bin" "CC=$(CC)" +TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/trimming" "CC=$(CC)" default: diff --git a/test/trimming/Makefile b/test/trimming/Makefile index 63114a2764570..e3c7536bbc92c 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -16,7 +16,6 @@ endif # location of test source SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/../..) -BUILDSCRIPT := $(BIN)/../share/julia/juliac-buildscript.jl include $(JULIAHOME)/Make.inc # get the executable suffix, if any @@ -24,32 +23,36 @@ EXE := $(suffix $(abspath $(JULIA))) # get compiler and linker flags. (see: `contrib/julia-config.jl`) JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- +JULIA_LIBDIR := $(shell $(JULIA) -e 'println(joinpath(Sys.BINDIR, "..", "lib"))' --) CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal +# get the JuliaC build script +JULIAC_BUILDSCRIPT := $(shell $(JULIA) -e 'print(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "juliac-buildscript.jl"))') + #============================================================================= -release: hello$(EXE) basic_jll$(EXE) +release: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) -hello-o.a: $(SRCDIR)/hello.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true +$(BIN)/hello-o.a: $(SRCDIR)/hello.jl $(JULIAC_BUILDSCRIPT) + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true -basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $< --output-exe true +$(BIN)/basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(JULIAC_BUILDSCRIPT) + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true -hello$(EXE): hello-o.a +$(BIN)/hello$(EXE): $(BIN)/hello-o.a $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -basic_jll$(EXE): basic_jll-o.a +$(BIN)/basic_jll$(EXE): $(BIN)/basic_jll-o.a $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -check: hello$(EXE) basic_jll$(EXE) - $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming +check: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) + $(JULIA) --depwarn=error $(SRCDIR)/trimming.jl $< clean: - -rm -f hello$(EXE) basic_jll$(EXE) hello-o.a basic_jll-o.a + -rm -f $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/hello-o.a $(BIN)/basic_jll-o.a .PHONY: release clean check diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index 69d9adf9b003f..0f6d452c25c9b 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -1,12 +1,15 @@ using Test +@test length(ARGS) == 1 +bindir = dirname(ARGS[1]) + let exe_suffix = splitext(Base.julia_exename())[2] - hello_exe = joinpath(@__DIR__, "hello" * exe_suffix) + hello_exe = joinpath(bindir, "hello" * exe_suffix) @test readchomp(`$hello_exe`) == "Hello, world!" @test filesize(hello_exe) < 2_000_000 - basic_jll_exe = joinpath(@__DIR__, "basic_jll" * exe_suffix) + basic_jll_exe = joinpath(bindir, "basic_jll" * exe_suffix) lines = split(readchomp(`$basic_jll_exe`), "\n") @test lines[1] == "Julia! Hello, world!" @test lines[2] == lines[3] From 13a142d966691d4153d243781a8ff08d44d11b5a Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Sat, 10 May 2025 20:27:42 -0400 Subject: [PATCH 250/662] fix Big hashing (#58377) --- base/gmp.jl | 6 +++--- test/gmp.jl | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index a91226cebd737..7ec41b189478b 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -848,7 +848,7 @@ if Limb === UInt64 === UInt # an optimized version for BigInt of hash_integer (used e.g. for Rational{BigInt}), # and of hash - using .Base: hash_uint + using .Base: hash_finalizer function hash_integer(n::BigInt, h::UInt) GC.@preserve n begin @@ -856,9 +856,9 @@ if Limb === UInt64 === UInt s == 0 && return hash_integer(0, h) p = convert(Ptr{UInt64}, n.d) b = unsafe_load(p) - h ⊻= hash_uint(ifelse(s < 0, -b, b) ⊻ h) + h ⊻= hash_finalizer(ifelse(s < 0, -b, b) ⊻ h) for k = 2:abs(s) - h ⊻= hash_uint(unsafe_load(p, k) ⊻ h) + h ⊻= hash_finalizer(unsafe_load(p, k) ⊻ h) end return h end diff --git a/test/gmp.jl b/test/gmp.jl index 8caf69b36fcc1..991e4613bb621 100644 --- a/test/gmp.jl +++ b/test/gmp.jl @@ -808,3 +808,11 @@ t = Rational{BigInt}(0, 1) @test Base.GMP.MPQ.div!(-oo, zo) == -oz end end + +@testset "hashing" begin + for i in 1:10:100 + bint = big(11)^i + bfloat = big(11.0)^i + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + end +end From 051be135ba4c30cb561d1ee8177954fee232e780 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Sun, 11 May 2025 06:48:26 -0400 Subject: [PATCH 251/662] Fix error hint for `missing(1)` (#58250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: ```julia-repl julia> missing(1) ERROR: MethodError: objects of type Missing are not callable. In case you did not try calling it explicitly, check if a Missing has been passed as an argument to a method that expects a callable instead. The object of type `Missing` exists, but no method is defined for this combination of argument types when trying to treat it as a callable object.┌ Error: Hint-handler nonsetable_type_hint_handler for MethodError in Base caused an error │ exception = │ 1-element ExceptionStack: │ TypeError: non-boolean (Missing) used in boolean context │ Stacktrace: │ [1] nonsetable_type_hint_handler(io::Any, ex::Any, arg_types::Any, kwargs::Any) │ @ Base ./errorshow.jl:1054 │ [2] show_error_hints(::IOContext{Base.TTY}, ::MethodError, ::Vector{Any}, ::Vararg{Vector{Any}}) │ @ Base.Experimental ./experimental.jl:322 │ [3] showerror(io::IO, ex::MethodError) │ @ Base ./errorshow.jl:374 │ [4] showerror(io::IOContext{Base.TTY}, ex::MethodError, bt::Vector{Base.StackTraces.StackFrame}; backtrace::Bool) │ @ Base ./errorshow.jl:106 │ [5] show_exception_stack(io::IOContext{Base.TTY}, stack::Base.ExceptionStack) │ @ Base ./errorshow.jl:1017 │ [6] display_error(io::IOContext{Base.TTY}, stack::Base.ExceptionStack) │ @ Base ./client.jl:117 │ [7] repl_display_error(errio::IO, errval::Any) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:565 │ [8] print_response(errio::IO, response::Any, backend::Union{Nothing, REPL.REPLBackendRef}, show_value::Bool, have_color::Bool, specialdisplay::Union{Nothing, AbstractDisplay}) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:582 │ [9] (::REPL.var"#print_response##0#print_response##1"{REPL.LineEditREPL, Pair{Any, Bool}, Bool, Bool})(io::Any) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:556 │ [10] with_repl_linfo(f::Any, repl::REPL.LineEditREPL) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:829 │ [11] print_response(repl::REPL.AbstractREPL, response::Any, show_value::Bool, have_color::Bool) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:554 │ [12] (::REPL.var"#do_respond#73"{Bool, Bool, REPL.var"#setup_interface##12#setup_interface##13"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::Any, ok::Bool) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:1186 │ [13] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState) │ @ REPL.LineEdit ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/LineEdit.jl:2852 │ [14] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef) │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:1657 │ [15] (::REPL.var"#61#62"{REPL.LineEditREPL, REPL.REPLBackendRef})() │ @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/REPL.jl:648 └ @ Base.Experimental experimental.jl:325 Stacktrace: [1] top-level scope @ REPL[1]:1 ``` After: ```julia-repl julia> missing(1) ERROR: MethodError: objects of type Missing are not callable. In case you did not try calling it explicitly, check if a Missing has been passed as an argument to a method that expects a callable instead. The object of type `Missing` exists, but no method is defined for this combination of argument types when trying to treat it as a callable object. Stacktrace: [1] top-level scope @ REPL[3]:1 ``` --------- Co-authored-by: Ian Butterworth --- base/errorshow.jl | 2 +- test/errorshow.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 2e798b60cf733..384b459f99972 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1056,7 +1056,7 @@ Experimental.register_error_hint(noncallable_number_hint_handler, MethodError) # eg: d = Dict; d["key"] = 2 function nonsetable_type_hint_handler(io, ex, arg_types, kwargs) @nospecialize - if ex.f == setindex! + if ex.f === setindex! T = arg_types[1] if T <: Number print(io, "\nAre you trying to index into an array? For multi-dimensional arrays, separate the indices with commas: ") diff --git a/test/errorshow.jl b/test/errorshow.jl index 8d582ba8e538c..246a7f16bfc90 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -406,6 +406,8 @@ let err_str, @test occursin("MethodError: no method matching Bool()", err_str) err_str = @except_str :a() MethodError @test occursin("MethodError: objects of type Symbol are not callable", err_str) + err_str = @except_str missing(1) MethodError + @test occursin("MethodError: objects of type Missing are not callable", err_str) err_str = @except_str EightBitType() MethodError @test occursin("MethodError: no method matching $(curmod_prefix)EightBitType()", err_str) err_str = @except_str i() MethodError From 1b1b5d57765a72c5f10186f0142ace4c59654cfe Mon Sep 17 00:00:00 2001 From: "Bradley C. Kuszmaul" Date: Sun, 11 May 2025 07:58:21 -0400 Subject: [PATCH 252/662] Implement `PaddedSpinLock`, which avoids false sharing. (#55944) --- NEWS.md | 8 +++++ base/locks-mt.jl | 55 +++++++++++++++++++++++++++------ doc/src/base/multi-threading.md | 2 ++ test/threads_exec.jl | 20 ++++++++++++ 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/NEWS.md b/NEWS.md index f74d00bb1fb7f..d2eb96214beec 100644 --- a/NEWS.md +++ b/NEWS.md @@ -24,6 +24,14 @@ Command-line option changes Multi-threading changes ----------------------- +* A new `AbstractSpinLock` is defined with `SpinLock <: AbstractSpinLock` ([#55944]). +* A new `PaddedSpinLock <: AbstractSpinLock` is defined. It has extra padding to avoid false sharing ([#55944]). +* New types are defined to handle the pattern of code that must run once per process, called + a `OncePerProcess{T}` type, which allows defining a function that should be run exactly once + the first time it is called, and then always return the same result value of type `T` + every subsequent time afterwards. There are also `OncePerThread{T}` and `OncePerTask{T}` types for + similar usage with threads or tasks. ([#TBD]) + Build system changes -------------------- diff --git a/base/locks-mt.jl b/base/locks-mt.jl index 5d355b9ed200c..237e0d9856996 100644 --- a/base/locks-mt.jl +++ b/base/locks-mt.jl @@ -3,7 +3,7 @@ import .Base: unsafe_convert, lock, trylock, unlock, islocked, wait, notify, AbstractLock export SpinLock - +public PaddedSpinLock # Important Note: these low-level primitives defined here # are typically not for general usage @@ -12,33 +12,68 @@ export SpinLock ########################################## """ - SpinLock() + abstract type AbstractSpinLock <: AbstractLock end -Create a non-reentrant, test-and-test-and-set spin lock. +A non-reentrant, test-and-test-and-set spin lock. Recursive use will result in a deadlock. This kind of lock should only be used around code that takes little time to execute and does not block (e.g. perform I/O). In general, [`ReentrantLock`](@ref) should be used instead. Each [`lock`](@ref) must be matched with an [`unlock`](@ref). -If [`!islocked(lck::SpinLock)`](@ref islocked) holds, [`trylock(lck)`](@ref trylock) +If [`!islocked(lck::AbstractSpinLock)`](@ref islocked) holds, [`trylock(lck)`](@ref trylock) succeeds unless there are other tasks attempting to hold the lock "at the same time." Test-and-test-and-set spin locks are quickest up to about 30ish contending threads. If you have more contention than that, different synchronization approaches should be considered. """ -mutable struct SpinLock <: AbstractLock +abstract type AbstractSpinLock <: AbstractLock end + +""" + SpinLock() <: AbstractSpinLock + +Spinlocks are not padded, and so may suffer from false sharing. +See also [`PaddedSpinLock`](@ref). + +See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. +""" +mutable struct SpinLock <: AbstractSpinLock # we make this much larger than necessary to minimize false-sharing @atomic owned::Int SpinLock() = new(0) end +# TODO: Determine the cache line size using e.g., CPUID. Meanwhile, this is correct for most +# processors. +const CACHE_LINE_SIZE = 64 + +""" + PaddedSpinLock() <: AbstractSpinLock + +PaddedSpinLocks are padded so that each is guaranteed to be on its own cache line, to avoid +false sharing. +See also [`SpinLock`](@ref). + +See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. +""" +mutable struct PaddedSpinLock <: AbstractSpinLock + # we make this much larger than necessary to minimize false-sharing + _padding_before::NTuple{max(0, CACHE_LINE_SIZE - sizeof(Int)), UInt8} + @atomic owned::Int + _padding_after::NTuple{max(0, CACHE_LINE_SIZE - sizeof(Int)), UInt8} + function PaddedSpinLock() + l = new() + @atomic l.owned = 0 + return l + end +end + # Note: this cannot assert that the lock is held by the correct thread, because we do not # track which thread locked it. Users beware. -Base.assert_havelock(l::SpinLock) = islocked(l) ? nothing : Base.concurrency_violation() +Base.assert_havelock(l::AbstractSpinLock) = islocked(l) ? nothing : Base.concurrency_violation() -function lock(l::SpinLock) +function lock(l::AbstractSpinLock) while true if @inline trylock(l) return @@ -49,7 +84,7 @@ function lock(l::SpinLock) end end -function trylock(l::SpinLock) +function trylock(l::AbstractSpinLock) if l.owned == 0 GC.disable_finalizers() p = @atomicswap :acquire l.owned = 1 @@ -61,7 +96,7 @@ function trylock(l::SpinLock) return false end -function unlock(l::SpinLock) +function unlock(l::AbstractSpinLock) if (@atomicswap :release l.owned = 0) == 0 error("unlock count must match lock count") end @@ -70,6 +105,6 @@ function unlock(l::SpinLock) return end -function islocked(l::SpinLock) +function islocked(l::AbstractSpinLock) return (@atomic :monotonic l.owned) != 0 end diff --git a/doc/src/base/multi-threading.md b/doc/src/base/multi-threading.md index 81d1d83d765ac..88dc2b7514a2a 100644 --- a/doc/src/base/multi-threading.md +++ b/doc/src/base/multi-threading.md @@ -63,7 +63,9 @@ Base.@threadcall These building blocks are used to create the regular synchronization objects. ```@docs +Base.Threads.AbstractSpinLock Base.Threads.SpinLock +Base.Threads.PaddedSpinLock ``` ## Task metrics (Experimental) diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 70de1fb29aa46..66191f77459fa 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -110,6 +110,26 @@ if threadpoolsize(:default) > 1 end end +if threadpoolsize() > 1 + let lk = Base.Threads.PaddedSpinLock() + c1 = Base.Event() + c2 = Base.Event() + @test trylock(lk) + @test !trylock(lk) + t1 = Threads.@spawn (notify(c1); lock(lk); unlock(lk); trylock(lk)) + t2 = Threads.@spawn (notify(c2); trylock(lk)) + Libc.systemsleep(0.1) # block our thread from scheduling for a bit + wait(c1) + wait(c2) + @test !fetch(t2) + @test istaskdone(t2) + @test !istaskdone(t1) + unlock(lk) + @test fetch(t1) + @test istaskdone(t1) + end +end + # threading constructs @testset "@threads and @spawn threadpools" begin From 9f9d3acd9736762e04ad9c724a820fb3c0dd85ef Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 12 May 2025 01:12:41 +0300 Subject: [PATCH 253/662] deps: explicitely ask for make generator where needed (#58284) --- deps/ittapi.mk | 2 +- deps/libgit2.mk | 2 +- deps/libssh2.mk | 2 +- deps/libsuitesparse.mk | 2 +- deps/zlib.mk | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps/ittapi.mk b/deps/ittapi.mk index b62b981a34ddb..f27ae2d6f77d5 100644 --- a/deps/ittapi.mk +++ b/deps/ittapi.mk @@ -10,7 +10,7 @@ ITTAPI_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DITT_API_IPT_SUPPORT= $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-configured: $(SRCCACHE)/$(ITTAPI_SRC_DIR)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(ITTAPI_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(ITTAPI_OPTS) echo 1 > $@ $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-configured diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 8b17ae6d70424..d051da9df6a35 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -51,7 +51,7 @@ LIBGIT2_SRC_PATH := $(SRCCACHE)/$(LIBGIT2_SRC_DIR) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: $(LIBGIT2_SRC_PATH)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(LIBGIT2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 3f802db15be6d..2108ec4d2ef7f 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -37,7 +37,7 @@ LIBSSH2_SRC_PATH := $(SRCCACHE)/$(LIBSSH2_SRC_DIR) $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: $(LIBSSH2_SRC_PATH)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(LIBSSH2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 85b2c23473a18..4bfeb0742fb7e 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -58,7 +58,7 @@ $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/source-patched: $(BUILDDIR)/SuiteS $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled: | $(build_prefix)/manifest/blastrampoline $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/source-patched - cd $(dir $<) && $(CMAKE) . $(LIBSUITESPARSE_CMAKE_FLAGS) + cd $(dir $<) && $(CMAKE) -G"Unix Makefiles" . $(LIBSUITESPARSE_CMAKE_FLAGS) $(MAKE) -C $(dir $<) $(MAKE) -C $(dir $<) install echo 1 > $@ diff --git a/deps/zlib.mk b/deps/zlib.mk index 5548a0791f4d2..347b8d4cf53b6 100644 --- a/deps/zlib.mk +++ b/deps/zlib.mk @@ -10,7 +10,7 @@ ZLIB_BUILD_OPTS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured: $(SRCCACHE)/$(ZLIB_SRC_DIR)/source-extracted mkdir -p $(dir $@) - cd $(dir $@) && $(CMAKE) $(ZLIB_BUILD_OPTS) $(dir $<) + cd $(dir $@) && $(CMAKE) -G"Unix Makefiles" $(ZLIB_BUILD_OPTS) $(dir $<) echo 1 > $@ $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured From 2a5cfee2807add7babf00ddd93da505e2cce5719 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Mon, 12 May 2025 22:45:10 +0800 Subject: [PATCH 254/662] Some fix for `broadcast` with offset axes. (#43414) Co-authored-by: Matt Bauman --- base/broadcast.jl | 10 ++++--- test/broadcast.jl | 76 ++++++++++++++++++++++------------------------- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 2a16cd11c1d5c..b86baf08ddfe0 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -585,15 +585,17 @@ an `Int`. """ Base.@propagate_inbounds newindex(arg, I::CartesianIndex) = to_index(_newindex(axes(arg), I.I)) Base.@propagate_inbounds newindex(arg, I::Integer) = to_index(_newindex(axes(arg), (I,))) -Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple) = (ifelse(length(ax[1]) == 1, ax[1][1], I[1]), _newindex(tail(ax), tail(I))...) +Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple) = (ifelse(length(ax[1]) == 1, ax[1][begin], I[1]), _newindex(tail(ax), tail(I))...) Base.@propagate_inbounds _newindex(ax::Tuple{}, I::Tuple) = () -Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple{}) = (ax[1][1], _newindex(tail(ax), ())...) +Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple{}) = (ax[1][begin], _newindex(tail(ax), ())...) Base.@propagate_inbounds _newindex(ax::Tuple{}, I::Tuple{}) = () # If dot-broadcasting were already defined, this would be `ifelse.(keep, I, Idefault)`. @inline newindex(I::CartesianIndex, keep, Idefault) = to_index(_newindex(I.I, keep, Idefault)) -@inline newindex(i::Integer, keep::Tuple, idefault) = ifelse(keep[1], i, idefault[1]) -@inline newindex(i::Integer, keep::Tuple{}, idefault) = CartesianIndex(()) +@inline newindex(I::CartesianIndex{1}, keep, Idefault) = newindex(I.I[1], keep, Idefault) +@inline newindex(i::Integer, keep::Tuple, idefault) = CartesianIndex(ifelse(keep[1], Int(i), Int(idefault[1])), idefault[2]) +@inline newindex(i::Integer, keep::Tuple{Bool}, idefault) = ifelse(keep[1], i, idefault[1]) +@inline newindex(i::Integer, keep::Tuple{}, idefault) = CartesianIndex() @inline _newindex(I, keep, Idefault) = (ifelse(keep[1], I[1], Idefault[1]), _newindex(tail(I), tail(keep), tail(Idefault))...) @inline _newindex(I, keep::Tuple{}, Idefault) = () # truncate if keep is shorter than I diff --git a/test/broadcast.jl b/test/broadcast.jl index 0f5bdf7a40bb1..a751d9d381ce6 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -53,6 +53,7 @@ ci(x) = CartesianIndex(x) @test @inferred(newindex(ci((2,2)), (true,), (-1,))) == 2 @test @inferred(newindex(ci((2,2)), (false,), (-1,))) == -1 @test @inferred(newindex(ci((2,2)), (), ())) == ci(()) +@test @inferred(newindex(ci((2,)), (true, false, false), (-1, -1, -1))) == ci((2, -1)) end @@ -853,29 +854,34 @@ let @test Dict(c .=> d) == Dict("foo" => 1, "bar" => 2) end -# Broadcasted iterable/indexable APIs -let - bc = Broadcast.instantiate(Broadcast.broadcasted(+, zeros(5), 5)) - @test IndexStyle(bc) == IndexLinear() - @test eachindex(bc) === Base.OneTo(5) - @test length(bc) === 5 - @test ndims(bc) === 1 - @test ndims(typeof(bc)) === 1 - @test bc[1] === bc[CartesianIndex((1,))] === 5.0 - @test copy(bc) == [v for v in bc] == collect(bc) - @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) - @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) - - bc = Broadcast.instantiate(Broadcast.broadcasted(+, zeros(5), 5*ones(1, 4))) - @test IndexStyle(bc) == IndexCartesian() - @test eachindex(bc) === CartesianIndices((Base.OneTo(5), Base.OneTo(4))) - @test length(bc) === 20 - @test ndims(bc) === 2 - @test ndims(typeof(bc)) === 2 - @test bc[1,1] == bc[CartesianIndex((1,1))] === 5.0 - @test copy(bc) == [v for v in bc] == collect(bc) - @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) - @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) +isdefined(Main, :OffsetArrays) || @eval Main include("testhelpers/OffsetArrays.jl") +using .Main.OffsetArrays +@testset "Broadcasted iterable/indexable APIs" begin + for f in (identity, x -> OffsetArray(x, ntuple(Returns(-1), ndims(x)))) + a = f(zeros(5)) + bc = Broadcast.instantiate(Broadcast.broadcasted(+, a, 5)) + @test IndexStyle(bc) == IndexLinear() + @test eachindex(bc) === eachindex(a) + @test length(bc) === 5 + @test ndims(bc) === 1 + @test ndims(typeof(bc)) === 1 + @test bc[1] === bc[CartesianIndex((1,))] === 5.0 + @test copy(bc) == [v for v in bc] == collect(bc) + @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) + @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) + + b = f(5*ones(1, 4)) + bc = Broadcast.instantiate(Broadcast.broadcasted(+, a, b)) + @test IndexStyle(bc) == IndexCartesian() + @test eachindex(bc) === CartesianIndices((axes(a, 1), axes(b, 2))) + @test length(bc) === 20 + @test ndims(bc) === 2 + @test ndims(typeof(bc)) === 2 + @test bc[1,1] == bc[CartesianIndex((1,1))] === 5.0 + @test copy(bc) == [v for v in bc] == collect(bc) + @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) + @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) + end struct MyFill{T,N} <: AbstractArray{T,N} val :: T @@ -1118,24 +1124,14 @@ end end @testset "inplace broadcast with trailing singleton dims" begin - for (a, b, c) in (([1, 2], reshape([3 4], :, 1), reshape([5, 6], :, 1, 1)), + for (a_, b_, c_) in (([1, 2], reshape([3 4], :, 1), reshape([5, 6], :, 1, 1)), ([1 2; 3 4], reshape([5 6; 7 8], 2, 2, 1), reshape([9 10; 11 12], 2, 2, 1, 1))) - - a_ = copy(a) - a_ .= b - @test a_ == dropdims(b, dims=(findall(==(1), size(b))...,)) - - a_ = copy(a) - a_ .= b - @test a_ == dropdims(b, dims=(findall(==(1), size(b))...,)) - - a_ = copy(a) - a_ .= b .+ c - @test a_ == dropdims(b .+ c, dims=(findall(==(1), size(c))...,)) - - a_ = copy(a) - a_ .*= c - @test a_ == dropdims(a .* c, dims=(findall(==(1), size(c))...,)) + for fun in (x -> OffsetArray(x, ntuple(Returns(1), ndims(x))), identity) + a, b, c = fun(a_), fun(b_), fun(c_) + @test (deepcopy(a) .= b) == dropdims(b, dims=(findall(==(1), size(b))...,)) + @test (deepcopy(a) .= b .+ c) == dropdims(b .+ c, dims=(findall(==(1), size(c))...,)) + @test (deepcopy(a) .*= c) == dropdims(a .* c, dims=(findall(==(1), size(c))...,)) + end end end From b319a959b0a2944f76f3ff1615a594ca384e782f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Tue, 13 May 2025 05:48:56 +0200 Subject: [PATCH 255/662] Use `get_ci_mi` in `store_backedges` (#58394) This allows `CodeInstance`s with ABI override to pass the check, otherwise they were systematically ignored. --- Compiler/src/typeinfer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 9970235e5babc..bb5ea6d19d9bb 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -749,7 +749,7 @@ end # record the backedges function store_backedges(caller::CodeInstance, edges::SimpleVector) - isa(caller.def.def, Method) || return # don't add backedges to toplevel method instance + isa(get_ci_mi(caller).def, Method) || return # don't add backedges to toplevel method instance backedges = ForwardToBackedgeIterator(edges) for (i, (invokesig, item)) in enumerate(backedges) From 2fee9e02749abb5048c49f1004ebff86015ead72 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Tue, 13 May 2025 11:21:07 +0200 Subject: [PATCH 256/662] Remove `jl_line_info_node_t` (no longer used) (#58393) --- src/julia.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/julia.h b/src/julia.h index 08ae5d6eaed62..f696fff21271f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -232,15 +232,6 @@ JL_DLLEXPORT extern const jl_callptr_t jl_f_opaque_closure_call_addr; JL_DLLEXPORT extern const jl_callptr_t jl_fptr_wait_for_compiled_addr; -typedef struct _jl_line_info_node_t { - JL_DATA_TYPE - struct _jl_module_t *module; - jl_value_t *method; // may contain a jl_symbol, jl_method_t, or jl_method_instance_t - jl_sym_t *file; - int32_t line; - int32_t inlined_at; -} jl_line_info_node_t; - struct jl_codeloc_t { int32_t line; int32_t to; From 0e9509313aad40ea173786b403136e1f637807a7 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 13 May 2025 17:07:29 +0200 Subject: [PATCH 257/662] Bump LLVM 19 to include ORC patch. (#58400) --- deps/checksums/llvm | 244 ++++++++++++++++---------------- deps/llvm.version | 8 +- stdlib/libLLVM_jll/Project.toml | 2 +- 3 files changed, 127 insertions(+), 127 deletions(-) diff --git a/deps/checksums/llvm b/deps/checksums/llvm index 1c9598e36d5e2..01e7d6a172902 100644 --- a/deps/checksums/llvm +++ b/deps/checksums/llvm @@ -358,125 +358,125 @@ LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/6afa6 LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/ba0aaa45e8fbd8f3c3506f1ebb7c7d58963108d2be7a6f2991e6f834b06037d7915333f1a562b06b811ba9b38ae287f32f90b87019379a2a55e654e84d7267ca LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/6e5db8ede54690f4ce5fe58024e903a1 LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/5d1402963f9677dd9bf8364f20a832a1fe6466abbeb5837882c68bdbade3818798d3e4a5a2c1cb56efa7cfef1b57ff3d1a35fc3a19ba14b33026ecfc3d5e9597 -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/58b1f2a32f0e3adc617c0027eb33142a -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/4ad6eaa6166dfb47bc3b5e46c7e7f1c34666efe4adb20a917e7514a69f611e9b04dd20b40d709ec3a52e9744e187cd7f14d42db7bf96d4dd44d335f2f044f2ea -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/6b335d68cc47c2061bcffb5f36112ab2 -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/7fe5aa51cbe352d0bfa52f3b87feefa615ed52b9fa57852f85cad5305aa8321864b18626de47d8f026d61579de05b230872204ee2482a25922927cac07a79c8e -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/8ad2caa455a2fcd642f3af38f5d0e939 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/b8ea397d73dc3be3634d15b601ecdf8ab87dc210f35728af6c78b6cc8db2b7406efe27a22540dce1184ba0928b5074e0a760b85e3834a41a5cc36f7e2c0be397 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/70897ebe7ebb83e58bc3aec3f8314b8f -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/29d571dea6ac9eb2e7bfc563d2789b77e67e65a955708413b51083ca9ee10759ac338999329346c08a556b5feb32e8cccac0a86533a25fd3c539542fee375401 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/582520a086a3412fb8412b63147e90d0 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/ae4c033c21aa0486bba755ed2600c17570d16b6cc9acbbf96490f6107d081077928fc716417ca4c55aa1301207cde78e06b3cf839abb80210c02a78af003d649 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e58868cfcebf4d8db2121d8fd5180d7b -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/32caaa07e9305a0415ddbdbb931d48964282822097bcc6711d262f08f7c9230ffd2633a7aa859faf6e983b3384744f37b0d0e45775a0fae31438f10e615708ea -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/f8afc23c3a78cce871e909986de0eea0 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/1910b76f06ad6c7756cf1fbb83f26252c2a3fc86e244727bfa8e0f34ca218e674e654bd8d1dcc87f9b1aef35d5c31390c4d0b58757be58cec3a3045ac5ec703e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/58faa63cc3c61d3d98b222f661f37b3e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/7ba4c020332cb2d64dbb37783ec9b482e08158c4ff9afcac23a8439aa3508779238df1c932c53a4118494632fda2b69a7ab4044f454cdc80ff11469cc17deaa1 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/72c37072d8447d181f4ed593d4ac49d4 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/cda3f4d2972c9bd71125f62b3df54abc44832d1364f8534f77891b21346bbd05605e3a646e9773789cfa7895e188dc46848a3271e7f0566783c53431bf3f423e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/525202f68ce6dc1ab0165e582d3304f0 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/4e4c538b5b7ed504118bd659dca74111c3123317d0cb8fe1008a253ff6f4f4405121e49178a719d892320aea27540f9284957de65e39e0ddcc65d852ed29d37e -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/28e5a013899637843d2c1dd09052af9f -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/dd3b60bf1a97ae344a22d1abe4310ae6db096a82a6c88b4940843ed5223a508ce1689e03380e0a299afe400e953a5ca5cf0c1c4669c868aba37fd79d56194f2d -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/fb44bbbe4485665870045ec3d7bfed64 -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/dacab6ed13aaf495a761ec3bd22aa959535141c33c20cdc6cd3119e2fe177454090b5acf7f8c61ef1aa24bfffe6afc583e17ec67258a5089fe3e8a8eb93b6b33 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/2397a9ff620a996a58242c61d8f32adf -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/d21f0d8a27c91593247aaf00a30d4ea7c4f4f80935ad7d885dd04b7b96233fa0981b3f0df37bfd8ec3a36c4ca86e03581eb91b2fd1349336b327c364c3613920 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/698fb604e9ad4e6c1cf21f5a367e3e42 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/029ba8e516b216f0631d280bce741fdc661c1ea34970c61ccd94397bafb88f0f382e363be9b63b4a15ae390c8bdb64685cd5bdea4bce0dbdd56c48cdd1ee4b67 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/c5ab553a5644b66701d68c7d8f6a3269 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/c371920abdd10633215451555faf4369f813295c51b822ef88b86de19e319c511f2bf718b29b70849d9ba03febbdffb7b5e6a43e819af27c04e8acd8bd165943 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/0900377096a24801c51908aaf90ad8c7 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/7de7b22352f0205619f6c67de031b93e9995d0ec39f043a8040b9a0380605300736ae9f4a7e6cb8d6005e30851223750767dbee83fdd40ab81ad668947ae12d7 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/63620db5e64a4d6e58567f082580d499 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/2eb8f33d1473c6e5a94f795d2889be3a91a67eb211c23cc96d33d30010a41628d2a3215928617c22ff964913db481ed4c656972317b84a8f92831a8a85d94596 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/f3e949271862c9e4635b354b2c8ef89b -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/e230bd57fdbe6b09978f99fc03dba34607d79be4941155b1680a1d07abeb4e6dd53276f47320b3d6017aca0bc0f7ee2da8750f957dc919327ef94f1fe4403ecf -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/152fc75ccdc0967a829f71e458aef79e -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/ce38103c75c41b151d87dbf18bfb6362706bbe7c2a992a9cf98d7255ce778b9029bc1c82ae638b7d6d3d1ac58596184dd04a7d712d7bbacfdc389bfb36a89ea6 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/c93b5ccf7d1cf11de0e4abb0473fb9b4 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/07d925c9d459adde3bc7830077de29f8e906d64a37560a4faf2ef11ae7e3c237ee65701d90765ef791c3d8e29e8f41bce160ed113855e66b1a2fb3d4a5bc678e -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/298bd876e6fd8283eb69ed8e625540b3 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/df219d36c96750e75118defbc9f8d717df7f8c1c8d3ed3f3147ef71eea346169146961147a9d73f0f47b702770d382e580e4ee56f741f8be353526a254baa31b -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/0be962fa7752e35c8e9e2431fac55158 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/75fa76552e2dc4f3220da6d5c9fc0e8eb56c15b59bfba720f5a337997cdefecbd4c0bd21ce8b8a55317b652b8708317892c135bc090276f525b6bd5d98e3d3b5 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/c6d4f9e1ab5576d92f7e7246d1f6a062 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/02bd4af64e17c4713070b6b68dbb30ef7c3846a6bb6f014af5b7f044369720ea458cffb1e940c1f163fc3e7dd4fa9844b4b7c084477d496c2486b8d1770fa123 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/b20cb4b6a59b746954e3400fe27404ff -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/383c758cb9d99018d55c0e185823711359c7b9f7dea9b847cb842943cb28df49f44d590972e4b06a0e826cb7341eeed8facaf67bd05cf8fdcb63e2ff8d21beb9 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/931705b532d694b8a45f619577039cfa -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/54652a871995517c73f1deb105d9ec3da788d162f2b44812cb05c97420069f93e080fe1f552dc14294fd189806bb2954f2a6723047ee6867a30a2744c492885e -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/7dec94a6fe9dc615ea0ab95942370cb4 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/625cfa1fbb0ab6ce0b0c48ff0e6739dee0a8809f41682793d5f76f6a6dd8f310fd7088b0dc18675d4de901af67f032abb3ffd93eb97b407f4fb337afeeb8fabd -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/619798baa20185a9f9f4b526be6be262 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/330e9b0dd923a95b91ab896dd6af1eff717f8606c9c25c00915e38336eff16880cae961480c83de7ce26236d1df675f5641cdf5b835c81f1cf6836a053f4d42e -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/c8b5c3d899f357fa51dfe1faca315e09 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/eab18a3b083145ac47bfe9364359e2ab0103cbe81d64599e344187b41afe20420811b17196bd481cdbfd572fd9f3205396b5d05d40618fdc650366c0c94ad519 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/6d8f7c937b779bc63616f55d29a614f9 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/736abc6019cd97c682ba7217148f7bfbabadd05f624d96a49bf497f69e74f5be713eda5a201339e24a9886d3b4d032a14f3deb0182a1dbbca260e852158d56bd -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/32272e346edf8266a735511c56370be1 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a8ec52808123319acc90a376ed1e2324a6760406d61f0b274af709ad7f79e2eaf3e7bb4146aca6c5f366dd29e2704ff2824f8dbbc044a6d563199662c8deacd4 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/793bd357314e062040f97d6540ac5ab9 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/6afc799e8d917fc7260ee8482a62fd433c40ad63c9285c68be08c55333fa03bfdf22aeac706ae7035ad70a9476278b46b5da804105019057db7aee87e6c94421 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/c8aeeaf25ec324f42f54e862373c40f6 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/10fd104fc4cf4d840c7d1087bc11ae62c33b590d93d9ac33bf39d4d13e58708ff9c39ea085b4b08f8e3a6220dea59517ea73f90af18739a32a1c9f6ab2be0991 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/062fb449ba5bca7fdee92fd88111d67e -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/9c4933e45b2e5b6ba09b0817c317acdec7ac1e7a5d6dc465d272449ee5be964d1034c93c993015bd9192655a7a54dff4f8ccfcfc81a7a82c13c5c2e35d7c1099 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ffef665d1e163a45cb9f8eb8942b7ac9 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/1d9e9aae1596fb2fe852881294254a500314b92a007897abcfde64a62b4990b52ab7afeae50ece62da10ac828964bfb4f3f51696d198b8987bd9eafe1a6cd9aa -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/98dfe522508910bb49daa96409a7a9a4 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/1b3186f215c35fdb44da2a4c6893bdbafd783f1be5cf68845a5626803bc4ff5ee9de1ea009c4780f8e95cf27412453855dfc17f399434fb6578e01b75a353916 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/a69aeeccfc58c037825bbb3dc0a279b3 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/f8c1ae00dc45fb5c4d9047ba9ce2b43ffefd3420acfb5b72769554532bb819086ed60c2c043586a6cd524fe5ea30bfff213d90bcd03976083b1c04b80750b95d -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/904b518f2a7d00882159b8a62362e645 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/f79a8f098f12a331866f10d2b767a41dcfdca0cc16808ffef22d4c6c62bfe294845d3bcaac93173909407c703e8ab406f555d69a9dfefb3b997afb0b6545becb -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b06c7546c197affd4293394ee00a95a9 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/8459c88f791b059e38f0f6a20bbd6d784f95d050f2dd029f8b83804b5b72bcf99f912224b47364adcfecb487b0f06f8acf98bbe0a0a99f89a97ae26413e4cb96 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/294d4456537499b5388cbcca3b531fbe -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/74173b6a9f144bf0197afd88fe4dac24a1ed5481f4000092b9e9ed43502a412ed9d77ffd0d74dda8be36c4f10e8af1032ce0c7fd8e1f5a59317b44d8247a0fb3 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/337ca0b2f64acd793fe5064dd5fad5da -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/76245773da9b24ebac526a473b73f3ca39f3fd6856f5e11bfced43a4f283601f572701207e27903549c269e36bd388194c168adbbb2579bbe39c3240eeb5e662 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/9331f7f745d396cd0b2a25fe89cc2a59 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/7a0e15ea5fa2fa680d00c42dd79ff62aef66ffc3cb5158129eb140c2fc07aa0bcc10089e30dfe84431036ab71f7e20144f9166540c9879107849463ff5267c19 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/58d9896963896bf2b71eca47e66eef8a -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/c51d5330dcd68938b868a0750220d6ebc341b7d6a0a51bcd2664632dfa9cc691f1c20d41ce95ad8d91cf419f09ce2e62b4612b30508d6ea40f91bcaa9f0680ac -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/4672ea06acaa3216bf3a8f39e82026fa -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/af5173944499bba6543e82e1ad4bacff0b3606660abe4b38955899ecf723c5bfdd8eec3a7ea67e1f951ec7c5909f4e88f25a25d4f8c865fd1731fb20dfafdb8e -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/1db5d937eaceb816d5ea22971986a614 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/582d866bb1d4552cb3d90265661cbd65c08bcc73c1d8739855bf148bded253a137699865a913bb90121293b78382ec71747f3b2e100b87b63c91ae47cf21bb76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/d106579015b35e555b7eeb73af9cba76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/51b369ee4e259f28b984f91526575b0b7e20dc9649151adafbd26a52ee32ed28b4f158234abca18b9c16e0dd885e942ad2fb107268292f4ce65f724d92610abb -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/c64188802f790ef1ce30b1be9b6d5e76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/1a0e3623fc2b4ad49c3a466ec03738f74d720c0ec967f6c9b981a14d0f874cb5805918682e4c9aa415a6109bc8137bd8d4795a0ea5ef242ad5ec48339d407780 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/384e285954d7952d4cf7cb836dc6e01b -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/af84ddf41e64735872b7415c50361a62d1bef7cb2a3df1960201555b88112e78cb4dd8e267036b3322595e217fb6405e2b34f0bd3e4183b02dbea32d11ef2ccb -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/58ff83045ba6cd9986e39bee947de9db -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/016881f83cbf58f35086d34170a266b3b14a7a71ad6e31840efd59e065792ca4f33d86fdffc69da9d41041df445161c29c2e20139753b1f16025088a857544be -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/5c8aa1aee42bd8697c3d30fe9a0c0f77 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/f7d77c8d3c598ea87c3860623fe32099f6d10875bb186482f32237dc265ef17ab903edde00683f04071b880b55f73768c64fadf196110f90b25471f66f54a059 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/8677709cc7e0a6e28512a8687cf632ea -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/6c7a334810bb09af3e4c8a93e8374cb02277bf355edfdb4d79caffaf5fdbf7c7c58a9c2e8d8ee569bb3683d32a7f62dd499fd1fdbd4c4096e3dfc51489b92797 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/c28519001f3331d87d0083e38ececdd6 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/36959aa78954214840a41e58978fbf1a3b0b5fa386312fc1f51165587e0e7162051e653137366e43172431f73a9222a32fd9c7994b9fff4d6d0a783744244aac -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/92d230bc3aaed389f1dbcf77eca3329f -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/25987d5c15f4b42b3a419d848b76cbec8204d24562e7043a9f9c16bc2317d66b6705fa7eda328843239a10d19922b093bd7c33fa68c996320300423387bebf96 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/5b53565b8d141db7795641b8eaae04ce -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/c7f1472bce0cefe3decd65453500aa7ac0972fe0fda85e7f5217ff756b99ec7fa2fac0d93487b649e1fa68cfbd158cae74cce6e521184a418ac409883f40be4c -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/964eb68197504db676b74868043a3527 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/dd991f7cce7eeaf243b09caefafaecd510a523dc9b271ede3084d59edf49275f8542a46d052d855509822e776e870f8293491f5af092e367d13e2e2a87b65bb2 -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/0eded78842fb7d484b0d803e21418ae8 -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/2b18a47acb21e8e2048b1ef28d40facea0f5dd9bd50c52c8a20eac492143bf1cd0a33f5c13b938c6afcb215c179b6a010e25c3215cfdda35ddff6c7f3f1482bd -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/40f497d34f06bb6df9b56012a6b52a4a -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/29042d5d12be54af25a290d84e44315c35f7f53ed96ef17d50d21d6ef414d34e4c6d44d74777ef9d204f975f0fbbf9b8476876ba416796a790c506ea90d1b7a4 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/2e4b98a1ba70d612d6f5d0f904642581 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/1a8918e3a3042e3c4ee45093d881052c10a654924012f12251ea4bdbbc7350fe6a8e83040bc37e9fcaf3e997bc8afb9a7beabdeeaee4029b038111c59ec2914f -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/83473c82c8a2b13dee2f0d65a05c8fd3 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/8e3b3f8d2e9a50663b556d815af4db21141969cfa8c4bdc3c9c85313056747f9acae9ecab1b7ad3602f7e6944b368bf46d237333e79a94d3d9956a1eabffe812 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/03b84dc6c792103fdf5068ecdc7f8107 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/173432f74fe546df74a4f0100f48ab3f0779802f30258273004842f31f575cd72250db7b95a967269ddd8d732e1e9fc06bb2f01ceab4d976444db346d34994a0 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/f906192c572dbadbed09de707a7e9604 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/3be4cced3e630158b4b40fe337a5852eb5a080db0d562e1b0f038f3c36a0cddad4e1be1ec2b91ce2a85d67cdc97daac68ebdf00f993861c2cd283079cd8e80b6 -llvm-julia-19.1.7-1.tar.gz/md5/d8ce0cfe5e7ea8b52763a345c8410174 -llvm-julia-19.1.7-1.tar.gz/sha512/1299591bc245c0405866ab9d54783af2ff85d735d321decb0e38241991e5f13c3f7b4d8588664f1c054559d283ee19cdb99da858d76f9f3b1d6b68d5292bf83c +libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/414d73e8bcdbd0f0261bb1a1a3b7442b +libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/bfc718d113655d9cf364e33b5c879a49295eb6af91e92300ec54a56674bc3d60b0de7310939cb517818ba95d9fae57a94e784d85a3024525904ace8f37ff98a2 +libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/663ac8ada6ef0cd870b6e56f312a066b +libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/32853f6a0319c468b97a0a1fd61acbd3ba6ee57592dbaff12da4be22338a31af6c39e9502f530c49292cec40e3609fac2512ee8cc9bdcda3d557e1d1f59a5237 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/3a063d5a5ee3a71ef4636baf53ade001 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/fab4187b43850cdfe1cbe96d8718f3f35b5541e9ea0283ebbe67578b0aa302263770a37034fb6ab7b4e9d72d258f4fc0c2b18ae75e26c001af2a8b6eced51f5e +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d4c298518b77070d45344ae5a1bf9d10 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/6ddd7dac8bfb5629a7132e8aee613bb5ee2715cf572eeec3457d84b989aea853f960c29b8b7e365a91ea7ba7e48c97fa957ca74b1e72f6d3deb420b0bbef21d5 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/22c4ab82c08691c8ad138c0967a5935b +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/f4ce3ff2c167181bfffdc741324ba45ca58bca1c684c3ea3f86032e843cabf339dbea78802f51b178df155250fbed62140d29be070e249e79d2cd3b10fc4bc03 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/9f0b2dfbed1e396825f207564a29e750 +libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9341ea31a62b67d780b753d0c8a985229b172e281945bf295d1ac279974ba50120736804d78e61cba846007ee081dc995b73256c22a4941c30cdbd2c1c86f0ef +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/31484ad466803d66b9ae416723d4e037 +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/64dfe954af3d92089eca0a79208388b5907bc5999c74f151fa81d8e9599764785424b43e1364d1f86a156e0166e9a719382c0a5955270bdb35c570e6bb0cc95a +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/1981bdbf499d94de6d235a4930af1865 +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/d8f348a7e77bf23c702aebf627c6cf0e67dc9009f1e48d38d2736f957ab10042de76fdd81f49d83d957d3877afb0093910e02269ff29780ce18b590411871435 +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/ab5142c9e83f21ecfc1acb376f2b1f7e +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/a17492d497275f070d7b4aad6c9d97189f90a91ad6b84ab77ff5ebb9c7f69e2c6c95631cf700fc28d012a202ff58221d76f3300d2ec9a53ebbfa1fd94be9e62c +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/1eb8f80a63bf70c8554fb4b2e0d0341f +libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/92d3c3118fe2146c3265f37a3647ad673e0bcb3be773727391f67ad1298eef477e9b73fe7eaaec71a30a87ad3d7d2acc3adf913f4c1c3adc3d5e02bdce52145f +libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/933ba6700d32831a66e18549cc59974e +libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/b4dbc072056b39c78f403740b56994cb1ac627f3047c3016ed6680622ff022d874980dbf39cbb9c80a06cc7a5b87d2a696ec68f8a895e00ace8cf9198c13b249 +libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/583775ff40ea2333fea9a372d338e0f0 +libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/cb51a42b46d844b5b1b7b4cc616f08ab3069052cf6f29692bf258f1544aad3df8b56889ac9153a8026208c2827ca73be925bef87f8cd737b467fdac2ce98ad1e +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/324beb57142320dd719500a76bde4944 +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/5d608cd533f3a67445202ba2df5b09cf1bd7ddc5916c43d9ca11837167e2a0d7cecc11d3ade380044397e6f868c110cab6c724f0abdc099c91be076a131345e7 +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/af4bdd54bf773bcb2deca69b71b4117c +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/48d102b7c6ebd4c607ed77cd7c1972f16433b69dc7cdf89e36ab4d319c6900f928c2306d582b90b8baa0b0e22688b0b37cced966aeffa3dd8d2910d950bdb9b1 +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/29edf8ee693e07fc92bc72f2fa6bb0cf +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/da7865e87c2edd8468c3b065b131c24ca2e24fb30943945dfe74c432b4c4f3996c9c03dcad773b0686fa88631ae8c400001c02af80d23b5b7ee0ebb6dd1a0c08 +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/ddb507f775fdc97eb951d5c0cb257652 +libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/46995f66cc9df7d57d47f6bfae98dc6f3c8cd0030f175148852605f4fe40bb67d7dce76466b07384ba9d48234410c7307243a0d6f6f42982395bb16d46ef3dd0 +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/4a5d9ac2d21d5c01ff03b8d1d65deffa +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/9488068c391e82940dd17cc5d1448d0b87693e64ab7494f9fb56f2a1b0302d71fe25b37fc19a826575e52dd0fde1ef2f7c83b2ad58a08447a3d7c86d93a2f806 +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/d1df8509299ed74e342957daf51e6bce +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/b584858e130696df1f1501a3482407ad06228c91576b7025f5869e1557c9b38e46f75d016d8fc745b71b3b1e9a03dd7883b549c917dfb84de10d8cfb6036e9d9 +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/83f0ab8635d809655dfbeddaac3d0db9 +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/10fe730ad29a1da79aa6d87e1a0bd6f88631ac8376ee4839c911cc287b39e18b417c2329a2a978e5d676f1d312b13c0633808cd6a0acad41a5fbde6d82e1933b +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/0f0d000e0d765d1520de3301c7c66c3a +libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/02e2b31e8d95f4255ff9e97abcd79fbb5b8e862e7df1eb6610b028f337dbdeebbddc8cc7cfccc674cf2b76ef721742c48dedd0219c43f2d1e93512114045172d +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/b4a98372aff1b3ebc8b5926038e66a1c +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/ef6a534dde44582653b82de9cec6749dcc0cd341cb98feed2bf3751cec4c4d959466f11abc32fcdbd0f98d0e84926d5c54661e744c035a51f5a6ef17af245062 +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/10e9f3444ef8aaf41887f6739d0d50dd +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/2c2cc404ea533ed7bafeeb68803d1cdf39281f02005fc14715430f5cee656ce38a618b84e0fa4788abff6ee20e5458975c348951704fac68e0787b8756997b67 +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/b5f6699f74e39ed8f629569919b79a80 +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/f5b90ceb0a9b475af565f115fb2d3ac52658c65f322ef3f4398a4b3e469d17c59c22fd5ba17ed9fd3244050d2d86b4aaad6eb7d0b3b0affdb6dd79493aeb7475 +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/f26c2a94e0f07312f1424c63fdb3a28d +libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/b00ac9012ede2a3f3ea8744d6f4162871aca4337cd5d5e73f7cf9a384d5bc6fe3bab43a1484f18d116970bc9e687823537a672a41b582f64c78662a80a3c4827 +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/92c24048d8fa760f36923f73beed5d57 +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/7d43e502662b706e24959ec2d08f75106ba8aefe139d732a6eaa255b732c5c463aca5d126cee35205aae69af28f4bf808ea4076a52200e53010346a12f923260 +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/db00cee39f2963de04aa9e2b2259596f +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/12cd18203a911f8b778fc2df9877f7e1b4ce8c226297055178b377cfe5fb1d0d97f1ab251d0c82a9ef3af5bf22743c1befb93d98bba04d9faf10eac50a4eee5d +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/05d97f1e6370e7e9327e50c42e6d392e +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/ee8891f07980d2d8af1c82128ce63c54305803ae2bd30ccd8318fbeec13a9ccd09a8d5fc56751a884a4c63cf61d89ef951b1f2df87927e6f5f3e061a5609fedb +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/8079d01030c4562c64732d74cb452aa9 +libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/a830745afc3fa75f84fcd7cedd4eb74d121b69a3b669f58d5190f6df1bcb6f9ecf2fd6433066651cc1f95c3685ad701be63dac42d00b0d2901bffa1f9516bcc8 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/11847604bf9b4cf4acb7532a7e06151a +libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/2da15ec3b4c6f2c26689ece112eeb91d9c9494b2cefd91392339eae2d2bd411a022d5f4c27c36f6879080ab702827aec7fdee1a8aa29feeeea31ff79eb34c87f +libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b0a9f639d0b07f5be3d42de12ade2582 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/ef430adb4879322ed8079fab9dc4aa09090df64d6fb1bfaab285440f7fbd3df3c9ebe7d1f8079b933c7f51fbc51ac637cd777cb0d477b482c6c732dda6ab23a4 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/a320320ff04256936ae6c084444329f2 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/94bcb5e51c9f3315c8bf44750f3cb5861b90a48ac970b261a446493d0f47d5017c68adbe539a3fc4b19de58fddb0dd31f4f23759b0bf0b7eec2e9d946ec8a139 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/aa678417c504201f9578211117d73833 +libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/1a4da7deac68f82f12baf4fcd419cb8dfe1c668de05acd16e17e4bbda0773a13352d2491ae7cfefe9904350718924bddf4bba99b983f48b55b06e88e5694138d +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/f010d5231511c4a776aa9c0673babda4 +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/de9adaee0be3f3e280858a01a08305c7d4e8276d3eaf92d7703629097ec44d3e08f13c926a3ce2999c89badbc454fcb696bead5213bf1574e3385709bf12b24d +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/153cae59280035a21762c9e9f716d285 +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/8d991aeecfb232b1df775a559ddde067be57dcf99f38936cffb1d59a442a368f17063ab9a57aa2fe8a6ce006012cd249bf589aea7acb42f50b78a9dfd96e221b +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/e210e23a2f5d3b9a35970af599af833f +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/a37a36b126befe75f024144d308480841d36e1d7a40b1d88496d85947cf74432e32904766e5a24ab593ddef9e1428972c1390d9c443e20f7174c21a1cefb802a +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/12c00e7a7acb8f7387dfcd735a7c1963 +libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/33a44799c8b1654658076040c7a384cdcb70aa8e17aa0a9fa592dd1b2430b1e8fbae22fc4eca44e653b6deb46c040940157b36959e85ca0a19f02626b5172394 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/4d64a25987dafe940071785c713a5d38 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/8833f863f9dd13f26da8d016a2749bd7ea34cc691107710d03d328a8ab9fa05cecdde09a8d42ca23d4eca9f0337db03748729c0bd91c3d1a3527ad53f1385d92 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/3b6c89bc3f2236e6b325cc41c36289b9 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/87752a65df9b6b613f6eb1894086319d099e7433313cabf046cfc499134142af213b39eef8660bc8017b01858f1be986520754d7cede79daeeb246065bb21443 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/d6a14d1b2ec1b342f082d6d4f9d14d94 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/deefbd80cc10d850c28d5e405b268d7bf4d19852d12468bf5960990a4b43c81af8271267d39ce8560c878b65c876dd276182a45832c18fd3fcf7c553c412421a +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/4b311159cc9ac9f8c37d1ed86ba3c659 +libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/bf7e58bdde3f8ffcd37a7c833b2b109704b1fbf577f1a487715b983f86cb640c4f4ee6e417760c9f24b4d1b6b17a8ce780018c40ec84ddf8b8ca841595226803 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/ae2ab5437d31b92ab6e984c25339eb53 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/9b884936659005e7bc1e7f41375875918af077eca470768ae76005891b1454bdd5009ffe80f1bca9bfbba1bf71b2093c28b37a84414d5df14976b88f7032b866 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/ff114ab4ce06f21179e379158f0d65e4 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a023548d64d0ffdb961ca5ea4364b48609dde9ae165198c69d03e640fe6f2b14990f9a500b3adee4af69e706b365aceb1c3efe5cc30f9ad379416a243f588bf2 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/bfb6b758ad82b56f318959f4f10e1883 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/3862a466117e9bb02adbb24307708a6abfb1fd171de9d353513e43362eb4faecff30a77d7ec3b8e3db92aa21a1d87fd1c749cb096295583672c440c08b977c62 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d7b60b20a31093c80816b18f02889846 +libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9760c4d2958b5229a878c32151f875c27ea2fb37b1de64b18576563c51ed1e99e248dc83798c7109f7526e235ebc72e2f453c5b31a2f6502dfc3c68b01b17f7f +libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/bf7a932266b662efb6ac692d9e97304a +libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/d7e8495d36f128e6e5041897a35aa686dab6f603434d35cec09a61c6f7709745012f0c7761c1d6047ab5b06c1ca094f4e88423cb67177534650e39d1aaeeb41c +libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/096932cb91faa8edb4e7345e58697e3f +libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/6b06e807edb350e92b43f6b2a2ce3e682ae27c30e270ec86edbdbb8c2c1f6b39bbe2f29356b390484931c61f3ac11ad1bed7d7ae255f0e1506e663ad4e3f2944 +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/f52c04d48c275818a377962066f8ba56 +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/07c2aa32b22bf777ad8a404eb0ac40cf98f1bfeb4b61c27e6b7a6094454ec241cc964d24465439e78461469c05fbeb94ad456307a8296be996d8d83dbc84593d +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/2164af5b06a5859a69c8ab8be960978b +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/9cebe82a03e0658498b57e343f1d8d354d0d08f5a343cc39bce5f29bebf93cbf7b2106a7c24bb6622fd0b2fd2c6eaac8ea10eb75ac246a422b729b15e9082fbf +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/4f5e0b71aae35af5466b1b8f86d46814 +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/09aada9df91b2dff204d2007cba6d3f53e2d581dfbd4598dcc74b3c8219eb37e9f11955c8407a553588c663fd8bc8051e9197c8137603a9b6ff77f787e04347e +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e29d70370604472d35c0886a72f844ae +libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/90ed51b8ea3ea52edf66026a35eb9d46c2e0e81f0a250e037835978e71cfb6053cbc80379203a5fd64024457134abd49ea90af778e350bcbfece2fbb43437846 +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/71b4fa440cc6156ada7470ec602b0274 +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/0959f4ddd8611820ebcd78c664a42f47655fb634fc3f920d6cb96e92174b7a7f6ea4dda893fc60c0f7c68e94ee5005f730c0b8ea173b3b42da153220d1c44248 +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/4b876a413a2a2bc33c56a80f4328d91f +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/1eca49b4784646b3b45b338a48e896878fd4e0e968471233c8d3af4f26016b03a790ac91cad9cefc2f2f19930554909a7c1e119670cb744fa48877c072039d6a +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/3bbfbc74fd56dd9e7292bd8d9eaf977e +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/71b85345b66f56382db362c80a75e8575d1117b367d23770c91b486144eba8516ac925b8cae72ff5b092b3a2749f99fd1594df221becb463048f7e57babc4eec +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/62c6a92e23f798224642c1b34e1e510e +libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/5393936567cf46febf5561714bda850309fcecae68e184e93222ebab61376192d3df88da6c756c02167c7b46e5a4598c33450eaadd7fc00e8a33d3df7ac0d11a +libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/88e19d775f444b6243964ed7b9d77e1e +libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/c9bb2ac36fb7f1be1a940aca0e8d9fc67be318bce118930bf1db1c997f757bd57424786e719e18b428e03095a289a9f5e07cf421e68fc7cf7cd223f136a6feed +libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/4d3bcd16e07c9b777556cbbc8ef5469a +libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/9f112fb9d304c4f839f412d2925dac2a5165ad7a6dddfe3d22260c00acf76df4ca494dee405cb02d7faaec116a01da3feaff9f532ce39b49e185d3eb6c858102 +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/c67ac7d99576d6d5655be59ce48198b5 +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/bf6e23826dfb3e2464fbfdd27b13bcea51884d2e77caaf9ee4cf8b6ea2c61509f19f66e0fd6d754880b74f23f77e5634a7cde57b70dac970712eb6645f79bd9c +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/8e6adb6771dc1b4c33db58d18ba84411 +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/83759fb9be7f80e8be8b5e35ceba6bf38aee176378252a4f673b82ce08bc6ec35fbc44e5f7c6ef1f421e12338803a7b71233a4d72d0c7c10c8aaf35092d563c7 +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/24c9936d93198392ccf7308577737fc5 +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/fe08b4529633f8ff2ccf3a0ffb7590e5e06b7e04aa95d6ce1a81869c00cb2a7dfe672ebf62502673d781d2d1a5c6d16a892bedb2eadcee40c629b387230b40cb +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/047883d414e84a470a537f60898ce09c +libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/08da16bddbc2567d5c3b1dd13777625c1dfdd3b9ebb8f655e5dc5c7eff1ed5b5cd352daa9fd2bb4b841ae9e92fd5fbd3975ab0569801e12ac48771623be708a3 +llvm-julia-19.1.7-2.tar.gz/md5/cf75ec3da324ff16892bf530a0f176b1 +llvm-julia-19.1.7-2.tar.gz/sha512/1299591bc245c0405866ab9d54783af2ff85d735d321decb0e38241991e5f13c3f7b4d8588664f1c054559d283ee19cdb99da858d76f9f3b1d6b68d5292bf83c diff --git a/deps/llvm.version b/deps/llvm.version index 194528a2643df..4881d8182e22e 100644 --- a/deps/llvm.version +++ b/deps/llvm.version @@ -2,14 +2,14 @@ ## jll artifact LLVM_JLL_NAME := libLLVM -LLVM_ASSERT_JLL_VER := 19.1.7+1 +LLVM_ASSERT_JLL_VER := 19.1.7+2 ## source build # Version number of LLVM LLVM_VER := 19.1.7 # Git branch name in `LLVM_GIT_URL` repository -LLVM_BRANCH=julia-19.1.7-1 +LLVM_BRANCH=julia-19.1.7-2 # Git ref in `LLVM_GIT_URL` repository -LLVM_SHA1=julia-19.1.7-1 +LLVM_SHA1=julia-19.1.7-2 ## Following options are used to automatically fetch patchset from Julia's fork. This is ## useful if you want to build an external LLVM while still applying Julia's patches. @@ -20,4 +20,4 @@ LLVM_JULIA_DIFF_GITHUB_REPO := https://github.com/llvm/llvm-project # Base GitHub ref for generating the diff. LLVM_BASE_REF := llvm:llvmorg-19.1.7 # Julia fork's GitHub ref for generating the diff. -LLVM_JULIA_REF := JuliaLang:julia-19.1.7-1 +LLVM_JULIA_REF := JuliaLang:julia-19.1.7-2 diff --git a/stdlib/libLLVM_jll/Project.toml b/stdlib/libLLVM_jll/Project.toml index ca342518b6ba1..87eb4263ecac9 100644 --- a/stdlib/libLLVM_jll/Project.toml +++ b/stdlib/libLLVM_jll/Project.toml @@ -1,6 +1,6 @@ name = "libLLVM_jll" uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "19.1.7+1" +version = "19.1.7+2" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From 68ba7b6fb0885f8db44bb5d6828049958d95325d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 13 May 2025 17:43:57 +0200 Subject: [PATCH 258/662] Use a nonzero initial size with llvm::SmallSet. (#58399) --- src/jitlayers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jitlayers.h b/src/jitlayers.h index 49030fbe8cbf0..43fdc9b83b89b 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -246,7 +246,7 @@ struct jl_codegen_params_t { SmallVector cfuncs; std::map global_targets; jl_array_t *temporary_roots = nullptr; - SmallSet temporary_roots_set; + SmallSet temporary_roots_set; std::map, GlobalVariable*> external_fns; std::map ditypes; std::map llvmtypes; From 87ef4b338662172bf4dcd7961fcb338eb507f2ff Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Tue, 13 May 2025 12:52:38 -0400 Subject: [PATCH 259/662] Add `Compiler._verify_trim_world_age` for better printing The `juliac-buildscript` is quite aggressive in how it modifies type printing, so caching the pre-buildscript world like this allows us to print stacktraces in their usual fidelity. Before: ``` [1] get_size_dict!(ne::StaticNestedEinsum{Char, ?, ?}, xs::Any, size_info::Dict{Char, Int64}) @ OMEinsum ~/.julia/dev/OMEinsum/src/einsequence.jl:269 ... ``` After: ``` [1] get_size_dict!(ne::StaticNestedEinsum{Char, nothing, ('j','k','l')}, xs::Any, size_info::Dict{Char, Int64}) @ OMEinsum ~/.julia/dev/OMEinsum/src/einsequence.jl:269 ... ``` --- Compiler/src/typeinfer.jl | 3 ++- contrib/juliac-buildscript.jl | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 6eb4b140fd16b..1a648910ffb42 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1641,7 +1641,8 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m return codeinfos end -verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = invokelatest(verify_typeinf_trim, stdout, codeinfos, onlywarn) +const _verify_trim_world_age = RefValue{UInt}(typemax(UInt)) +verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = Core._call_in_world(_verify_trim_world_age[], verify_typeinf_trim, stdout, codeinfos, onlywarn) function return_type(@nospecialize(f), t::DataType) # this method has a special tfunc world = tls_world_age() diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 0549afc0e1508..4bfbfd2272220 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -4,6 +4,10 @@ inputfile = ARGS[1] output_type = ARGS[2] add_ccallables = ARGS[3] == "true" +# Run the verifier in the current world (before modifications), so that error +# messages and types print in their usual way. +Core.Compiler._verify_trim_world_age[] = Base.get_world_counter() + # Initialize some things not usually initialized when output is requested Sys.__init__() Base.init_depot_path() From f0a8dd8ea32aee6b0ace6025513a03981c12d551 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 13 May 2025 13:01:43 -0400 Subject: [PATCH 260/662] Fix signedness typo in world range update (#58390) This manifested itself as a missing invalidation downstream, but I don't know if this is visible from Base. In general, we don't set the InferenceState's max_world to `typemax(UInt)`, but rather to the maximum world age at start of inference, and then we check at the end of inference if the world age is still the same, and only then raise it to `typemax(UInt)` (which arms the backedges). The downstream setup is a bit more complex, and I don't entirely know where this leaked out, but this change fixed it regardless. --- Compiler/src/abstractinterpretation.jl | 2 +- Compiler/src/typeinfer.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 058476dac0096..dae80bec55693 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3545,7 +3545,7 @@ function merge_override_effects!(interp::AbstractInterpreter, effects::Effects, # N.B.: We'd like deleted_world here, but we can't add an appropriate edge at this point. # However, in order to reach here in the first place, ordinary method lookup would have # had to add an edge and appropriate invalidation trigger. - valid_worlds = WorldRange(m.primary_world, typemax(Int)) + valid_worlds = WorldRange(m.primary_world, typemax(UInt)) if sv.world.this in valid_worlds update_valid_age!(sv, valid_worlds) else diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index bb5ea6d19d9bb..23ba851dcc591 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -475,7 +475,7 @@ function adjust_effects(ipo_effects::Effects, def::Method, world::UInt) valid_worlds = WorldRange(0, typemax(UInt)) if is_effect_overridden(override, :consistent) # See note on `typemax(Int)` instead of `deleted_world` in adjust_effects! - override_valid_worlds = WorldRange(def.primary_world, typemax(Int)) + override_valid_worlds = WorldRange(def.primary_world, typemax(UInt)) if world in override_valid_worlds ipo_effects = Effects(ipo_effects; consistent=ALWAYS_TRUE) valid_worlds = override_valid_worlds From a62a74eded874f6273f0c60a3d8c7eceac763ddf Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 13 May 2025 13:01:56 -0400 Subject: [PATCH 261/662] Update Documenter to v1.11.1 (and deps) (#58404) --- doc/Manifest.toml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 669c22ee4c382..879476c7d0bd9 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -43,10 +43,10 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.4" [[deps.Documenter]] -deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "0efa18ca40b9928422d782b3191c032fd0c90682" +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "7745f07eaf6454c15caa21c5ecaebef3afad32eb" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.10.0" +version = "1.11.1" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] @@ -64,10 +64,10 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll"] -git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.3.1" +version = "1.4.0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] @@ -164,16 +164,22 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2024.12.31" +version = "2025.2.25" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" +[[deps.OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "10.0.1+0" + [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.16+0" +version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -182,14 +188,14 @@ version = "10.44.0+1" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" +version = "2.8.3" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.0" +version = "1.13.0" weakdeps = ["REPL"] [deps.Pkg.extensions] @@ -197,9 +203,9 @@ weakdeps = ["REPL"] [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "0efd947a0c34740721a1af4225fe83431b40c950" +git-tree-sha1 = "516f18f048a195409d6e072acf879a9f017d3900" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.3.0" +version = "1.3.2" [[deps.Preferences]] deps = ["TOML"] @@ -213,7 +219,7 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" [[deps.REPL]] -deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" From 3fc2a159aeeb295dd28e4814b177be6d538e8aba Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Tue, 13 May 2025 20:37:31 +0200 Subject: [PATCH 262/662] Documentation: Remove warning that nonstandard primitive type sizes may reveal LLVM bugs (#58262) --- doc/src/manual/types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/types.md b/doc/src/manual/types.md index 926b33e17460a..17314f8d1d288 100644 --- a/doc/src/manual/types.md +++ b/doc/src/manual/types.md @@ -301,7 +301,7 @@ a name. A primitive type can optionally be declared to be a subtype of some supe is omitted, then the type defaults to having `Any` as its immediate supertype. The declaration of [`Bool`](@ref) above therefore means that a boolean value takes eight bits to store, and has [`Integer`](@ref) as its immediate supertype. Currently, only sizes that are multiples of -8 bits are supported and you are likely to experience LLVM bugs with sizes other than those used above. +8 bits are supported. Therefore, boolean values, although they really need just a single bit, cannot be declared to be any smaller than eight bits. From 86a3e1a90b39e01fffb0441fb1262d25c9a69b86 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 14 May 2025 06:15:38 +0200 Subject: [PATCH 263/662] check that hashing of types does not foreigncall (`jl_type_hash` is concrete evaluated) (#58401) --- base/hashing.jl | 4 ++-- test/hashing.jl | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/base/hashing.jl b/base/hashing.jl index c409d3ae7940f..01274e3182245 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -34,8 +34,8 @@ hash(data::Any) = hash(data, HASH_SEED) hash(w::WeakRef, h::UInt) = hash(w.value, h) # Types can't be deleted, so marking as total allows the compiler to look up the hash -hash(T::Type, h::UInt) = - hash((@assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T)), h) +@noinline _jl_type_hash(T::Type) = @assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T) +hash(T::Type, h::UInt) = hash(_jl_type_hash(T), h) hash(@nospecialize(data), h::UInt) = hash(objectid(data), h) function mul_parts(a::UInt64, b::UInt64) diff --git a/test/hashing.jl b/test/hashing.jl index 41a1d525961cc..4e05b8a0102eb 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -308,4 +308,10 @@ struct AUnionParam{T<:Union{Nothing,Float32,Float64}} end @test hash(5//3) == hash(big(5)//3) end -@test Core.Compiler.is_foldable_nothrow(Base.infer_effects(hash, Tuple{Type{Int}, UInt})) +@testset "concrete eval type hash" begin + @test Core.Compiler.is_foldable_nothrow(Base.infer_effects(hash, Tuple{Type{Int}, UInt})) + + f(h...) = hash(Char, h...); + src = only(code_typed(f, Tuple{UInt}))[1] + @test count(stmt -> Meta.isexpr(stmt, :foreigncall), src.code) == 0 +end From 2c2114ceed8a538290fbb6413e314c28bfd5b4f3 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Wed, 14 May 2025 01:34:28 -0400 Subject: [PATCH 264/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?LinearAlgebra=20stdlib=20from=2007725da=20to=203e4d569=20(#5839?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: LinearAlgebra URL: https://github.com/JuliaLang/LinearAlgebra.jl.git Stdlib branch: master Julia branch: master Old commit: 07725da New commit: 3e4d569 Julia version: 1.13.0-DEV LinearAlgebra version: 1.12.0(Does not match) Bump invoked by: @jishnub Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/LinearAlgebra.jl/compare/07725da80be389e170cff75cf7157399c2449643...3e4d569804bdc8eb891ff1982eb9e90246656d0c ``` $ git log --oneline 07725da..3e4d569 3e4d569 Revert "Cleanup and generalize functions of Hermitian matrices (#1340)" (#1357) 0b46d5c Prune `LinearAlgebra` module in ambiguity test (#1349) 611395d Cleanup and generalize functions of Hermitian matrices (#1340) 2cb1e98 Add compat notice for `diagview` (#1355) 0675cfa Resolve Hessenberg ambiguity (#1354) 3393398 Fix scaling unit triangular matrices (#1346) 6e12619 `BandIndex` and inbounds-propagation in `diagzero` (#1347) 87d4c8b Test: prune old LA based on ENV variable (#1335) 2148fa4 Fix scaling block unit triangular matrices ea8e858 Test Diagonal functions against non-dense matrices (#1316) 2d35f07 Update the docstring of ldiv! (#1344) ae208d6 Update generic.jl (#1343) 560276b `Char` uplo in `Bidiagonal` constructor (#1342) 58003a5 Document SingularException throw for inv(::AbstractMatrix) (#1331) 05b703b Update generic.jl f0e36ea Simplify logic in Diagonal-Tridiagonal multiplication (#1338) 7f354f4 Make `fillstored!` public (#1333) 2702a49 Change `1:size` to `axes` in bidiag mul (#1337) 6e0a996 Move testhelper imports to the top of test files (#1336) 5165fd3 Unwrap triangular matrices in broadcast (#1332) 9d26faf add missing methods for division of Hessenberg matrices (#1322) a2d52e1 Change the implementation of `rotate!` to avoid unary minus (#1323) c9b6456 2-element `size` in allocating structured broadcast destination (#1329) 841c4b3 Warn that inv throws on singular matrices a305c67 Change constructions to assertions in structured broadcast tests (#1330) b14390e Fix multiplication with empty `HessenbergQ` (#1326) 9b4259b Use matrix size in structured broadcast 9d139f5 `iszero` check in hessenberg setindex (#1327) dccd6f8 Fix empty `Tridiagonal` broadcast (#1324) 41db513 Eigvecs for specific eigvals for Symmetric/Hermitian (#1268) c33045c Copy matrices in `triu`/`tril` if no zero exists for the `eltype` (#1320) 3396f0b Avoid symmetrizing diagonal in hermitian power (#1321) 95703b5 Refine column ranges in `_isbanded_impl` (#1267) d15de4d `Hermitian{<:Real}` and `Symmetric` share conjugation (#1319) 8695b51 `CartesianIndex` constructor for `BandIndex` (#1304) 6e8f9a1 Symmetry check in `setindex!` for `Symmetric`/`Hermitian` (#1317) 5d3d02a Branch on Bool alpha in bidiag matmul (#1257) e4e8c19 Only `@noinline` error path in `matmul_size_check` (#1310) bfab205 Band indexing for adj/trans (#1299) e7a8a15 Use method deletion instead of custom sysimage (#1312) 88dba5d `setindex!` with `BandIndex` (#1259) 8e6dcfb correctly forward `io` argument in `show` method (#1306) 799ca1a Add `diagm` example (#1298) a145117 Check types locally in triangular tests (#1308) 2e428a5 Reduce allocations in triangular tests (#1309) d550716 correctly forward `io` argument in `show` method fd115f4 Fewer `MulAddMul` branches in `Diagonal`-triangular mul (#1272) e30c9c3 Precise `axes` in `generic_mattrimul!` (#1300) 0a84e1d Inline `generic_matmatmul!` branch in strided triangular matmul (#1262) a3c2681 Fix docstring for reflectorApply! (#1301) b5cc56a Test two combinations in triangular test sets (#1294) d451800 fix herk with complex α (#1297) 537ee46 Zero padding 14ac1f7 Add `diagm` example 27de9e9 Unaliasing and short-circuiting in `copytrito!` (#1287) a32a281 add generic syrk/herk (#1249) cab0dc6 Compile-time check for zero alpha in matmul (#1293) 78e6156 Treat real transposes like adjoint in internal dispatch (#1296) d21ad8c Matrix constructor for triangular (#1282) dcf579c Multiplication instead of division in triangular eigen test (#1295) ``` Co-authored-by: jishnub <10461665+jishnub@users.noreply.github.com> --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/LinearAlgebra.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 delete mode 100644 deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 create mode 100644 deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 create mode 100644 deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 diff --git a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 deleted file mode 100644 index 13eaae101948e..0000000000000 --- a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3a0109056e5135fc5f9c9ddb2c43c531 diff --git a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 b/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 deleted file mode 100644 index f1ae59d2204c3..0000000000000 --- a/deps/checksums/LinearAlgebra-07725da80be389e170cff75cf7157399c2449643.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9543ff47af4fe439e29119e73759ef06a0bbdd263a39cd323243c6f05ee44eadf015c335aab60b43d0f83327714bb71d316b2f4ebfaee205fa5368f2ab856a78 diff --git a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 new file mode 100644 index 0000000000000..5d2d7f6a868af --- /dev/null +++ b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 @@ -0,0 +1 @@ +00d7f962a817b254a248ce7078290c37 diff --git a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 new file mode 100644 index 0000000000000..4f02c121c62f1 --- /dev/null +++ b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 @@ -0,0 +1 @@ +d7bbf0b7a90cb46ef63e69a0adf2c50ca52c81b679b0d9deb545d98c4400500746ad76be58e3ab8319a6e2d17a5e24fd10e5a0f4ac29e91f3fdd9084229166f1 diff --git a/stdlib/LinearAlgebra.version b/stdlib/LinearAlgebra.version index 243522ef799ae..5136a508fca27 100644 --- a/stdlib/LinearAlgebra.version +++ b/stdlib/LinearAlgebra.version @@ -1,4 +1,4 @@ LINEARALGEBRA_BRANCH = master -LINEARALGEBRA_SHA1 = 07725da80be389e170cff75cf7157399c2449643 +LINEARALGEBRA_SHA1 = 3e4d569804bdc8eb891ff1982eb9e90246656d0c LINEARALGEBRA_GIT_URL := https://github.com/JuliaLang/LinearAlgebra.jl.git LINEARALGEBRA_TAR_URL = https://api.github.com/repos/JuliaLang/LinearAlgebra.jl/tarball/$1 From dbc38d6d34947219090fee160808a1064f30b4ba Mon Sep 17 00:00:00 2001 From: Zentrik Date: Wed, 14 May 2025 12:40:40 +0100 Subject: [PATCH 265/662] Bump LLVM to v20 (#58142) --- contrib/refresh_checksums.mk | 17 +- deps/checksums/clang | 120 +++++ deps/checksums/lld | 120 +++++ deps/checksums/llvm | 726 ++++++++++-------------------- deps/clang.version | 2 +- deps/lld.version | 2 +- deps/llvm-tools.version | 4 +- deps/llvm.version | 12 +- src/Makefile | 8 +- src/genericmemory.c | 7 +- src/jitlayers.cpp | 3 - src/julia.h | 14 +- src/llvm-expand-atomic-modify.cpp | 28 +- src/stackwalk.c | 6 +- src/subtype.c | 10 +- src/typemap.c | 21 +- stdlib/LLD_jll/Project.toml | 4 +- stdlib/libLLVM_jll/Project.toml | 2 +- test/clangsa/GCPushPop.cpp | 8 +- 19 files changed, 572 insertions(+), 542 deletions(-) create mode 100644 deps/checksums/clang create mode 100644 deps/checksums/lld diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index fa7cddc705958..9b373e0d7ab62 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -58,10 +58,6 @@ checksum-$(1)-$(2)-$(3): clean-$(1) # Add this guy to his project target checksum-$(1): checksum-$(1)-$(2)-$(3) -# Add a dependency to the pack target -# TODO: can we make this so it only adds an ordering but not a dependency? -pack-checksum-$(1): | checksum-$(1) - # Add this guy to the `checksum` and `pack-checksum` default targets (e.g. `make -f contrib/refresh_checksums.mk openblas`) checksum: checksum-$1 $1 pack-checksum: pack-checksum-$1 @@ -100,7 +96,7 @@ checksum-doc-unicodedata: all: checksum-doc-unicodedata .PHONY: checksum-doc-unicodedata -# merge substring project names to avoid races +# merge substring project names (llvm and llvm-tools, libsuitesparse and suitesparse) to avoid races pack-checksum-llvm-tools: | pack-checksum-llvm @# nothing to do but disable the prefix rule pack-checksum-llvm: | checksum-llvm-tools @@ -110,18 +106,21 @@ pack-checksum-compilersupportlibraries: | checksum-csl pack-checksum-libsuitesparse: | pack-checksum-suitesparse @# nothing to do but disable the prefix rule pack-checksum-suitesparse: | checksum-libsuitesparse -# This is a bit tricky: we want llvmunwind to be separate from unwind and llvm, +# This is a bit tricky: we want llvmunwind, clang, and lld to be separate from unwind and llvm, # so we add a rule to process those first pack-checksum-llvm pack-checksum-unwind: | pack-checksum-llvmunwind -# and the name for LLVMLibUnwind is awkward, so handle that with a regex -pack-checksum-llvmunwind: | pack-checksum-llvm.*unwind +pack-checksum-llvm: | pack-checksum-clang pack-checksum-lld +# and the name for LLVMLibUnwind is awkward, so handle that packing with a regex +checksum-llvm.*unwind: checksum-llvmunwind + @# nothing to do but disable the prefix rule +pack-checksum-llvmunwind: | pack-checksum-llvm.*unwind # override general rule below cd "$(JULIAHOME)/deps/checksums" && mv 'llvm.*unwind' llvmunwind clean-%: FORCE -rm "$(JULIAHOME)/deps/checksums"/'$*' # define how to pack parallel checksums into a single file format -pack-checksum-%: FORCE +pack-checksum-%: FORCE | checksum-% @echo making "$(JULIAHOME)/deps/checksums/"'$*' @cd "$(JULIAHOME)/deps/checksums" && \ for each in $$(ls | grep -i '$*'); do \ diff --git a/deps/checksums/clang b/deps/checksums/clang new file mode 100644 index 0000000000000..2d1b19d45c84f --- /dev/null +++ b/deps/checksums/clang @@ -0,0 +1,120 @@ +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/744ad2230a1594acee8781be442cbc65 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/118f67c2fbd6b11e3d850a38ea331f10403cbef563f22bb1102ff5960d1dcb3e264dcb36e71534828d422e669de1495acdc14c9d452991fb74b68810acc721b2 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/6b208a76bb52b83bdd3239c93fc37876 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/8eb5d93c894a830321f58ab18e4aad58a52895bfe8976728249758f41b439e249beb35990daa2fc635a4fd5369aa89e710771d06d7242ee7c34002a18c939629 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/69d3fd7ba2103fc9074af330e30b0397 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/f29d2552626a7f3a91a50984bdd396beae9beca65d119e3e25f427334723d40dea4672e27e6b2057bc1ce354eacda57dd06b6718347d9d075f455dbffbd972f3 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/778dcf8e95ff94a946726eee2987f3f8 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/31ac240f96d0081a20284de2c18c919ef74cb1734426051567eabedc18752f18550a1fe48ccb66fd89f3605a50ded57b61631aa48ff528c20a07dfb73b2bb063 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4239b43accecdfc6fcb96317a84f267a +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1df612a15d6e9f59235a0d97f8aec1b1252320077b8b3ddc5623bbfa531b7b089663f4c826e1b881401715a288cc170d79b9953ad987caabda234b3c1fbf9efb +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/bc3e688489a6440a4eef0cb3731d979f +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/7c0b96a9b927e3679ec79a45db388a3bf79d773ddc3d85da9f42a92c197c5e9886d13cbf00e7189e24b1d3a2ad942ce7c0cc50d3666a14c6e051b1d01e639145 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/770f427ecd0f6900eaab9bdcdb7b1964 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/8263c16d644c94ee21d5b46c0b515fad5b73dab7bd0fe658eef239bdd83b9c10e26546706cd22a892ea9d8d43e054123b04c68f586407634a4eb2826ea93946c +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c02058697c68d62d94b68392b00da335 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/ca4de5088f9f016ad0b782b9816a06ddd2dd2413e28c587cb8968308b89ef1295c7432de271c1839a0d436d670e06c64b7989f44f53ffbfd1ad82ca35ef910f7 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/edba25d64e92d9bcdb6380c92779ff68 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/2d080e0f159ebb3e91ade714343f597d0da9733c361ca998dca9cb436f5702604ca0c20b6ec547b230f77f963d232e38cd139f7ed4203c3fca96bfbe9818bbee +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/414dd96ffffc43202293d0f2987ce478 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/41aeff9a58036e38c5d714ef03a7850b52b93dd84e78327075dade971027a3d714d0f3ede85144a4ec7b10449ce5d5dfccf6c1123c04a7da74abb5b25a38ca79 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7c8b3eb507228e61f7251d7e8b6dd578 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/0acbd2fc5cc95830ba1cb2efa24b0a7b0bfe66c9f2592a8fd166a2318727346fa4827097601dbb9d915473b90e8ef5cd88c8d0c73947e27f8ce1df9a74879d03 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/74f4afcdc240f3e3c56edc6058f04530 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b2b815a323994b3bf5a386d10d126f5bd5572ea4e883c5b28ad68cdadab4c8c5d4eb7ba9799eff97abe556de65f725c62426dfc44492240f444206f7ae666abb +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/ffa2f021083b5d27e9de06b618364079 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/9295f6f080eeeba21570165435e937a7347867977b6a5e5abbc4be7150396808e938b27e7cef5b0ddd9d1e538a04e17af795fa3c3508f179ac8916157fc61f52 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/4b33633c5f9dbdede7721f815c39b9e7 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/a358b4898a34a2882029652769faa8a4ce0a2f5ed36c853ed40a5875797dcd14ece37dadd620a5f7a25afec1bbc426fcf5710eb9be54023c03cb0c50087ead9c +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/25076268c657945ed492d46c2dd819eb +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e8bc9533ef8fdbee5eb938f9a764d69de52b47220d8ed97b9a23a7ddcb62206e54e479abeedc14339fb2082ed451f861d1b19eb93830dd0a06d609c8b85fdec4 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/e0a95901306ab2c66a6cbdf86ebb0c5c +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/8713ac54a84e44b1d565db8334befc271a8d5ca8646f46f75b43754c4681f433b8ecc239b3103d917ae28ad258c33f6e656db895d449a0b97c3af3adb3893ac9 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c52e8e1667745e3886223c214abe8191 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/abb67bf8041cf61a81f37f109a30fafc595def65b11d5628a7a7cfd095b0c6b2303913a9149df7f1a2996893233f6f8152a33a26943368c62763efced7ef8e83 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e51f6b1a3bdc815c164440976b3eb380 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/2633d7c1bde3bae99d81c2865644f15e02e935587b7cbadd63b3bc90eb9539dd455dfe592283574aa56030b70681b0e4e1950f17643b352e4053d63e61cfc93e +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/b1c191536aa62bb9f514b0f5615bd322 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0973319c7d8a3f3058cf8b710afc0efc7d28679cc3182982bfb1feee431fb52eab88c3387aed5e351294e2bf4155775bf2b20ae6c35e21c5f123937140bfbf40 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2ddbb22f341c0fc79e02ddd2bc60abae +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/e829147ede7726dfd018183b1c89cf976e2160cf33f5f39eda0f0c9389e31dbe52cc127cbc3f43c19253fcde84ba6d8bfa4eecf6d6a3d6704519b578e77778ab +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/739d03b8f5f11ef6ac54054fc767548b +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/af3fdce8bc0920e028adf5126dcc442f8f8658707f1f5b58ce4179dffe3351b3337e91908ab50a309789a568e0110930fe6a64ace2384e5631f960847ee696ff +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1c1a6e5abd46f5b82d16af33f1f04055 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/d2758bdc071eb5b5cd9bd6e0fb215aca8989429b6fbdb27a2874b1b3c7988624c311d9f71cdc76d8fda2fae113ada8d18212180ffdef80071ea25123686d16b7 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/fc073b15200b2a4514289c1670761a32 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/7be72926f46764b7b106e4a7623ec7cf54da57feb9b12b4310ca565324bd277b06197fadb3cdf4aeb8ae804b5e68c66f8d18d0b9e8c07a0b5f2acef6befd7001 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/894be9034932a5ddd8868030470aa1fe +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7460e69dac9c61942494d4f2a48d3cc52e4741395b4185fbad72b418a86a825f4b872e80b89324e465a2b83e8fcf3532aa081f9706c29c7040c61ac6fb1071cb +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/1ad205872d5fee1e4d5821c6569ed8b8 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/e3004ec1a7e1424e1dfc44f3b8f4e87edc1c9d490866bfb22fcbfc79d9494e5cbb73b9d7859de061ce2c8bd08715eb1b5d9b0ba23e0231f4a607ba7097370b88 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2ff49e5d12a9f5db5758deb116fe62ee +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/29ee0cb52dfe643f99f92a50e0f60d6fc35aae8311e0419106862b9d14c08ee162fe7b0a2b04a291176f72cb789d006eae3aa41a50e9383d6acb03aee99c2c9e +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/89c6e3312234835ae172ec9f7697b888 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/44e20f9ef9dae0e961bd5b469e930cd4d701f228a48c867d0655a7eff220bc060a8e769a520af7dcfa1019bb058516c95ed9bd25564488264604609f76f60a68 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/ed6b213ee57408a819912740dd5d2248 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/43d68c8712bff0fc260be31df1ccfc657d725971f5b115645d5efeafeaf10209401752c5d5fe0b0f3e4be003d68a2d75c08f97df7fb8d8aa0ed1a49c2b148a75 +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/f813d4d256ec68b000be8e7dd24172bf +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6a433e3459ae36b6a131aed5fde1bf35af7abf825d5d5014cd82be0117087fe138a3d3ee883c12a56cbfe5a0ccd797508dd38587abcf80bd1e06847a522c6acf +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/1fe9b72f78ac84c375cc5ecbae608e14 +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/51663a78e81635ae0a65fb0c7c4ebf66080fc919271736ab0895d3661686d500c6dfcf4181865ff6e0fa645a359a24224f84e0d2aa6c9b9a81431e4905f81cf1 +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a3bfe33b1a5ecc815b81cc5719c5df3a +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c7d341cc38fc895b6cd884bffdbeb1d2cdeda6cdd90fe3e85ee2fc909ea6e6c2267c1d165035f8f8c1ae7e5d96eeec61c65e803149f267cc1ea54599c49463ac +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b77d8afd01a62b31eb5f62a39ea7c834 +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/141248c50ecb5178722c2be6540e12e7e62f99903d06bf7f1e401f9ea27ccb42dd146aa66e08c790e9fe663c167a3ea48809023daf95cf7e32fb0f177147f4a6 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/08b9ef36900ec33066898bcd173a5d33 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/1e2b9f4011873cdabfcd3cd154bf47b93da682f4ffc1bd41cadcee47e693f921ea62d7792f23b07793b2731434cf6cee91bd11432bab6f6b0ac8969aef507e84 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/c48a8ee67bcbb13a5263f79e757d2fbd +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5f24c4d8b76fbf9036bcc3344436a5906eb025a9949118780e4586a5da6afe3d5a326011fdc01bdca7595940d2eb94d668dabf2df65af33e31637d9ce9ca374a +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/ec43a0ba8d6957e071936d85eb5e360e +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/9a145a4ec60f3a3826001cbe4b089e7053e10a9443182145b44fb9b0a0c9051c0d277632e17eb989cfac9746441cbe3fd3d8e68726e48727d63aefdcdc1d012b +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/08507753c774304274f1531c36f12426 +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/73fde12aec5fa8a6cd1037767a24dbed2c496071b2dac6df98624d01316c3cde85f7fa176e9572b5b74c2f2a7010fb2545bc4b949d27291c926e749284ddead6 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a118c6ef27c7b69f2877b10f532bf2e5 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/620727a97254c81d16ef4e1bf3ca4013aeff75c069f12de4b5d013e1df28e9d1804f2dcac95c116c19a537db39ce190064a2e39db9d2f10f5b83e93145ebf7f3 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/13a9e37303fe1e793d2782690159403a +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/4a24a1cf630dc16c9dce2b9eedb7ef6327b82eed288cb76e42d227db9651694acb52b9c59b96736af7a19a5d4be122935e5694a0384184bf28d2b4419de8a9c6 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/402a534e78daddcdf42966ba51696663 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c3835b05b621b9a4f0b7a2aae44a7d53e64164010a054785ff4264a2a369d1f5b9df84c1a4e2522df55f240790cf1a939becadd13893c70c3a9139d2703fcf5b +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/9b7a6b511b728972c0ccc3e1b33855c4 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cd9fa85402fbe04ee55f6b69ffb87d3a4923374c40176199d4ffe10f8e8aba54e24645ac30ad0e74ed60e9ab5119bd75f089d8dec4a132ba29a6960fbd0338be +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/fadabaefeabb7c1df445473ed5ece280 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/aa6556a9d27703fa8b49d682c60a1ca843ae887b400c3ecdd3a78e8ef456b1feac5b1b46cedc6124f04760ca485d028defb404da808a96bdeb0b65a704928d06 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c7dfffb39c3d193785b4b31ce58dab0a +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/3c64e13b5846f21ab9796326f2858eadd77bb8825a840ed14383012e651f93c6d74fbed3cee9c324cd3d9256feb243dce15e10bd7962ffd668fe83c9772c6beb +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a4596e099c5edeee4a204305b74be573 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/bae493823a3eacf6a6f7c0c53a78903425d8ea71e846e5d28b583ee067cf9c8dc7c0103f53f30b08d271f4867a2f3513a187c04b4e9136eb7c4fbe1334be5240 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/f3d54eaf5770a465a510c4370ee96548 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cbdb8e0c1592f9c4bd8b1e6a15ed15decc92502009d11c75be7ca8840bea066f341e6bfdd4b60dc51e2098103425b324096870358bbca36f8e3af9eb4b1b9559 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/d697a3ea0444d2a583ee37b609ddd545 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/49651c7e38e41aac7645c7f8e8fd5f789b742ee8e945b59181675b40bcbfcba4aea8d8d54e7dd703ee40fc91cd76d02f1f7999ca040beaf49c2d605a1f76b818 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/43a0fa128b939182277f285b9d6f4007 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/c7ebdadf877fa46f55e2366b5ccc18499a15fb01888e6373f1596ba10530c2c7d60f248def10b6c196fcb5d50c9bf9e27bad4b7a0c3e5291f411ef36a604328a +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1a43bec0e1fc4cb6b94885d4201e75f2 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/76295abc2d2e1c4c2f56719f6926dee6002f09472266f77be2d097d2f2f1da706174215d9de59cf2f88807e6df3dd38050935e82f8e66b0c41f5cd8a1dc2ba2b +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c3ac86b91ee81032410e732722be1301 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/dd370f92c522ef8fcaf85c2a87cddfdff74fee92bcff8c3b1d5ae8e12bb3f90186ead91d22808f96d1262ca84806938506333fb3dcb5bbab51ca845de6d256e9 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/60d67e563b4f8d5660692da70a399b92 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/55c135642fb4b9e61eac6fca3cee9b2d0f9f83456f9670bb90428e1fc0c28642f285a2dfce9cfa32a585f9174645ab6b2ff8943367b9653792233f4c32a78b0a +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/2448604ea7a346df8532511e36d9a59d +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6627693edb541e700596212ea580d18a064d8e765642353d4b5e474f394bd6d9120b781f5c02840bd327b084ce0228b05ff17a58bafb6161b933a789759eb0ac +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/de3ab95295e91a8dc55beece1c094489 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/4b2f1d6026fd8680a5929de517a42bf84f66fc6cbd32f8becc8b0601d3b67de66c78ab67a94662184ba5becc28bc4fcc2644f17731b838b8de46f909e4773252 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/7d3eec07c5d71a2a90fff278116f5b09 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/8d837e4f24fc80f3860716aceb23961ef33d499cbb6ffb5bc89b422b795ef46c60a787038e79a1728471ee2f0807906b28e9aed7fdf3772fa13d99afda0d2332 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/82298efca03959f3c01c91d0da4b8b07 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/90bdf33dfb8eff0a3ad9cec730067900283a5e31f05a008a873c8673a603e8a3e704048729c15b841f0adc29c96daaf1085cdcb3029cffd7d42f8b99116dcc86 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/c9671bf1a659a52fa7866716472dacf9 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/7d6296bc6c44311b763d70a3a2617c7d1e7835a0df1c232b5de453611d71eb42dd43f659fa2bfb320a100d87594e71647b17c7f108772d444581daae567b152d +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e9457d3c788a66624a2d34c20d1c300c +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/4aa67eec6c4f8db62fd88e568f7ac5a5b24ba5ae92ce6a779dac570a4b95a373bce2ca70b46ae1f919459e44697f9aecc861e9375dfa39511bfdb43b8bca0539 +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/21dc84915705793f2c0045db6430391b +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/d2a1d55d1cc04f4ea0c890dd52ac2c0961d64c3f9295a2ddc8b681f8a13cfd32b0b05d34aabbba5708fa9b0e68f5802e33267456dbe42154771d5c610d5d1f02 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/37091fed96267a3865f837d637ce9140 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/046ac9f2414eb9e163e7520d7d57e2a6caf5279ca6bb7297eaddf5b36b2218ad83ea222c276455111bf1b32e16a56e40ea419698ba2f18f93ec271ffda9ae433 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/4fb68bfc1f784051b212f13f1c766aed +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/c32936a923b4af1106293e24ea07311bfa728b97609540f55bb65acca8d2fc937fb718796cc785fad2260c22631e09e96782ceff8a7e3b2b2659f6eb55e3c74f +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/edfca70811c32f08b758cab451b6d254 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/8eec4b27eea12a40c77c47f19fc0ce50f88aaf83822dc76c274ee9b051da8b41d37bf4a7fd3d0a6124f1a6fd58c52ec86cd8ef72f7844a688554744add4daf1f +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/97ecfeef24bf469e594360dd1cbcc177 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/e6ee100c2a944dd7d6cca577f8cd01fd1e0236720310667191bfe09990a516b09d663889ab29d8819f8341906d397d3871c28f351c88c144de9d238215c985df diff --git a/deps/checksums/lld b/deps/checksums/lld new file mode 100644 index 0000000000000..6e8fab41edbcb --- /dev/null +++ b/deps/checksums/lld @@ -0,0 +1,120 @@ +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/8cdfcae09219dd5c42a575ee95dbe933 +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/050783b4d922a806a566e6d5e5afc77c5d0bd935cc8c5538f388b6221d37f387bc8e81682196c8f07b56100ab2a0aec331738dc5815925cd09bb1b5b0cd68ebd +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/4beca49829fe5fdce2674c778df340c8 +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/c7593ade99be7e6da301845525428ac8feab063b2349bce266e02325dca806e875fd8586ba1f9252bb5e7f701c1e750dc26dd7ddb4fffce3f48be918d89152a9 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/34d2f188679bf5abf4c28c7ff4e30703 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/e89e8156ac291b3373f5b723638affefe43a7a6845ea49720d6e39387e3cdeb34e4790de5221df1ccb50939eb8b41141af94fd4436bcafca49e24e3007bf1c16 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/3a431153ad307ff728b3c07f10aaeb17 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5526f8e004d93b54689f47b62067d689ffc5471289d9f43f2ddacc6c17275d7133e86837de1895b40d0cf969006d07b8fed2ecc96b7b65472d6fd4972f4ac082 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/e9d0903886a7be9fdf5421540899418d +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/42f29306911257a2982b0cb9630a4ea81b50add6567d7135c9f3098162c699c2d234256cff80ef789a0cdc6ebafdc2ceed9009bb84aa18bafdba5d5d69874210 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/97ef4ddf834a725a7d13faa1ddcc3906 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/09a557558f51b3e22fb8a93c5dd458c5d0b397858df2acfd2cfcca11e8fce11666eae867b8e3f51c3b296a175d72fa5716135f0c0612c859bc293d8fbf98d226 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/8e5b12a175c78b24ce43e442c735924e +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/00492672666c6ece6a8273811e1cd0ff1a4fac3f32ae33ff003a978c44d2f6e1024e24f9e29bb970882f766f922987f7fcacc79d2c81c29acea5b397406fbba4 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/b68c5ed9e9f52c3954d1664c49ad1755 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/5a2cb3cd347ebaa05c0aeef636e8dac0878114e73004f2f903608a87567a2a4bfd36d633df7cf48e20b60965fbb2c8e9760180f0ab03e51101775c75029f5a97 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/53f810e9d3d0442f2290954c70f93f37 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/417dff21f10eed36ed0b911341903914658240c157ffece632b2e0d28b65b0aeda376b1dbeb1f4677ee0b434c63d763fd44c9d6195b192a94ca575314fbfbbd7 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/25713a33485613c69d718518aeb49259 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/4542b3d1f98a6357ff7fbe686803103223c82c56096696bc90d89e1378455d3eda19fe45835f6e6d5035e754ad5aa51f24678ad20ec24fcbaee873cc729fd189 +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/978b490710efe5019486c88d19dece8c +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/7b47a5216b765f7c54a8f48006d351a01ee38ff379b003f62880894b4a5523c5842fb1205edfc2dc71420f7d19cf31f417e6aea4f6b30fcd905671dafb73dd2e +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/52562639d81bfca12e3586740f4b35bd +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b230117d657a8d18d0c43a2d3da025890f4df3b11ceabd7fd883621c3baff6ce1e8dcbce213edc2fdc733a59022889619a56fa7c1d773d6039150b6e2e44eea2 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/01efa3cd3ecdfe45803800b8b8bcddf8 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/8af2ed095fb79e831b613cb5ad83393b5299cd9df1748f1a6b5771c51d3df2d14af286d00e0fff432b2e97be5991bc6d515fbf2089f3f58ea0cefdf87eb0622e +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1f5c758df3e347960a5429ec7cb84849 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/1a38fd706b55e202044db984be82f96fe21da67ae39edd396e49a5dc86daaad47185877a1863f7ec21e1303b103fc533b0729221fc87533d3c02efb9e05338c6 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/bc0e9ff37b2bd42460b2627e973f115d +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1926cdf82fb5807019647133a602ad34a57dc6b45ef072443ce0a5eca02ff7466d5daa93b45e5fc861113e2aa8818f789afabf86a99a431d73bf29e5c0951a10 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/5426e6022fb0646d61a8b48f4e91f50c +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b4a0dd3f2b3b9d4d643405ac9ccac3448ebfcf199568ddf7079b6b77c5f29b42d9210ac2a6881416449b50d4d52b822dbba3ffaa007cc79028c8ef49c07f0c31 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/08563d88b2ced2f5ff6f2a876b10e88a +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/4bb9d42394c5d6895dae473257af00950692be4d07a7d393e70d8a7397535389f095c4c65d5b54342e51068c76157d9737c404b01e18f7648d54ee4aaf1b36d8 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2c20b768d5c77175613b59104b848086 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/122177d274de1867fd383a4d90e25564d5079f9e7cd09e5167fd449ab0d0c5cefaca867623921a6163872fccf6b65dd0640a3b22d33f30412e3e2f458f9f4f9e +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/d301952a3da16a4d6a32e5027d946982 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/90dde9fa203766ac128c80a167046508b9a1e31121b5498e986e469559876a6c587c01016aef6e1ebc328fd15f8f479dfeff94042f8e172b6a4e8582c3696995 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/364812da3ab7e29e11c14503d676d01d +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0a0a819023643ab2d06800e7dc75bbd0b738b91a11e2257dc752f56b8d7e6438b85ab5695daa72672d4797d871b912c210b6df2c80f516429f9e8335a97a18bf +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7d77b4217d5bdb192dc545749cba9fe7 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/16cc0e199d37d4537505b1cca4e0c648d5e9e49b2135dbb2b96355501d9121ffab32ba1a076ae1b77a57378be3bc32822d2109bd85a4c6a5747e50e1ccac3965 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/efd07e543cc5bbb3767f9d64c83eb385 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/5ecd6029bec97fe1cb56dbbc1b1eea39df5c049b23465c8eb1232c281e733d59f730dc38107368468ad56a07b3c864ae4a0d51fd2503d3d4077923e7f1ba49b8 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/cca1d7d104f95c432ddde95969f43c09 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ba73da3f31e1fdbcde0289e399fcd34b7064f626d32cab724559f8d0a893287a1aa67965be5ef85c9f7e4ba645f338e852e0f93ae8942706deb5e7654a6e38c +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9414dd3e1cd61b52f3ee312e5fa4faec +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/d60b7482c276a76e288360a37c885689dcbea4a00b555768cad635a5d04227f294e85fbca6f586036ee3a33ec77bf9f9f3bfd110fcc697cee8f4586935067ddb +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/22fee78e5d5fbfc99d426382d1f1d7df +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/195c19aa3ebff8795cc6c1862b9d2c8a577809a94da20a860de57e4200baac1c2e76683c278eac6d54db4afd976e11dfbf055a008e7e44d22579847cab7f2eb6 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/356ea4ed4353c3d7245987e55a5f6897 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/fc071585b97db5900a6d72c109d78596c731a2a0971f46ffe26b4423bb1da3488df350aa5eb9e7ee03b5b2fd66c3c80950c764e0703a7c6fd6a4015af05b994e +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/3dfb7a907a0b18a971e4227f82009dbd +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/bfe5601d4d4654f15330eae9168327f9dcb91bc658cf7514526776d13f4b74ce95f99a5dafe48d59deca645f22a4f5b6103bbb0e35ebfcec85e82767ff5543a4 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/fcc8d611a6bb893408da646268faa061 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/7d09101bbd6ee53bcc4ecbeb0883a5a4fe440de49902595c53e6f56d8e0c67a350b923210e17d22618569ec087ecabc097719307ce18db0cb7cea7c16e0b805e +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/74faaf27eddb69690ab6bfd66479ab05 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/23aca5a1f80fe35fd1ce7a3ea563bf490215c9e2fb82dbbe7fcfbecb6b5a73dd1643e019d0446c57c6f31e9f7a1f5c52c8264555f23ba3c5fa8275edb6715ad9 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6ce4b175bfd0242be28b02d22b8da351 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/e0b0e9f27f0546df44985e6b1cce85711efa9c3023d9bad88c25111ae86107a72af5c2921014173f8449eebe9f57ecdf6e1afb0b10d5593ce9b90e0415aa4954 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/83c135cffa8563f67f0ec1bd8cf271c5 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ab03926e7be1691e9e79b15785caacd00de08ee666a0fca264fff5fab1c277e368d7c6eb77beca32709fecdac730f7d9df03191ec2240d0ba8bbf6ad9494330 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5242e12e1c74e2dae73c6b4b7d4307e8 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/0f88cb79e838b2b92111473638fc7f2e7973b6301597f80caf66e378077cd306768d6cd18035c26262eb6e79525f1690dedc6d6db2b7dfaa1483ab2008bc7d9a +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/7290bc875a2fcac989aa8fc3122ae6b6 +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/a332d69b92ee11fa2ab5ba909c8d3632ebda2bb86d74b1a4a93caa62d6cdff5e21356d92b6e5a5744a5d0b8bd94c0fa1a4e69ada9c5e006cd35ddb658e37352b +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/645a495655eeed199af92f26c0c5325e +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5c6b16c22322bd92a0de52a393a87085d1a3836f85a75257675194a6fdf6783aea9e77670e176371cceb637b9450c7c5b2b4d03f82e1580b813548b078f7b254 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/f060e69be07766a078cc9d8158abdde5 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/e4f006aa955304144983cd3437fa2ed213e0e27d904d69deb916ce82f352809311e443522d31d928e7f4c1907c0bc82828d422f5b1f0c9647abdc0c4fae7c6b4 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/ee06aa4b93bcb60c4434339eb68715ea +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/cc6f319855af8e35c282428bb3c9012f0c4c9c4a8dcca1065d2d88c732bbeef6cd21cd42708968335ef4625fcc50e80d126e1caf545dd79cac9a23b4753bb16d +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/80757f0836a949560b66c03891472b07 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ab3f0bad55b902837cffc1e747a39a7a0887ac5dccbfc6fe31571de58bc88340fd2b848d7cf806768e0dba4b895ae9c945214757789c9ef134f4b7e48afdbee2 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/716af34d7c11db49857a31b685bf30df +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/9e05fd855b39a8b33781902d801244471f38bcb766b234dfa15f71f3a3bc0a1500b2f2a56c137b71edc4c1f3bd456145ec589a188640a2d0eb2ece0c3c361075 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/44f9e0d5ca28b85641b6c06daab74733 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/778d85ab80755b95a6b2b61b676026487cb8dccde3c68a71935e9b3db4cd113577d5072d4db1d09f0c7233a24bbca5f1e4bc1fc192b3a7839fbdc0a68b649b2d +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/d2b52996d60e0e4f3b3012d481840842 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/082980ba92a796db4d9b757445ed9400099f1b2167e79b868da3aaa5d3d2e627fb0980b8e9351d8f2fce9845839aca6d386ea59a1639f5aeb5f7060904f6cc1f +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/df3f2fc3722ddf530462247bc484e6cf +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/cc881793eb7f2ebd9bd37058b50e87afb3fccebfa545a3422ba2019d51d17457d4a0cb6a49b8c91e74e76ec9cf923aa1d8ec0d8e8e285d149a521622b4c0add8 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/05cebf19f1eb5c1871efd39de205d490 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/0cfb6fdeada331e5d0ede9a7c50b1677d3b4bb5a36149a24f957cca4944e6852bf016f4ca2aa99dc7d9a590a28dfc2e02399feb32846064bf3e3065920400198 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/bf911ffeef7eb44907d288b0cf18f434 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/66fe8f97f4e200eed12d3e7dfd79f1062318af5f0d4d96fadd389f7582146fc45a6dab593d8346ecf3bc23f02a84035a22ed42cabb7222baf406890a5f80856b +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/45556dcd1655ae34564a17bd4fec150e +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/bbee6388e40007eec26c766dda6a6e4f3ccd84656157e1e81ff103295cc1dbbb42db6c858f9f5519434945c27807d880934ad791eeb8e13579003edc065a272d +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/0647f954526b0ff65dc35a1b65b4b756 +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/d614f0ae0da90505f6d8710077c7e9a06e815f4a7192c167f791488ce5fb8d0ed2e6e3cba1262c8d78917217315e820dd180f8838cc9fd739eb4dfe8b7b7793b +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/e13e78f28632eb19ed1451ef1f243328 +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/d035ee204120915c9ff7fe3385f1066993cb7c0b21df970be62ff573a4adda395341b0539984f579c236a973e18f05ab8e8a4befc2bae34fde08b03e9b975807 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/6442326b50f9b7cca3edcce9e671e544 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/294b2bb952ebb724c57c9f3f25e66a87224d9686053b3e4d517ca82ad728093371d193852450b579a586bbf2c2e9b4fbbc703657335524253a23a3106ebab9cb +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/51b96db9df5fed624bff8a89bca2e96e +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/27086b42e8ccbbfad398250853cfb6941f5e6a2d81d6dfeb99b6a08e456c017cba3ec7978e4cdf2f48b3ce771b7015bac7e5953648db0a1e65c601e9dca76b56 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/9485f990275be736e6ed8fe9c905db2a +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/58e4f4ae61ac789030322125ffb6b11a16eda792d1f9e8d01aadf5e399375c40d263184f05143e2d01402bb60df068c6028ba2c3bc9757037f776c0f63de1be8 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/1bccb2ceb7070060122dc2c10811be4b +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/06adc0285afd6f5de86b533ab73cc5005769d44562e748b55062ef0c195e8d6b0c287a20b99582e6096ab0c989a072a1c7bb1d347a37cbd9634d165fdb78c7ac +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/ad160553bf0f20984759686db7b87243 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/f64d9d3ba6b9595ec745f9326eb7f1564d99330975e2997f387f969c21bbe757fa8f63faf583a115887a311f8806e70ec3b9a87ba6a8894a09c0e9fb8496c357 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/0e5433d428daca35c954fe261c82a874 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/67de9856e05cdfca1ec25ce9661337a04b01ee49f6f2eb05ab970d19ee585beea425c5d8c6cd72a95c8d5da48ac7968a1ac2ffeca9817fc695d2be020149e76b +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/c8a10b058232cef8e9ffd0d1ccfa6c4b +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/15d54ad7337f337a340a8ed0b2ef2bf9b75687ce691369fd9780e50a3e064e1866295cfc9460889f095501a3d48b7f891e1af22d8d197097640ceb94bd257aa4 +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/4eda180a515b58400fe5230fa309bc54 +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/87c0a59e425768fdcda295ca8323eb6eec2690aa9784dfd8d6200a69820c2dab8d8fdd7497c2b53e255a6a5bd855ec3ada9d8e1f65fd12322833c58363772137 +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/f971f5bb4255f544511e689bde1a66ef +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/43367f52d2278cf8f96bfa60aedca4174e9a97770de4b23f4a586d644439bcd437ae9cbf4a802ddb22ebb6f5411084d1fbada762f29a34ba8ea5d0859f1100f9 +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/6ff7a32c9089f1c0ddd192ff037fe1fa +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/af9e2fd3f6cf68d45a6469f1458e2b3b21ddf8522fe96a6060a511e80e9da562c7c13ddd4ecac0f77d2e432618f617f829514684f9f7cab197adb9ef8f68e01b +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/d3cfeb84b4b46d430a859cd2460f1a89 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/bdd8c1503844f8c99960fd7d72faad2993ea56ba1d0b3af65b37a4fb3df586779e864379cc53a1387b50be55259a8f2f6d9f1f2544c1e09365d2ae65528139e4 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/32d94428edf18af0a887570b1a757a3a +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/55df0a8f4dc2ca2df5926562e40065fc01ef1f9a3f88d5ec0e6dd40458ac66922a6292091c26b48bd1bbb599a081c1e73ccf34093b11f2888b97ff979dc75dae +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/462735ba1292bb622c82ca8652d0d587 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f0fdb9764db4870e4db9bdf2fde99caea1016c3b499dcd406a188a5c973a5d3e804625fabeab8bcbba51a45a503fa9157a02aa9c772538d5a0b8b7953642fc0a +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/0c86dee0baa0e9e71c5c703e45d1d726 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/28f9d913805287661765bbd24c15466a39c6485ab96699db27cf359c18165dc8121e95359065c16f16376369fc76a63ac67a451ab2c53f189b064c637dfdd0a7 diff --git a/deps/checksums/llvm b/deps/checksums/llvm index 01e7d6a172902..efefff6d50a97 100644 --- a/deps/checksums/llvm +++ b/deps/checksums/llvm @@ -1,482 +1,244 @@ -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/d24be00d258e18c03f5dee3f866438ec -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/ba4e7fee70883076b3b42c0999a34ca9ee2070ee25134848c8d045f37f121ebe0843d2581160536ce2e09065a988dd203081638cc63f59344e025fdbd0b669d2 -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/14d4438d746cc21a9e3658d306cd9ce2 -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/cf1152f18498bb407e61b927fbd99c6699fd2f9e39040769541961d6775da2f6b41f2e61591642ba56cb221ab64354bf20973616d30a47ebdb46550df0400721 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/54f14052b86c7ad407f5b8fb9fb4654d -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/de0782647aa7d1c48932165b9b5327a46e9f22319d26752fefa4900afd561bf075bb7d86165e54b032be00ff4169348bd95ff846a5e6f0407832fbdada386c0d -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d18298df54c86f54b95000006fb941a7 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/12232f07920d9b5bdea27d3b465ec5391867279fddf98c10a7c93244a773864999c2f3b4256a94ca80eba5309c9d5cbb0378e1a5392bee429a386da0691b0896 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/2be14c518b0281b18d3fb5201d01e44e -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/4b164e20ca8248aeb2cf4fb2330d518a325cff42dd7376cbd53e2063bd1f297bf6977f1115c8e893a8c37a31934c4012768fe1890601a82a21ac801cb45a23eb -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e88aa91527728e3b04f97ff03cbfb2b0 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/472ea39ff1023d00761364e9f760f26fb1c23b42a437319004ad41970f1a43a5911292e4be2f8e9c599bfac068276287bf7af6b0562e77e65a30907de18a1780 -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/451b4bd9fea31a0f9b37629e550d5269 -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/37332d2ad5022327bfc5d86f7ec8d3fa36396bf830c8a9c95602d8cdf30408e9d6d5acfba1c5f2768b481d7b72aa1bf5ff3f72c1752a00dd4a562042e586111a -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/48aed502cca5dfa35ccee123f4d661db -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/59a0bf98261e24a66bb899632261f639bb58b3129e31795ae181e6bfd21e6825a4bc18779a76da706b520e70b6fa719bc96966d3fa2785c2b95321430f986d78 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/9b7568486b5bf931a109d491a5425e9c -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/7d0d8effef66ece3aaee119b563bc52f5224ecaf417930ffde312c7f781ee06d5f5dac3cd38da43fce0aa9b05342c6a7d5dafd16a642648d10015dc1978dbcb3 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/463a505772082e1c0642dce32dc20f45 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/47e2681e9cc79fa6007ee5675f5cd980fda3f2e2cdbb2b99ff23ceb6dc1e379056ccb85b4e8450527d09d77ea959b1bc53435c664a58d13f86830ec225e67c30 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/30cb13a56111c5e9e6383eee7500f391 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/1a3b794f38833a152bbf293caa867c60eb99efc31336333d83ae4ebaf9b72186befedf175c7061bc200fadd4b6910716bb792a68364baf3764aa79b4149b97ed -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/22a26b9d7a0b713c83845b41132a1158 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/d89ed637970713b54a91112a536dedfdbd3276381f4fe210f67110a9d2ef51f6baefd954567b6389dddcfa753145c21d4ba4a3e27e211c22f898137317597ca2 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/4082236dc1e82a23ee8bc084ff648880 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/f544b692090d1c813ffc5f4fe0e354145b883660f089946d0a98fad1056d2720d6e73bf4052d92bdc717cf8c1f5bf724348c95e87fef789b3bad30cf318b0cd4 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/63a6ee473fe3e516c86a7d6a97ab5e26 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/4ba0f2cbc3e7f8a4bbbcd7b7221b29dc24fb02fafc93402f7408efe7c6b6e5f4782ed18ab466527f400723732cfa6be51995199174eefa2554f6dc290c184a15 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/aeaad6628f961ad8d74a51863155be1c -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/77a3fb05722045dff7b691b5709fef72fcc998497956a5e570d23d7c0975d7ce38a42dab7215158eb37be4cfb8c0eb4bc84be8338b53cc9843a5a3f164078406 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/1c0b0f913f5e70d1b5fafc7f9a4e8c86 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/c7fd5a7efdcd2344c62b9ffb535296b70756884dd31d705ce712f3083f455ea72c403290a6e20d95012676580ab15161addb0a25e1fb70e725f7f1f569bf8387 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/6c34fbdbc8ae24ed6fed62ed2ae9f021 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/6e8242592602f2e147a38d61f76e31b1049ecce05e205b6e1a521061609914e0fd5a339d958d0effa114b3b654d13aa9ec903159a3fdd00192a78f21ee164ff3 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/5a6978a950e3100c637e226ad6a42ebb -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/1b4249ae27b4ed2ba6ddc77c395cd74fd7f55328214a854164c0ea8fc2f0d0d7314fa1755bb7c89b1da6aebaa38779478c9143a3c1206775bfc537949bdc6d9f -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/88e2412fe078d6b3036edb4465407bc0 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/1ad968ffa4e19b2df03a01732678c5b9711023896c2aa99d8ac6b40a614b706778409bac852563982e43216d1f7702941810825dc54da5e1a88c9c6fbe80425d -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/ee0977fa4e2b6d8e5c8114d90194d7a8 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/ee190f8431736483b7beeb8acdc4136bd0b5a1d8d48d44c6f6ee297fb3102a89710e8e0799ab04a67a9e1f64092640325aa553f6b4cb7a0e7e53c3ce55a67e30 -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/76af69ba0d5c42db1c95d60ca4dbcc98 -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/eb442c489b0cb29e083461a099a544ac742d46326aa804a868d839744128bc8c2229d9a046ccd03816565f7b064ce347af0f72088eb6dbe4cb5731e22ebef77f -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/e0c1af7589a6b9961c75f6bc78df57ca -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/b436f7873eb9de3591149f2252fd5d171f05a8dbaab3a6f316e133700042450d267bdcc1340e25762712e227012688285282835937655a534dfe9d1fe7da8c6c -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/25fcac8f8e5b915b25ef0985ba5189da -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/6eb943416307ea668c044d66385dde58aaeed702226da0cc2c0fd09bdffee7b446a2f1f296ab0b95193853dfb88d13f21fd3e7c69171ad8f666606b176eb358d -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/e02980fb5c1d4bde16568f515c792f0d -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/dc8ff2276611e99644df9d9d98d1f24c2eca6ae5f1618293fee0f55263ad42928d18aea65d8441a70b99ff865dbd48c19a5727fd4a4e89c72666da85f7167aaf -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/0b48668e223f3dda099bfef99931c6a5 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/2b8a89b62f53c7fc32a16bdf9a34617774bb7c095958667187c0461d8b2c6bc66e1da5a238257f21d848c9a2bba9af6e27cc5f19d9baad336fcc9550ea1b7920 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/811735d7db6108ac0e75c6fac50d2cb7 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/88eeeb1d64b4c66958ce740ca682aed23157162e9de5e3ebb66d75d0dc98235e39aaf0c9ab2ecf9e7dd0bdfb0dde70265abe86571d39e5217bf2abd562a0f202 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/9964abfde7b35e01ae9acf2b801d2c1d -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/651f4876e66e2e54d46ccf436264f82ad28dbf082ea7d37d2c094727f25ef045f58ed9af04d00652fa2f606b8392186f53af39e6a763579c57a91332b6545a35 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/85346afdd02d674b3b3a0fe0e1c28110 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/c89b54a68d35aa5e50f79198afb5e82e5d611fc06c8df143de4a9f7a9c70382d84db94cbac5eee05be06a83a4db1ebc913be21b6a6660674b350c51c77a74689 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/cecfe0fb1e42b08e8e6f3d2cdc35fbee -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/069cfe48cfcbf8af2113bd2714005ce713f8e0e1f1d59821b2f3b83a15f58dba89b8c416debfc6deaf1752b315e51968eacf093f6e062f79925584c28e3d6415 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/6cf9cfd6fda8b453ec5c4f26ad243bf3 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/43a6cc58e5ba7e17c474e23cd72d13a1d947cb9ef63671d292b6eae74e131956cee57e420f413d70fadf41f55ef17d3206b8575bf51df98790ea02f7a7a29e67 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/11b4b6c30f47d88ae6e145d4e32fa942 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/c4fab2da5ca763fd0667babc5752b9b6b185da75518c79f2c3b8fee1a95e14d8b379d0d1d66492cce8200078d3ae796e670b0648b6d654f29a7e8ae54fc6c725 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/38716e1b6856f825be45fdae522d8d65 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/212599b9710a59a5a8816e20c9552f65373c687677253eb5596d48d30cf18a0f85094d821a7e23a0a5561ddac00cd29fe7f86e001bc60dc6340a6c283555a7c3 -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/31216157dcbeaa703bce9e4b9099df8e -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/83f2d4571aa5aa77d88afc33d96043bc599963f5d7843819c84ffbf9084aacbb0b34728e61d237eab66b8edebb7978824654a7f9a202ce1c202aa65d3001010f -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/05a2a02ad12e22661d922feb9121d8bf -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/f344bfce1cea72c19024736f8b35ef536bd443bbe946ceb61a79be81e86440bd8e52d17025924c74fe7c75a63ff5c3931b282501f3c65765967cbe54336d742b -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/e0b77baa8ade7e007b7af33ae8bc0097 -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/9e40df04bb5ba23be9866063e99ad3a93d00901fc31c07ce5f1b2d58d54fb8e16d47323c81492d30009a94fa0951688a893d750f90b938cbd800fee257cb278e -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/f0c68c6feea0845279fd576980d001f9 -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/8c9f07250a25a5a46898fe5caf06a6b9f5a5041fdb1edf1ee7b46ffdf1da90441967ceedf087c8295a22424c11e7acb4643c36b05252db0355ebd2e25e2af30c -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/d4a0231358cebe08f3e406fe3aeb060f -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/2a2d28319f57e23f9acca7d9438a58ce21c69d82cc8dde57920fc6262d92eafec068fa86be2120ff17a6720e7fbbb23623bf18937d2f5959932161989e62413b -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f35ba6e11afc974917ad3f9b6dcf50c3 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a928e5c3d6a87076a9006f2c2e2ba7ccf3c701d7ad016f356cd9baff84e6f3994387a318c4d76e3c176ff5b80d947ab1ab44100bf76acc1185924888637d1327 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/20075bf92d9d3131399d87d0b288d8f8 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/499b59f86661184cf4afc5da1031ad8173d1d36a8a48c6ac2d8b545477b21a8757e03586dca35acb591759e366bd1200b50651dc9aaeddced8a5f367d2f74a04 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e4a9013e9f5c16e7d4dc8b8fe59b6967 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/c1e6aa9a039e35bddcbdccbd951dfea5bf0fa1f647dea57668c679ab0a24db9bedd3965c36e7f49f113ebfd5332dcf7167cc7e8cb875c8e2fa3c77bad8882733 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/47ba8c618ae6000c62e95e31b61c113a -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/e53f815ee9e4a9b0e150bfc4a38582ed69d87e76d47612b4c1e6c91cfa2a2c41f0221094dcd1a77410691a63cbd01d5f82c305200d37b270f4490528f16ad88d -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/9bc07ba5ae5580ed126c5c099f41fa01 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/58c4c1a2eed1f5c824e845dc117534e8e9a2e2f91464e48741764e382a44372a854b6ef576a95a104073236d0f84300819eb5f43ce06e6e6298dc6825ff1a1f5 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/e20b9db2305a41e38f810e418ba23771 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/592913681cca072e3f8d5f2b893b8d81d7d210a2e3ecacce1040cbabc56d0f2bda19740a54d271dcf805805cc398a9aa13ba095b53d5e3cc3385cb398d8def43 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d983df7963c49546970843ed9055fb04 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/434535d306979f47053997510f1c5750be4fa45cb3939fff878ddfcf633fc79cc506258be5cb478a32ff88bc3fb643eae8a98c2e39636c60c3a70ade135211cd -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/337ac6885ff6686affbf2cd00973a288 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/edb86fcb10c92016facceb3a5a0c1027a4a7b30c1936d3db15823f394c6a2c41fb6b1331df885152afa59cc28122040d702e76f00a5a680a22ce1f1804ce5450 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/bf6b2209c091b552298e9381264e82f9 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/2ad551a65004764ce3a135f50e58c5c1abefde69f8cfea84292e7f105a317a49a9609c04461693098be15d3a9479e9fb0803cd94c390d6024dc9b761347e3083 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/c9a517645db0b3a2e54eb1e02e22ca66 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/91dc8174e8c88944e54925364cbe1aea3b1349e2bab32d240d03ac99251ed7118a18138bb91432cf674744559d257db843ff3b264eb512281e606ed5b4fd2086 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b1b59fe70d9027bcf293948202010012 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/56aaa7c6442cad375c92c506ce274a41a509b922b8c58665c2b4e34117531d408d6887419ee51c207102796d6f67b0073ef52af2cd6b5153fbe0a4dc69e005c5 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/7a4d68543169bc9e923c5cef8b033ec3 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/135bcf3f69e0be614eecdbdce51604d8a468c46a7ea209da3359cb6ad45dac3f0308718e5501e2cf981913bba37d105ac55ecd7e272687684f2c8e03a742d150 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/2abc7a6d61ac3d943e237df602897c0b -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/8ac94e63399be1fa9f4de2cf5cf95e1976f1ab875e2266f5ee4c9bc238f8e2a6a8b9db9d841583da5b0f49329cbe9df45913319e9def6c1ed9666829afdca0bb -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/fb4e476d0c97e84a4e7c3681bf3d8eb4 -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/8ac633d7c661d8c9d3d2142ac320774623a1511747635a97e1b4818b76a425a72b32ad8f255b9f23a39dbad7b2cf7098892f4062a5d86702cfed55308f4a589f -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/610bc5117561824b6923ac7d3c87914e -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/e9dc69adf57a3af37d3ff88971fbcfb3c667fda6263e537491b80ab2ecf21892edfc52b4f531f3d29f11037383d1d7e0fb45417f9372995174026d5f2577b337 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/c44c57244876f2bd809a5fe58995ed75 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/a74e14b00d01bc387808fc55942e0af8a5cd1bc5e8e6d7517e0e13c9b11728faf8877bf596244f3f2f0847f1cccf3e96be0dafe6288975ddb212deeeae3c2c61 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/14e163535fbf9df6a3fe3022a0d8dc00 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/911e33e66c3b097c003057285920c20690ebecef6237cf47b4f0ce54f44c1d0a512d76c33b4e170aeb6c92a7e88f849eabaf5a137e3358d3a8b9091fd19c1883 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/8841e6dd4825e16ceeccb8a623b99377 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/3873039dcc7135a5d8b0d5165284fd16ff9fa27ea423443bd9dbb1d7d227835b53b4b85ad33f7b78c25bcbefef8afa7363467ccca3a3156d3131fed92f4d622a -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/5af0f418fa2c946bc1718304cced08e6 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/de21210e81f3845c022b6d74d3316d51b6f4fd221d923c059cbf67af5edee3f40f6d61c13537ffb856316cf2576508cf80a88c348484fdae009b45ebd26c492f -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/4d42764ec888ed63603fdd5ddc2f9cac -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/a20eb88efb467ff2ec16368f443128fcaa1624b0ffc9deef191cc4364ac0407689674839aed3f2cd5b9db001732f17eaf9f2b15341dd1d4ae8015ff38a295447 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ac566aa0848d55f53d2d7fe3d7724e8d -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/7bd3d606cf4ad6f2ec0af5e8857965564c14f7049afca3d975e4dd2817ac740694af489086f8e9c9a55039a2d1a0e1438ac8d08d6b19c391d9c3462bc251ee36 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/4db9ef315252cc72acb792c102058726 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/0e1bc63206529710dade1210bb29a759e8e264ad7405a87e870d09cc0cf229c51376e7a8014c42fe5fdc4a72e661c538a66dc3f06ec7b8f10ece330731ecb7f3 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/613278da489d7511f9b6633312e81945 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/599b76f72e771ed6050975ffca0cbbcdd49000dce6804828ac4e95e28eb802788b47ba8f211f7078a93ffd316be2b9692dad08c5814b93c79fad235c5823196d -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/4bd306f4d6b381430b541600a4a28c19 -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/300fb7a3813128bee40922c276eead1975900887a1ba33fa196235f41e71c8c093df864a0e4ab4b5397e4340d04e61e165400a33dd5b7d2e6dec95b450589382 -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/2f1e6e33f03a27786b0a94700fbf175b -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/daa63ceb7afd2032d72d8731cc33bcf499b1168b629fb77a5905962832b5dd5f372fba81917525606e1fb5b861c18b96b183b8af43c9187d64a1d5288e28d8b2 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/4fcc5f0fcacd6870e246dff5c5a800ed -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/f04f135bffb82b9ad6c13e72f8d6257af0fb3648afedd05e08c8b96f667c523a8614fc8143ebe3abd1043fd93b6295bd3b7f1cc19dadb6b6c29915401f3d8bbf -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/a2ef0ff32a8716b53e03e06406a5195b -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/b7d700be2b662619bb1f3695484d8027fa9eb2bf57342c4e19e14133ccc623d260a444f9364be8818e95c27a14b90f8c6c35cbcf422eddaab350325a557a68da -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/27e11239cd800b8ba423cedb915d035a -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/55ff701624fedf8befac528436385711e8ec7c7e0596ae23dc9727de4330d66417ef4a3055d47c58ab0bb68c60340b2c2dc4913340c04cea057def03451a2ff8 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d742fa681b4d9cb903c474d7e485f109 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/b498591b431b8bdef3682e2f1366ef25913f93be7106ba86f6099750f01082d3ed8b34abc3957026abbe8ffa1941f8feede452d85caa45dea6b7a7cbecc1b072 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/3bcbca15cb295ff426a61e9f6fea8852 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/4a326f6f206d3fcb6a22caf9098954eaa8477bbd5b7dfc58372faa8120fd33746461342391c31946fe30cbb35a4f60e1f2fc9944cb23ea1f833d9179a865f52b -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/6ddad390e091228e45d30d0c7da18441 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/640b9993974755791133902c3c54d96a730e4522165cacc90c352d7c97dd56a8723fcfb564cf560ea1ef0405d6e7fead9aed3c5eade081745b0ff51c8f7b1018 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/f84d0e5e29b8d1cdec3692d46a2247f5 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/1463bce08e5ab76a9d04f914ee0c0e56f9523d5dc45127345a09932a235b7adf26d33706837fa962089ef401d676ddee24cc7b2bec6031f31d3d3d78bbb03f68 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/66d873d9b2213776eb35131d505a40cf -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/fd4e7f5ec55cd9b5381a5320b1b1642b7d7b0c4500bf85196167ceb7ab67d80fc01d0304bf12a18dabbc04460901c44c6a3e445e25c2c357690c9b3e7223c8cd -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/ac90f023cece0e8393e322734e7c42ae -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/c054a6ecf4c0199a144e222b24a72613f17af3e8ca33d3674c43582c3b6d0f6a4013d2a1ced263ac64a7f6fa9a10f0821523102c3c6e0fda41ff1fceb9ac2c76 -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/64acc7d428bbd83d566d3bab45b7d0f0 -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/fd67b402a64532c1c32f76bb070c547c9852e5cda2c0c0797d4f200610d7365b9e23a31b3da205d6621960e8753b28e337cb90bde5253444d25b16c25b015985 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/b3dee71f00faeb6d296c96a4966f785d -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/21003195e2060099c7b8343eca34fe73091e01734a53a1502be01e1afaf59f1192d58372c0d99e9a08b2a28e34c8054dd858b88744fc82ac5869ce923e9fd45e -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/bbd802a7be8d924fefd147aeb0e38238 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/2153df14d2d5792b4ce12f9c4f54f90de6bc13c08b8faf41eb0892b27e2d87eb9360eeff0131b69ecc64128ef48ef5c12f6e514e9207a96d910608ccdbb70c51 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/8e81e8f6e755bedb21b383fe138d1a76 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/3c91e58e26fe7dbe2e40070a8670951aaac79a1fdf7e569f0a144add283b59fb08ae9664559560d67c453412c063a9779af930c919dc2b70119de5947666e409 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/323dd4b5737b7ef970c4e8bd350b5912 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/a4a06313d06ab90765d61b2795340a44c97afeb2e17cc0b2d01b08909d7239a300c97eac9d1c4e13ad4bafab76102eeeeff73f5d3674afbdd976baef8beb30cf -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/91833fdd701af8a9c697359ecf7d8aa2 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/1f292b47a88cf8a36e046df4350b97f026d5c3f6cf1f8d1da16a7a320e51934c4ae68be4253bab4c0953cee9849c289fbf5232a5eaa6358d6a07173ab66f0e94 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/0d1e06843e4d743ea3ce03841ed23598 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/13d046ea6a34d23fc47fec998067981fd1bff2d8d7e5077546f3b716d5c5634f0a7981502bba869847ce2159a5b3093d2dda93c6e0f3788733b6901dd5554088 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/87ca8fcac9b4b85c0e466fcd98b5d420 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/6146429da998203953297397f7c6546b96803f1f0c3962eb0ad5e30ae68555132fd88fbe0238fab5f5e13c4da43afc7cc6e0d3a1695f4d0bd5aa1c471f5c1a8c -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/670af9bc7d7c0323105de61f4720e238 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/7bc4664b634495639d761e7186a4efe85dc3b485bf9f69b752b375077048326d8303343c23145b4d1840b12eae5dcb8727ee717d43a88c721ef63935b31f599f -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/a5c7e360cc1c7702c0bc3f14c5e3b09a -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/eb1754122ff8a435a91761386c70e9e1d00b9f5e76452f620d265c191f43d2606cb2623699ce68ee25951ecadaeefd3c548cc0862069d84ed8bf58618c04b1c8 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/0b8f88aa139ea1c76d7bf61b0b337673 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/b1cbbee4aa54617a683fc56ffdd5582e4260a5135d10a4c40cf0dc9dd65b5b0a86b78d4c22bbf40088248bb9676d7801c162149319fce6213c4eac045072bc26 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/2f9ff3a30f68f6a5bdf70e0600fb10f6 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/86f57613aff3b844d23956057b6c2ef1f84f58371cdd4375ca696d7058fa92691b5ee9925b3ecced7eb7d3593682531128b44eaa14d3c09588e0d14b13851bcd -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/4ec21d62b5c3eda8c99e6ce11db49a32 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/f002feb859e326505c26c4970e027c0eba65575c5645a852e77ac8d01d8d445a227d2f85127204df8961142907dac492d07ea81f053394ce61ce1e02a73d00ad -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/3822c46d8db192fe84a526e0f8edb960 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/974670d88fdeb11af6b1a296042e555f4978a4593d317f3057427442b5766d8266a9df5f70688e2e0cf55453438e381fe77ff6c3782aaab184d3766a896e790f -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/56ee0119e1fae3e2f4418f86c71c953d -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/27aabe86dfeb070675630854439c0ca2488bc32224476d5dcfd21ea481691edacebe20eecba0ffbe3e0fdf9719cec688fd3fd5d13d08acf551d4e0ee317adcdf -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/501429cff1d0b4374e474833668787cb -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/86c5f4920716c888ffeffa3f274581377f222449bc8a806898bb65e416cd1198063144098f8b9190da06fc3ffef290f33b229bddff4d059271de8f02f7197aa4 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/805ce8c2df0c901d0b12971898f8e059 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/c5b93d8689ea302d02a619ea2f36b4c82c69f1e90ff3900c50c3bec4ff45e307135ed4f1a7dc28966c18d9336fdc2d55f4d60dcd9b209adbc0822af972323da7 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/31b3bdb0cd090d659e93930a8ea7fe27 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/efa27f69a5a42f9b933ff28be872ac815be47bbf4626ccbc836ce9a9fdf98412afde4f9e1640235342b13dd4a7592763d39986d62038b55cf5528c5a7b4f5049 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/095bcfc50bdc01fafd5f3e17d3cdb7cd -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/d1239fd0d0b75222030efc5a1604d1914de81a6eef93f8452d85870e4fcbe4e13864234dcc2d3a74f01812b9113a909c7985898091245c13f5bf94042c9a8cae -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/879ff2ea16a1fe12f324bd1963233f72 -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/571b20623a880a55dc46387619aa4cbb8ffb57d810108bd5c41f1299ade43a00a64b80b572d1b0caa49fbe458cfc1702a14b7954fca529031ec54485367b2e25 -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/4895c9717eb358de254a8fa4491dea6d -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/ca07bf2935d6f43dc75551ce5462259ea26eceb144bc31f53c56d0e3f4436bd26b72656e854837eb2e6c96ab69a3d2ebcc48db97f7607175315c58ae326d3488 -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/b33bd8c4c4814d4e893bd9e87afa68ec -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/0b6e18cea038ddc4a8070d04ac4e645dbdd34edcdf33dcbd2ee9cf03ccef42544d03824ce9a709d9e1a6556affda8955d1669a667bb275fa7867140dd5f5dc7d -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/b14c2f57273d416cb7a46da90563db12 -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/712d317e0048909b60d595c5dddb3e948e4945526629eeb9ba42c8d92d4eb2e5073169231368cbd73669b6ab31cb0311730480c4b150748864ddab052cabb33f -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/275dc909fee11673e80914c0d5ba958f -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/f6b5209090dec3a7d4ba9d39b215ee47c93bdfd3208563de6651b608aec99caa062de7e3d6cf744d4cacc0d4c89c4660541c9e65adc8a99de043457a2e882f3b -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/d62294a5252369c3c14c14402c3b60da -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/0dd65d3cf4eee36f3d63f6b2e5f59dd4b9fe3f40115aa83571697da2a4bfe3a381a8001770a32b0ee935de90c13b391357ec0dd6a3ea5421ce6379daa0ad1080 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/078e1ee96626fd7c49c959d102cc4816 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/a4b45ea18965d227fda92647a1a518dace7d68ca2b5ce6b2b378e46e9382ba56dcb55823cf6ba6ed224226987f45eb7d2c101acaf5c843be2f5d07922287647e -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f5e6d729156c48a0d158f791a7aa0aa2 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/6556e418cc07193b9de1999b8a8054745ba1b1ba1c1aec4723986d0d0338513259eb76bb84fe7421dc5123752c1cdbc1a292462d0490e414ed10f2a3a0d589da -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/ce065d4387ceac874a7704f47d169a3e -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/79a3d1ed6f90b64090a6719fdbcc4ed303276d319e9f9aeeb93819e08deed1433858a5c24844db308c8f873e3470ff228bbcc60f4bc141701a5ebfef18667745 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f3a6de4415f7460ff440274fe365b7ce -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/c6907493320c1bf1037400f9d88e428fb82d76d9767e4ef8f64aa7f1a94bab0dd3b464053ab3f5db78c8251477ef9dad92d9ed15f6423abc620c6d5219288496 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/8ff66a201ad3503b98170928303b4243 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/478555ccf774e3fb7bacd875aff108e46fd23e6301e9ae8c01116b60de6e1b08db7d1a86950df21bf056cdb4229f53efe68024682ff8a8f587390fc7b6feff57 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/2208e55d29b1d0854d5fe503381b9758 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/fc875886cac2504462bfc3b329451091fb47ec0b492ca64f5fa6b7106e34c4527e338f622134883500cdba501d21c9b72be7bf8520754487c541c05e276e3c4d -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/960dbe73abd59ec25d5cb29c6aa643d2 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/b8b06dab4c144866df3d207da01ce5e2c1287eb09e37e35b5f5208c46ca1771ebe5e4bc52798f494ea619b30cb7720326ea5418cd99944af591f89204a8ed5a2 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/8d6ff4b3aee7ae73347cab3dfa13463b -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9b19edbc489076c7435e51422956ad278bb51aa1150b84b1f540bbfcc2dff7aceb0d26f1e442d0a0f760d00c4dc433f2c3026595c1f1d862c590f473f7c6d508 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/c0e97cfe35911a1331edce84498fb1a8 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/4ef74c537a0f51c35915577c39053ca3f0b2625ae3f130dc4a605408424e49aef21aed00a894e942a25e57b58730aafd8c67c81784ef8f72885dc7412b5814d0 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/8a212805b53c001a1b934149f5f07a27 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/05eeae5ff8f52514e1a84220cb06b1d2dad5d70c023771b362864f9ea0b62138d7d8c80b58ea5c46cd51eae569b14b3ada209855fb36fe7648c81418962f2378 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/a95148da62df6df3fe8993b77ce4561d -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/c830d8774e2a36ac672b769f017832b41e38ea1b3d73b566dc099c1a63213fbcf2711904bddfe04e5a0e884e4f9ff48289697f07e2e84bfd665de55f640697d0 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/13b5e6eed527cfc0384f954b8ca2b11a -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/32d140225cd0410ea3eeb340679d17809f8f7e37c27ff63a7ae2ba4fc96afe71cf48caae39aa846369f5486518a6d184439cfa2c59bcbb93bb6dc0bc624c7138 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/21074f3a888ca8f5023b326e09c8170e -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/1ae17185c750a76b9015e6fecff2f68d39b4d85740368fa9c4d1a139cf0f5faf54dbf77d83cb3702393d76e59586022839671e97304e36a2607e2f3b3ad09800 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/a1a5b71aadadac7a1e90dc64b59467b4 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/d7c5aa523ef24900cc2b9b1c3391f30be6caf817adda0d638d1fee26af20550dcee8440671f02a075c25de3232f42a8bde5b8f967bdbb1821bd2fda97c11277b -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/d9a77cf3b3eec63bd3d5d3b8f557b8ac -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/8ebddba3ad1bbbbec0d52bcf5e8163d34b560116dc99190ccc61a069446b176e0a1f2eb3caab8d39bc1e09e596ea684703b003acac977d0164a55deaf0678bec -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/4f1191e1c3b09fe73e579a6b38f76568 -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/d2990e316a1ffcab9310fffea4e51906fce058a0cd4d10cf3605e624b67bba0b08b52623313d493c29b82e1766fc9e3effab52c7580be2b38e9f7a8c85d0e798 -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/026dfe3080246343b37cb62c964fd5f8 -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/fb6c1454e53b1eea7d96ae48d74595d43cd17991059d870d8e80622effb322ccb3f441e8fc5dbbd376fd8837c7d2d3ab7135b9ebe1a261872cfa5d45a7a3b4dc -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/a5a3d048f996d78f26cc9058686db33a -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/0023dbad7d79c0d6798fc47520ff7c07072efa5dbe2a5cc7aef5e5ba1443e83c7c86d124d4ac08fa9150df6bb3d213272b783fb929df899127f1690f93eef35a -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/bf265929e24ccdc52d580cb7bdc3a3f8 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/28b743402d04f2e5fd40e343ed48933ea7dd225aa4d35a8cdb097c547f39740708e50c5a9610e78aba4eb90f44c93b2530094adfeee8db0b72c71ebc0a846174 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/3724e687de73b4d91f4f56119c750179 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/99699b43d8de9d6b68da2908e15a83bbd6e583ae38ec489ed7be8894f4eeb282ffd8dd29d1204202c38e0f1178031ff42e61a1e7638c24f475f054352f953759 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/4dc9c96eb08bca5484e8cda69ec34e6c -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/4ad4b93e60c06b375d07a13aadec231fe017b48aaaad345eaa76fc1cf7bb5d093bc776303f04c74b45eb4a1c2e818109aa14bc7a37170bd3061b57eea93abe9b -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/5ee059f4d562ccfede9cf0954da887dc -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/a8daac9e5181e1a3a5b11f3853b4fa3c6bac15fb6d0dab54ace9e3c482c95d1efbd150136f3981e06d9518a747234316427be245869c8476bbbefe7358aec732 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/939490a7bff4ecb667da61755c7180aa -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/a89b327ef68665738c8d6af8d247da6362b425aee5e19770905d81e575bdd926a822e4f34c938dee985eb670ccdd5016b1cdf6046154a50cfdac328fcd58bf8b -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/8de3cba498a259ef2c51acbc7321f592 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/5e28eadba7aec85b1150088054b7b1e7c420c43159a2b0ec75c7bb2a10a919415b099c2e2f886c53a5ccf91a0e1dfc6da2a98f39dcda51d5280dcabe78618b40 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/e9a0ce56f669c371a2fbcd2fef1c8153 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/79bb18db7f6ca80f72b52488905a9cc84f9173a48f788162030b7818830695d6136ed1298d8200c33b45a28023f8de5cc14c1eb5512b22e471022ffabb968520 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/3abe35329a645e22be94d8ccc0bc0888 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/5ee5c51d9a783aeb8f4cb479b14fefbb921ac08d2e964463d362b4851d694b6735d98f6e4a24af1766b441014a6bfb96665ffbe6705b1b05e7d6fd3e406f869c -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/667f436a793522c55a6592c615f330cd -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/ce25daabd9a61cacfd41be983e0bd756cdc53378f4380b1b69c8b841ee1a282f13bcbf793c1052635f52f9c09efec39ed2969cec7b8038c77f4d878589af8262 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/e26f47fb96196d2390b285bd80f117a3 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/05dacdd9d16c585a98d899ee68db36c615d4d5432d22a75ca0813a0cad0ef0ed16a38f6a696ac32f892f01adfcd3c9d5d3464f3ee5578d345875b85849937d0b -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/70c0542ce7d8d471cb234aeefacdb370 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/5aa6e835d3d34f8535489e133a486dc8e6a6a6490095daa7056c00e0b90b801d6d2271a99e503fed2f2be3017e340c4e12f011dbfe89cbe4a0b810edb8ef5645 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f8e57dff4808081af9715d8ffd39967f -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/96b6186d30d4cddce4787da9326201f206c4cc9d6fe58b9d8d40cdcc78dde01965ef1e921d4c45f3cdc35d46220a3eff729e474bb0c80275a53d1f65dff90be2 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/8e415bf1c337a73433663396df4ad308 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/6a9d221abc26a9920e6f72ba3118df312f5291b49feb9a1d1671bb2818602e2ee46a3eebf7302b0d7c958bba674bff8641f1e34153e9fdd4b8cc0edc114573e8 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/40c9181205bffa4dfb78783d5b9f3729 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/bd2bce44797785f1438e42eff0cac3af927b501101baca6cd8e43683bd22ab85d01774fe2eaeaee1bec8089b61cc88ecc06568fed4448a48b5bda946823cf10a -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/c5a55408326f565be0e45066008703db -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/79136605ba3e35bbe14aed813b6a01be48dbd8ea0d20bd483183045d6f6aeab7b39f0702896796ad249d6a035bd97915c3401985ee1bee1b3cba2e9a89860f17 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/d1bc5ae2ebf454d45c7deef799394e78 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/1781237f2e510d6737edfc0704b3f4875b79c0bdeab959eef2c2802370f0d0a13be712c7d3bb3f0eb892961e62e9b99b7a57b539ef3fc03541d96bac07c1bc20 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/2f04b72c07dfce87916be643ab79e6a5 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/52541787aaee8df1e4f206038769060f2c53efb99f3d31c7cc61fd03e1abd5e674837584a8f4f6e7d6f923d881db39f7e0a88cbe7c4942602b19ad7923b665b6 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/3a3e235155d00ed229e8ec60ada8aac6 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/bbb5d63390780ca76de3471c1326d4eaa7a41d65d97b3b56e0aae48ada8d0d2358c7362a09ad3bb7ec7d8cf9c8aa4894e5369b6315dace3d4ab059ee96d2f252 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/7ffffbc226902f689e8f5b68919be1cc -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/a89a792a3cfb19fcfc3183b565c629a1bbf1dc66d2bd7da5e377109f64df769ddfc336440423ec54fd6ba49112acf5c3eb452929fae1801aa0338d0219b11db9 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/e280360085eca0bf61246b384ca27569 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/bfb6ff7f9558de84371f09a4830c6c8cd3aeb13b077818966ee0e4e69d8bd648811696f2bf6d2c1259f2f8cef2ed2da903c6f0f81a7ca03c9473e330899024d3 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/110a384a11b95ccfd825fcda830234aa -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/90be37ce2aef1ee5754ecad693fe8a65ed2a1f8639f5e6ed73a598b30636cd846ecd4aa487716af8af71866f1359d594f27121c217c3e84ce1d686847e1006c5 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/806814307fae511da5e7b2a7b8bd71f4 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/30c7d44e7fbad96812788bb29d2dbe3c537b852e72921952df5fe4d12800dc44cd499b7c551e7b14c3b605d5cca956634944cdacb5e90cbfcf4ce793f5a30910 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/2c7f8e8290b0bf857a6a510bd71913ac -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/6c0f02c1e9687031c692d2890fb19859d15b662f38ace1f4cc7502bc98c59982c39ed7931384bd993d5e5288be34ba75c0a1ad45fd8b96eea13e11a55e0a69b5 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/79ac969bd4fc54e8df9ad597c531c2e3 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/66ba7a12ce9fd3885a4842966a46d377990a12fc0cb0742e2a4bc45d5b1a6b25e5c1045cf9187267d91e228b3917fb604e3e87cd6e0f71fd63116164f9fa3561 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/1f93530df28dec64e65e8bafcac0396e -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/483194ff0b2d2b9b142f5ad804906f4e0429e1e037aea652ba99f98dc9f69baf4228efbe416690be1a7664f0c48663681fa639b86d7cac6a4e8e537bd1d0fff5 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/d2f918e3293f504ab23f877b154f531b -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/9a7d9dc7e98cc27c51c8a1669bbc558921bc82626eccf5e1eb114210a5a16f326b2d3e817c4d531ba7075aa0ebf0f2a34a5c69bafff184320a9a2520ef34924c -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/d89c8d6a2d1dab82f9fd9db8ba5dfaad -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/cf80e352db737ec6ff85034e559e1adcff653837dcd8fe4462ba8416ece81ab8c7109914f79cb882e5603e9d2aab86640453ed2724e1559d787cc37bb13d1580 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/7f09c2e06409525ea22dd8898d2efc7e -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/6f68f84c14e5ef1fa64b68dd6a74783be91706914a0bcd1b5a2ed51667c5ce37f151b15f621875d44ced56632d75c091bbdda12e246b9b5e5d944cc7101d7f4e -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/2a1765effead816a29428133ba21620f -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/b0c8414311625398441187d222611dddb77af83309eacf04a8a38d1b3fe30e73392cba4c68ca3f6be713cc4c25fb45d2e14557fabad5de9d3672bd270a00c756 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/66f95c5aa1e2557398dd5e3947fb5f53 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/79341f8e86a8796cc297242f455c87f8f984b1f0a37e72ba767276faabf8537d6827f96aab43192c5f92d4bb8528837be5509123614f86abf3b43478d7862970 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/26564072393a293f5b449340388ba097 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/b2d035eb429385d6d9bcca83d19ffec9a627ba93e0de9daf8b9d956321262aa43771e9f6cfb54b4582fd85563437e5bca475504a24ee077dfa7001423e4fcb4c -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/a4cbda097a9fa0a6382e968b4ab26eb9 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/3413fe7d5b4fe1fd4fdcbe9463bf377647c6022759d6deb0444ccffa4ee0ddd1a5339e0e3e7f8d0952ff90ee056c9fe35bf893c1d84646095110aa0ed54ae83f -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/295d07c1cd0b9e6b7841ffc9ef49f694 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/502916479d9ab19debbc1d4f5f6631cc1754bea79e8a90cb9528be2cdf3ca03ca45bd0efe7511edd1e5b2409305d3acadc0aaafc271d953669cf8a605da795a9 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/fdf675e16faca1de2f58549cff2c41c4 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/809de084cc33f45f67c4d782e6a282af1a24c01a742f6ad432e75f2da720af2787489f92ff8dcd66769886325eba2f066a00038dc750b97830d7e5c1c38b219c -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/fca1feb1d920392a1c8683ef3241324a -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/baf865e9717fb0a2155dcd4cfde3154850c6cfba86e72e50d01e79ec534ce6ab51f2a7913e3133c0f78f081c7a8a90782e5e2634cc20be87a76de263f6ec1627 -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/1e9fa5da1d51e9a7e82920203dca7125 -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/58ec85403e16c4504bd92b270873f0473240ebae25b088119658574b740ac68e23a88191c5c8694723f76248393e8fb1a05d2879750e25000bdf666260545a9e -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/000ebf842bddd43b28d02e13108fa9ea -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/2b256013d51e1f2099357e56bdba8a2910f06b09aac6f12b325d72105f164024694900286507b43739cc4195048fb9be0996279d44ecaa3e46b4d075b4936fc0 -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/ecaea52eb5b8ebfe81ac96b02a4a6712 -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/636bb31231596c04f32d233e447fed46c8df39ec717a66eec94676014b0bcbab982fa34fbd824fef5796c7502db38ec809fde7589026638ff8cd09983f43145f -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/058bf6012bad71b802341fac19ffbab0 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/890cb969946d4ce2f55978d1ccbe4dd4eca6edbb9c03f385b3921f87e5a55df2273d342c14afcc31d31f251c58aa3e95b67a084b73380a4ecb304c2b00e4aa11 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ec4c7d3a92e8ca0fea901e395f99aaa3 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/af83b9c1d404819c13fa2042541bd81556b434376700584efb603a07093bd9e14d90b42b4dd8278662be94ac6e856458b28b4acfd012ed30937f8a1bdac56271 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/9ab69e8ccd7f3bb52f3d334384f0b0e9 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/6a7899c208934e85cdd1513d34a6a222d6beda56bee3bbabbd8407dac74399f446e68329b95f38cac9d36e19545d6681a7416384db0d2cf51453703a4e567169 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/393592dbe8c83f7c9149ec59aa307999 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/67b6867a6ad98abdc291d8e961d76f44e9531a6576e32ff973dfe7fe4a215f9943b5ccef86502968eb36ca92f0b287f681b4c0a466f1ee394399c96ccab95dc7 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/fce0b3598cd022481644eef5e0026877 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/6d98970d16317c9dde7fa98a04ff730b58f198afe0b06e0114883a5fce6ec773b56eb3ae6673192fd714677cf7b30a80d17b7cf5abfdb63de388ee849aade61f -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/ff8bcf145a1addd88fe5b08a38926329 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/b527235e9c20838b3dc0285cb249634d46cb289c5f8750c9a9ebfc22f27881a0c517cca702ab7d98d76eff9621d2b897cc8855392dfdbccc497d2b3cd49aa9c5 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/0d00a69c0baa5c7675420740437a5173 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/827afbe21e90c678f607090faeaf7ed892cfebe9cdc6281ca8a2da3b2cee4a5d86fc9120122827c52189b40f2747e985561d35b67f7c6171787a8a4ba9e6b4b6 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f2acc4d798b44797f8540f9a8e9a209a -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/2261f77ca2cbe317030693d873c8a71d0c7293038f104605aa170aa10098b3df87cd54f0ef10455a8e82341ac33d8460edaee57ca3754b7de394cd1df2bd37a3 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/c9ed2fc9fb644ea69263ce913cd37bc8 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/d44b21e04b48ee8b9bb7cbc0351517207d3d07dadf444a054a3ab1cab88dded8743be7b7500d63da84cf935e6be976a178e6d3700bd45e82f2a98ed69a1b1f22 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f6a066937f6b47eb3c5cd8952605c2e6 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/f62a8999971ef84f1543154c4835332b2d9cb88a95116fe26af19a29eb918ce7a41218c15867d3b342384409f1ae793a8922c1db3693192e7a45246f8ce74111 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/c82b61f4c6c7279e5430bffc5cfc5099 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/c2fc473da360d25a15ae54b546937f9ae46093d44f063e2692500d8cf6bd7fad0f7f5ecb9d5e36f8bdea77eef75b189ee45dfc4d8b9baf7ebfa49ac23fe26d62 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/7e7a4225d0722fb37c2772ac90e033b8 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/5d4ea609c99195eb2818b54cf7c642b146885a4856e1f97ba1e96d9fc73ed73f3d530bd0f332b973dbf773a39694f82880d86312dfd401256c053f026e0206ee -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/dfaafe326c11b9b5bf7cf708a333719e -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/faef0dc66f71956dfc65c476d2d70e56e929641b88738d01dfc71ddb79166ace2f47daf20a40a09bf53cf23c9053ac2c9eeb7e79e2ebc131e7abf0bb8eb5cdd2 -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/8d2ed4cbb60e8a12ce9b95159f08ff16 -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/8eaa10d1658b88c688312ab35588237c299914b500283c2c575a7e6c30b0eaace672e886d3a2b8e5d6c6e54e99612055ca226577b7001f99078bffe6c11739fe -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/aecc45c03661ea035c9a3306deda612d -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/0dc47901b7b774ebcd91ee318544345afbe7db36423fe206c463d0f0e50042c78706519f25056869677466ae15f57b05d6964c1d72d0639c6b204b84e515557c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d937dfbfe05cec30207b5f0aac5e110c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/ff82a33f45c8a9aa72d72f85f7da05fafa733c86f4335455eb2ae2df42b64c2383579e594ef98fa1901f3d67db495a2e167eccd7160beedd56930fed6aa5beda -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/ca4e809b4192835d9816b68302f2439c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/0cadc24b66ae3fbde1236ff3a056d0e6c8654fbeea0ce38c949bbdefdb226b9be77a7a5fdd498258c0ee2ba7c30cf6146d497213a83d1c642fe6691540716939 -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/a432129e867d9d18a0b897782b6123e4 -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/df90c56ad7db6594057fe9a1ca2891f6b6bda4d05c1d194de67e5f2543819cdf094985ce31be121363d8f5b49074a034100b8c9437f790ca727a33b2073622aa -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/0a31fa99b88139a1ac9e085af50897c8 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/c58092da7d99749ae4fce1c3a9a86fb69ec45160109c35ae9f3b3903dc7ada6e2a8875bd76795fd221816d3573187fba59192315587c8347aba1210b4dc34b48 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/0fad86259377c37fc6b55c175691556f -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/aa34c020ca6993af5e5c1197c1647e084fec8912191fe8e06704cbe1a3a13d983973fce9362736c6f78afb24ed7ee125b58c0e84d498ab25500a75b0c11bddc4 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/7cfb079ae1580be11b732fc13615c0aa -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/185730ccc5361d9b9abc29cdbf99c89e6023276e5df14c9c5beaff2afcaa0c151d82831ad047309fabf8a48637ec557fdb27ca0ec813ef055e94245ac2763d86 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/b8b4631f5d7ac0042f83ce63611abee4 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/d98ccd1737a63dc57ef6ac9e8ceb305bf712ccb486b8627e0ad05f4615745d89786da1fa9cc6206fc613279b10fe45b18fb7a1bd669f98035048cfbad38c8da1 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/b6c9dca4dd513a4a13a651e121bbdeca -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/e0e7ff4c6a7e0fd13278e61f21488c06903b87c9aec777bf565c8d7de3ad775e85f625f17684183f00e6e409d069b02167350c0925f287b83dd9751d2422b758 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/dea961899e125ebff1008e1a546bf757 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/0da3ac53678252f51778857672370abed0f48aa8905307c618554cabfc19845fcbec7359700f268c8577a5244e588fd6fe922c5dbfa870031cb7e464344ea7be -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/d280bf08e1fe086a23400faaa6faa333 -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/76da76f63663cd54cfcf0391949e8064096ad436c86efca0489fef19473e2a8b8c24aa5647657fcf15c0ce2dd4dc4421b8de88532a768ff48a31fa435861d39d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/d2eaa8e38d7916ee3c41f0da782a891d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/bf76104a2cd3c2ddbd9526f94c29ecba2c40481397aeccd84cc21e5cc19b8cf5d5637d787cf12ca4976ca9ecda5596c4a13ef8194f8bfe17098d40300d0957bb -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/6afa61b0fc4949ec91a0def4b2c2ef1d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/ba0aaa45e8fbd8f3c3506f1ebb7c7d58963108d2be7a6f2991e6f834b06037d7915333f1a562b06b811ba9b38ae287f32f90b87019379a2a55e654e84d7267ca -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/6e5db8ede54690f4ce5fe58024e903a1 -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/5d1402963f9677dd9bf8364f20a832a1fe6466abbeb5837882c68bdbade3818798d3e4a5a2c1cb56efa7cfef1b57ff3d1a35fc3a19ba14b33026ecfc3d5e9597 -libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/414d73e8bcdbd0f0261bb1a1a3b7442b -libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/bfc718d113655d9cf364e33b5c879a49295eb6af91e92300ec54a56674bc3d60b0de7310939cb517818ba95d9fae57a94e784d85a3024525904ace8f37ff98a2 -libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/663ac8ada6ef0cd870b6e56f312a066b -libLLVM.v19.1.7+2.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/32853f6a0319c468b97a0a1fd61acbd3ba6ee57592dbaff12da4be22338a31af6c39e9502f530c49292cec40e3609fac2512ee8cc9bdcda3d557e1d1f59a5237 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/3a063d5a5ee3a71ef4636baf53ade001 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/fab4187b43850cdfe1cbe96d8718f3f35b5541e9ea0283ebbe67578b0aa302263770a37034fb6ab7b4e9d72d258f4fc0c2b18ae75e26c001af2a8b6eced51f5e -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d4c298518b77070d45344ae5a1bf9d10 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/6ddd7dac8bfb5629a7132e8aee613bb5ee2715cf572eeec3457d84b989aea853f960c29b8b7e365a91ea7ba7e48c97fa957ca74b1e72f6d3deb420b0bbef21d5 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/22c4ab82c08691c8ad138c0967a5935b -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/f4ce3ff2c167181bfffdc741324ba45ca58bca1c684c3ea3f86032e843cabf339dbea78802f51b178df155250fbed62140d29be070e249e79d2cd3b10fc4bc03 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/9f0b2dfbed1e396825f207564a29e750 -libLLVM.v19.1.7+2.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9341ea31a62b67d780b753d0c8a985229b172e281945bf295d1ac279974ba50120736804d78e61cba846007ee081dc995b73256c22a4941c30cdbd2c1c86f0ef -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/31484ad466803d66b9ae416723d4e037 -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/64dfe954af3d92089eca0a79208388b5907bc5999c74f151fa81d8e9599764785424b43e1364d1f86a156e0166e9a719382c0a5955270bdb35c570e6bb0cc95a -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/1981bdbf499d94de6d235a4930af1865 -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/d8f348a7e77bf23c702aebf627c6cf0e67dc9009f1e48d38d2736f957ab10042de76fdd81f49d83d957d3877afb0093910e02269ff29780ce18b590411871435 -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/ab5142c9e83f21ecfc1acb376f2b1f7e -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/a17492d497275f070d7b4aad6c9d97189f90a91ad6b84ab77ff5ebb9c7f69e2c6c95631cf700fc28d012a202ff58221d76f3300d2ec9a53ebbfa1fd94be9e62c -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/1eb8f80a63bf70c8554fb4b2e0d0341f -libLLVM.v19.1.7+2.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/92d3c3118fe2146c3265f37a3647ad673e0bcb3be773727391f67ad1298eef477e9b73fe7eaaec71a30a87ad3d7d2acc3adf913f4c1c3adc3d5e02bdce52145f -libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/933ba6700d32831a66e18549cc59974e -libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/b4dbc072056b39c78f403740b56994cb1ac627f3047c3016ed6680622ff022d874980dbf39cbb9c80a06cc7a5b87d2a696ec68f8a895e00ace8cf9198c13b249 -libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/583775ff40ea2333fea9a372d338e0f0 -libLLVM.v19.1.7+2.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/cb51a42b46d844b5b1b7b4cc616f08ab3069052cf6f29692bf258f1544aad3df8b56889ac9153a8026208c2827ca73be925bef87f8cd737b467fdac2ce98ad1e -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/324beb57142320dd719500a76bde4944 -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/5d608cd533f3a67445202ba2df5b09cf1bd7ddc5916c43d9ca11837167e2a0d7cecc11d3ade380044397e6f868c110cab6c724f0abdc099c91be076a131345e7 -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/af4bdd54bf773bcb2deca69b71b4117c -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/48d102b7c6ebd4c607ed77cd7c1972f16433b69dc7cdf89e36ab4d319c6900f928c2306d582b90b8baa0b0e22688b0b37cced966aeffa3dd8d2910d950bdb9b1 -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/29edf8ee693e07fc92bc72f2fa6bb0cf -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/da7865e87c2edd8468c3b065b131c24ca2e24fb30943945dfe74c432b4c4f3996c9c03dcad773b0686fa88631ae8c400001c02af80d23b5b7ee0ebb6dd1a0c08 -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/ddb507f775fdc97eb951d5c0cb257652 -libLLVM.v19.1.7+2.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/46995f66cc9df7d57d47f6bfae98dc6f3c8cd0030f175148852605f4fe40bb67d7dce76466b07384ba9d48234410c7307243a0d6f6f42982395bb16d46ef3dd0 -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/4a5d9ac2d21d5c01ff03b8d1d65deffa -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/9488068c391e82940dd17cc5d1448d0b87693e64ab7494f9fb56f2a1b0302d71fe25b37fc19a826575e52dd0fde1ef2f7c83b2ad58a08447a3d7c86d93a2f806 -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/d1df8509299ed74e342957daf51e6bce -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/b584858e130696df1f1501a3482407ad06228c91576b7025f5869e1557c9b38e46f75d016d8fc745b71b3b1e9a03dd7883b549c917dfb84de10d8cfb6036e9d9 -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/83f0ab8635d809655dfbeddaac3d0db9 -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/10fe730ad29a1da79aa6d87e1a0bd6f88631ac8376ee4839c911cc287b39e18b417c2329a2a978e5d676f1d312b13c0633808cd6a0acad41a5fbde6d82e1933b -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/0f0d000e0d765d1520de3301c7c66c3a -libLLVM.v19.1.7+2.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/02e2b31e8d95f4255ff9e97abcd79fbb5b8e862e7df1eb6610b028f337dbdeebbddc8cc7cfccc674cf2b76ef721742c48dedd0219c43f2d1e93512114045172d -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/b4a98372aff1b3ebc8b5926038e66a1c -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/ef6a534dde44582653b82de9cec6749dcc0cd341cb98feed2bf3751cec4c4d959466f11abc32fcdbd0f98d0e84926d5c54661e744c035a51f5a6ef17af245062 -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/10e9f3444ef8aaf41887f6739d0d50dd -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/2c2cc404ea533ed7bafeeb68803d1cdf39281f02005fc14715430f5cee656ce38a618b84e0fa4788abff6ee20e5458975c348951704fac68e0787b8756997b67 -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/b5f6699f74e39ed8f629569919b79a80 -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/f5b90ceb0a9b475af565f115fb2d3ac52658c65f322ef3f4398a4b3e469d17c59c22fd5ba17ed9fd3244050d2d86b4aaad6eb7d0b3b0affdb6dd79493aeb7475 -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/f26c2a94e0f07312f1424c63fdb3a28d -libLLVM.v19.1.7+2.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/b00ac9012ede2a3f3ea8744d6f4162871aca4337cd5d5e73f7cf9a384d5bc6fe3bab43a1484f18d116970bc9e687823537a672a41b582f64c78662a80a3c4827 -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/92c24048d8fa760f36923f73beed5d57 -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/7d43e502662b706e24959ec2d08f75106ba8aefe139d732a6eaa255b732c5c463aca5d126cee35205aae69af28f4bf808ea4076a52200e53010346a12f923260 -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/db00cee39f2963de04aa9e2b2259596f -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/12cd18203a911f8b778fc2df9877f7e1b4ce8c226297055178b377cfe5fb1d0d97f1ab251d0c82a9ef3af5bf22743c1befb93d98bba04d9faf10eac50a4eee5d -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/05d97f1e6370e7e9327e50c42e6d392e -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/ee8891f07980d2d8af1c82128ce63c54305803ae2bd30ccd8318fbeec13a9ccd09a8d5fc56751a884a4c63cf61d89ef951b1f2df87927e6f5f3e061a5609fedb -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/8079d01030c4562c64732d74cb452aa9 -libLLVM.v19.1.7+2.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/a830745afc3fa75f84fcd7cedd4eb74d121b69a3b669f58d5190f6df1bcb6f9ecf2fd6433066651cc1f95c3685ad701be63dac42d00b0d2901bffa1f9516bcc8 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/11847604bf9b4cf4acb7532a7e06151a -libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/2da15ec3b4c6f2c26689ece112eeb91d9c9494b2cefd91392339eae2d2bd411a022d5f4c27c36f6879080ab702827aec7fdee1a8aa29feeeea31ff79eb34c87f -libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b0a9f639d0b07f5be3d42de12ade2582 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/ef430adb4879322ed8079fab9dc4aa09090df64d6fb1bfaab285440f7fbd3df3c9ebe7d1f8079b933c7f51fbc51ac637cd777cb0d477b482c6c732dda6ab23a4 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/a320320ff04256936ae6c084444329f2 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/94bcb5e51c9f3315c8bf44750f3cb5861b90a48ac970b261a446493d0f47d5017c68adbe539a3fc4b19de58fddb0dd31f4f23759b0bf0b7eec2e9d946ec8a139 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/aa678417c504201f9578211117d73833 -libLLVM.v19.1.7+2.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/1a4da7deac68f82f12baf4fcd419cb8dfe1c668de05acd16e17e4bbda0773a13352d2491ae7cfefe9904350718924bddf4bba99b983f48b55b06e88e5694138d -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/f010d5231511c4a776aa9c0673babda4 -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/de9adaee0be3f3e280858a01a08305c7d4e8276d3eaf92d7703629097ec44d3e08f13c926a3ce2999c89badbc454fcb696bead5213bf1574e3385709bf12b24d -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/153cae59280035a21762c9e9f716d285 -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/8d991aeecfb232b1df775a559ddde067be57dcf99f38936cffb1d59a442a368f17063ab9a57aa2fe8a6ce006012cd249bf589aea7acb42f50b78a9dfd96e221b -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/e210e23a2f5d3b9a35970af599af833f -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/a37a36b126befe75f024144d308480841d36e1d7a40b1d88496d85947cf74432e32904766e5a24ab593ddef9e1428972c1390d9c443e20f7174c21a1cefb802a -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/12c00e7a7acb8f7387dfcd735a7c1963 -libLLVM.v19.1.7+2.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/33a44799c8b1654658076040c7a384cdcb70aa8e17aa0a9fa592dd1b2430b1e8fbae22fc4eca44e653b6deb46c040940157b36959e85ca0a19f02626b5172394 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/4d64a25987dafe940071785c713a5d38 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/8833f863f9dd13f26da8d016a2749bd7ea34cc691107710d03d328a8ab9fa05cecdde09a8d42ca23d4eca9f0337db03748729c0bd91c3d1a3527ad53f1385d92 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/3b6c89bc3f2236e6b325cc41c36289b9 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/87752a65df9b6b613f6eb1894086319d099e7433313cabf046cfc499134142af213b39eef8660bc8017b01858f1be986520754d7cede79daeeb246065bb21443 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/d6a14d1b2ec1b342f082d6d4f9d14d94 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/deefbd80cc10d850c28d5e405b268d7bf4d19852d12468bf5960990a4b43c81af8271267d39ce8560c878b65c876dd276182a45832c18fd3fcf7c553c412421a -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/4b311159cc9ac9f8c37d1ed86ba3c659 -libLLVM.v19.1.7+2.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/bf7e58bdde3f8ffcd37a7c833b2b109704b1fbf577f1a487715b983f86cb640c4f4ee6e417760c9f24b4d1b6b17a8ce780018c40ec84ddf8b8ca841595226803 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/ae2ab5437d31b92ab6e984c25339eb53 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/9b884936659005e7bc1e7f41375875918af077eca470768ae76005891b1454bdd5009ffe80f1bca9bfbba1bf71b2093c28b37a84414d5df14976b88f7032b866 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/ff114ab4ce06f21179e379158f0d65e4 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a023548d64d0ffdb961ca5ea4364b48609dde9ae165198c69d03e640fe6f2b14990f9a500b3adee4af69e706b365aceb1c3efe5cc30f9ad379416a243f588bf2 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/bfb6b758ad82b56f318959f4f10e1883 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/3862a466117e9bb02adbb24307708a6abfb1fd171de9d353513e43362eb4faecff30a77d7ec3b8e3db92aa21a1d87fd1c749cb096295583672c440c08b977c62 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d7b60b20a31093c80816b18f02889846 -libLLVM.v19.1.7+2.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9760c4d2958b5229a878c32151f875c27ea2fb37b1de64b18576563c51ed1e99e248dc83798c7109f7526e235ebc72e2f453c5b31a2f6502dfc3c68b01b17f7f -libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/bf7a932266b662efb6ac692d9e97304a -libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/d7e8495d36f128e6e5041897a35aa686dab6f603434d35cec09a61c6f7709745012f0c7761c1d6047ab5b06c1ca094f4e88423cb67177534650e39d1aaeeb41c -libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/096932cb91faa8edb4e7345e58697e3f -libLLVM.v19.1.7+2.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/6b06e807edb350e92b43f6b2a2ce3e682ae27c30e270ec86edbdbb8c2c1f6b39bbe2f29356b390484931c61f3ac11ad1bed7d7ae255f0e1506e663ad4e3f2944 -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/f52c04d48c275818a377962066f8ba56 -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/07c2aa32b22bf777ad8a404eb0ac40cf98f1bfeb4b61c27e6b7a6094454ec241cc964d24465439e78461469c05fbeb94ad456307a8296be996d8d83dbc84593d -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/2164af5b06a5859a69c8ab8be960978b -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/9cebe82a03e0658498b57e343f1d8d354d0d08f5a343cc39bce5f29bebf93cbf7b2106a7c24bb6622fd0b2fd2c6eaac8ea10eb75ac246a422b729b15e9082fbf -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/4f5e0b71aae35af5466b1b8f86d46814 -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/09aada9df91b2dff204d2007cba6d3f53e2d581dfbd4598dcc74b3c8219eb37e9f11955c8407a553588c663fd8bc8051e9197c8137603a9b6ff77f787e04347e -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e29d70370604472d35c0886a72f844ae -libLLVM.v19.1.7+2.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/90ed51b8ea3ea52edf66026a35eb9d46c2e0e81f0a250e037835978e71cfb6053cbc80379203a5fd64024457134abd49ea90af778e350bcbfece2fbb43437846 -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/71b4fa440cc6156ada7470ec602b0274 -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/0959f4ddd8611820ebcd78c664a42f47655fb634fc3f920d6cb96e92174b7a7f6ea4dda893fc60c0f7c68e94ee5005f730c0b8ea173b3b42da153220d1c44248 -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/4b876a413a2a2bc33c56a80f4328d91f -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/1eca49b4784646b3b45b338a48e896878fd4e0e968471233c8d3af4f26016b03a790ac91cad9cefc2f2f19930554909a7c1e119670cb744fa48877c072039d6a -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/3bbfbc74fd56dd9e7292bd8d9eaf977e -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/71b85345b66f56382db362c80a75e8575d1117b367d23770c91b486144eba8516ac925b8cae72ff5b092b3a2749f99fd1594df221becb463048f7e57babc4eec -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/62c6a92e23f798224642c1b34e1e510e -libLLVM.v19.1.7+2.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/5393936567cf46febf5561714bda850309fcecae68e184e93222ebab61376192d3df88da6c756c02167c7b46e5a4598c33450eaadd7fc00e8a33d3df7ac0d11a -libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/88e19d775f444b6243964ed7b9d77e1e -libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/c9bb2ac36fb7f1be1a940aca0e8d9fc67be318bce118930bf1db1c997f757bd57424786e719e18b428e03095a289a9f5e07cf421e68fc7cf7cd223f136a6feed -libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/4d3bcd16e07c9b777556cbbc8ef5469a -libLLVM.v19.1.7+2.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/9f112fb9d304c4f839f412d2925dac2a5165ad7a6dddfe3d22260c00acf76df4ca494dee405cb02d7faaec116a01da3feaff9f532ce39b49e185d3eb6c858102 -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/c67ac7d99576d6d5655be59ce48198b5 -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/bf6e23826dfb3e2464fbfdd27b13bcea51884d2e77caaf9ee4cf8b6ea2c61509f19f66e0fd6d754880b74f23f77e5634a7cde57b70dac970712eb6645f79bd9c -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/8e6adb6771dc1b4c33db58d18ba84411 -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/83759fb9be7f80e8be8b5e35ceba6bf38aee176378252a4f673b82ce08bc6ec35fbc44e5f7c6ef1f421e12338803a7b71233a4d72d0c7c10c8aaf35092d563c7 -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/24c9936d93198392ccf7308577737fc5 -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/fe08b4529633f8ff2ccf3a0ffb7590e5e06b7e04aa95d6ce1a81869c00cb2a7dfe672ebf62502673d781d2d1a5c6d16a892bedb2eadcee40c629b387230b40cb -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/047883d414e84a470a537f60898ce09c -libLLVM.v19.1.7+2.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/08da16bddbc2567d5c3b1dd13777625c1dfdd3b9ebb8f655e5dc5c7eff1ed5b5cd352daa9fd2bb4b841ae9e92fd5fbd3975ab0569801e12ac48771623be708a3 -llvm-julia-19.1.7-2.tar.gz/md5/cf75ec3da324ff16892bf530a0f176b1 -llvm-julia-19.1.7-2.tar.gz/sha512/1299591bc245c0405866ab9d54783af2ff85d735d321decb0e38241991e5f13c3f7b4d8588664f1c054559d283ee19cdb99da858d76f9f3b1d6b68d5292bf83c +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/038cfa3a9136493d533a122a0da0ba1c +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/dd7921a4a056cdae21a7b04c862d5056776a42e91e29799c1daf48e12ddc049140469ae12d342a87931b7b83f139555675051ceb89bc9930e6683a08f82d4da8 +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/26dcdd037bb2049b555f4b7da48b6ab2 +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/e39100ceabd0304eb6fe1b02292a9c25854c8e4229e36bf9831bb386bf08a2d2cc40719163111cfc624b6da0498e59b853624aa1916505d4db9fed7d799c9ac8 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/ce2172cfae9bc13e28f503ffed4c2bdb +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b4a53a1a1ff9aab16fd7304772096d73d9716831ec5accab6016fcb9e51e5ea83f8f17d9342a0b03837e710c4f88f418c300366624924df1784faa35c9adcf92 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6c0bd66796ef1982cfa3ac1cbbbf0e39 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/ed2d9753e7e885a6eb9d6bd5de186df8082280aeeb797e7c38dfa3821e1743ca9abc2c735b917720841c54a722ab44507bb125bb1c1e26f1bf2524b0ab0e5743 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/22fa2895daac28d10ad9403ca77d5693 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/29ce790dc2fc9e4c0079ecd5902c1cad16cfe2b176c9172656d14f9792f48ee3064bae94479fdf0f41c062842325a6c7ecf6f453c50304983ff48a2f5794e155 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/00aced1879f83691170582b689219827 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b0f61eb2a84b8a445a066abd54cb5c26dfeeafe02f2980bbb3216e8bcf562ac9890bd4dce89940b36937150e6f1d0be3986e3943b9240ac73df83e147dd1c2fa +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/e4f0d850f1ddf3c3ded9f9c783749005 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/31761e04c462a7bda09e6c85239ee356318cfdbfb353350faaca6563e60698a9916e362b560c8f28b88c3bbfd556d7fa1631232307a9c67129b12184f6b34a10 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/51603be4b6e55ec5395624a1ad89461c +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/b219ff439fb028a1f5c360c3dc38c7332b6abe107f9fb13ba22cf756bd3ebef10d000737d8be1569d0d81294bbc9af6f6e5eb8aea98e90882775523baf20151b +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/217b079b37b2038dff9905039ba12984 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/a5ffb0fca566384f014fd0ea0fb07977eea1f96c0e76a213864be472ad6021a8861055a1f8353dbe4cf85dba161d113240e732eb812ee1b821a3332e24d1147c +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/7fdd788e1993ecede1e7e1cfa66ce1c1 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/8483d21dbdbab20b40f453e776ebaed3e0ebde517646469e3848e066b6a69c39b9102e1a37e6d97725048423e6335313339b945f446cc0cb0f2b5c9cd336a742 +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7bf432f28eccb363d91411b2cb7e0dbf +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/d374ee3a252a09dbcb5f9ce516e1dbe1c5fd1489d07312213f57b690d245fd06d9e3ebec3ee399fd24222fb6fb9bdee92e7aa58896d2954d7f6c821ccde2751d +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/8be5b43865cdc03d55fe62166b10696d +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/6bf57f66b04d58643f0fbb2cee3c4138205e95f4978136b5b472450a5a6481fb69e968e84b63f2c7a4c6e4a4107e96742d04c30bfc7ca1fb3d64d0247c524993 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c1e18ad4763fedad78b138c8361f18df +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a7bc2154bf8d9ade8d346a3402995eeac3352dc7f3d058850d34df538fe004f8121fc91dbe1ceaf059b8215a5499bd03c71f2ac5db0a14325c3886e482c2a260 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/0720fb4c94a4d13d5eef479b99544311 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/c253aed3e4f7d0704083417631a9b64610db9a81c636c37050b194084631f1d918217dee69a78c517aa4972c22c68d96c64f1d3db067cd61ed7106f2e548cce4 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/df1648374d8feb6beb345bb3d048697e +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/74854e8f5fcdb068e13b6b3bf83369888d16e603a9cc4b524c7edeee9c95501e7bc39dfdb119696c0665914947b8363dae72949f76fd74942e15476cb5e5ed55 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/cd02804f5eb841693da41db5d6b61472 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/af22ffb9becb1dfbe529dd30975b258fd1a7bf17f416ca326834f5a7ec534e15a8d74f08b98087de457492cda5c7f2bffb0ea57bae5025cd992324582f2cb3f6 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/4519bd2b723309739ac6b1b8ec9b40e6 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a184299ad255e79b91232ee079fc8617bfd82b3b72f46c62e39a964b1a7bd664e134f0c565585c10cb7b78472417eb8fee0be833155bf2f2a42acc73f8f4f9be +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/bd8ff4a34fcc829c83ee9d634b2925fd +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/576771e02790b532f89b23a6c9df3434628d0bb48527e39a071bc458dc7483a040a6b0c136de183f3bf42fd95785418aac6cb85b8dc15c69245a8f6ed44effcc +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/4b9f611840f96c4dbc30771c72a93739 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/c10735cb77fbbcbf05e89811c87090fcfe19fd19f86cd2eebf8d09ff6d02f0b23c41ecc6a6c0205cf889047b3b5332aa0e362818369069e35212c2927a49a226 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/f5bd1acb396b3e8d87aacd01dc00a700 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0b0647af108b854e760ddd606ea9fcc0576793d71b6fe39e0f355fe03538e4ba2c8eb821a7468faf382de92adfeb2f12a07d47fa9f7b28cbe304d1e399bb0720 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/f3ff46e4cf9aea3fc8834d3b9e777048 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/3eb7d36b99e30c44cf189df640ae42316e664e6f4ad20dc83da4d1fe87a2a9375bdc4f54c0e751a6a84d97937c9b5add880b7413000d12e3ec34daf0830efbc9 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/d1644c95cd080315f31a7848f07f1866 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/22a438b6133ce7aa3aecad386e8e446b8373315bce6224fb41f183f2eca40f9c2a5d529221d3224f645526efc3f112b1dc6d3fec0c56cfa6b373d9f7667d217e +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88e5f3943425417e11e137ecf45b7441 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0d1b59de138a317a2e19860cdf4fbdef0e536cd3d22d1b8f3fe3e2737e08fde3521cc8e40562c5a304fe6e78ad4fade85af16bf7dae9665ce8b3e0b5c4212eec +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/7d57a0cd005d10393c27414e5294ae26 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/eed2f929a73f4cb17c0e338f7e06b10f9ca0973a53c0b8459a3ec60330c25f4b582bcf0908bc3f193853e7a849ac3da973165c71032b0f99ca80997575fd2f0d +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/8947b6739ff8ba38c5afba85bbb3626b +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/303094d1223c7cadc9f3507f79df7c97c3ff53c40d93136b5ce622403d6625753329b718f23a74fe66d1d31c1517c7ae3574c86c88edefa15671cf236e6f7a00 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/c72a78b88c7272823f5fda84ed05b379 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/a19cc0c639b4724d00cefba060207db252794bb3e14dcf0ffaef6787d6a4c3e68248cb62197fe4c9177098c760024ff3271e3660599d25f6e336d714ff2318eb +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/07e0a1e3f46e3601216188222c585f8b +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e56c24feaad72b16d80e63509449501d37ff5c31a84ebffbc4eb31608511a7ca6a30f6eeefefdb82558ec2378fc74d105acde60d5b930b4fd1bcff35c52ec210 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/8831ec22c84c9737661aee6684b7951a +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/c1205e9fe613538bde8a90ffddd6e21ac849b601e55cf48cd257eb323f2fa52a9ecf0a1bced274efe5d78ea9fdf280b6bc3a21943d2ccf47475f474bef1f59bd +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1804610fc03ee2f414b68722b1cb4346 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6f36e726a78f02507c93cb75ca1112ec41e99b2fddef8ebf31090996d9b4a5992186062f1fc120b67b19c6da0c905ea1bbe404774eaf4811bf782d3150160554 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/bda9d895d9ee5ab01b12c830b31f7b65 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/f72fc2e9d434b0de616a8decdd43145849220d230583284d9ac4497ecd4c06a77589db63290fcc3aa080364533d32c60cdd63f412795f8bf09a452f643447906 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/2d32e54cd77a9ebdc4e8ad09226139f1 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/2ba917662ee8a2a25827394207b25ec313d515ef4c942dba14d1ea8abcfc90c4adb472c557ff61c8f9f9655a0780d6addeef00a689852a5add01fb7e1e2f5b34 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5d96535fd770ba7c782ee5ebb14a9341 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/04127a1e3dfbfdeb1b7f644adc5832062f649068d161002dff376670dab7b1cef2d8ce91d2c7c1254b935f4ffda1699cbfa30f1b189eeec8c662dc6a00a351d3 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/5ab83d6a3b2a80a0ce26473403e9d6fd +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/57541c1956b6db41d6da6ee4f9ed8e81946c6f50e6e0e9e2d88f5c2049c13ec2599794f92a0f87794ebe13ab7bbf345e1b5978920bd4b4a9419a0360944703fa +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/1df5afd05dce8bfe48b4de3c83a1c169 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/8f6711e77fea3aa28c6c752a93b676b062cbe95b682c7834a7d290a9f2d37c8abe1a44daca86c73f31dc8d1fad5314866c63e766e8cb581a63a390e49027479e +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9f7ac8c4df65e684f31a77c6458cbd8f +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f01b4df2f56b3a7fce3ddc2400a71a7795a5e6cc1392f63ee2d164f35acc678db4b13ec6ecd73b716e6405212f15362b751b5e4fc6ebadb037577906421d3728 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/b6c3ec59f73a15d6d884b75089752009 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/c2d79fef639b06a24ffec333f7748298b19b40f3c1dbadadd609bc1c6a699c8ca3950a7f4d9715af66c01a74ea22addafa642509e61f71a13c7303ba97fa1f70 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a47722f625d9ed01e4e8c89c7e5ff361 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/8a581faac501cfe8cff7b27502e8bf5108135eb60544ae2988790fe856caea26151cd00c430c12bac15f56d41357d862b97daf34b7630bf6f3883824fb1700c2 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0deee3e8f23bb78aef9ab92901635db1 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/7c9cb3c1cd8b7b1f777d1ecc6bf3fa5b3536ef4bc74c2b7ac8dde9c82b52653890ea0c398ff4a53c9545381a4c1534ee6b1be3bbeeadd8b1d39d34ecbe66a432 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4cdeeebe845a999b0af9ac967cba2c32 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c1cda095c549fe4f1e8a14aa9318b0a151653daa10f45598fd1e3f72441ae98f69f3bfc1ee1ad1ab15f88243081ba7a65315ce2996510d09f3541af4a243d63e +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5ff186b5b66d3023298b02d3d1ba1ecb +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/733ce7757004f1dd06ee768389edb13dc86fcffcfd0678b90bf296205492ccb5206042f103ea65330aa291b0ab46208906d8fe513557bbf8da7b05c4e098a07c +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/07a298aa555905dc38733075e37bacfd +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/135deff2656e728dfacff4383e095876ce85da4f8de5ed178cc8919a949627f1d5ab61b2c4dab7fdc98e363b0fe630632d5a9bf9d6d074d33f0bc7ef62eb7ba1 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0ad39270c0136606cde9bc37c6f84540 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/64e3d2ef2130e83eee773798c968e65dad02cb0313e48946e6f8e3cbff98100d246b0acdaf5e23cc4d2a61cffca2aa237a7909887dbefbf79ca48ad6f7a80572 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/3e5b8898f2c67fec7ad5eada36156686 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/860dff38be4614e21e2e12349d4f0a776cc96aee3870254b42a0f2851e0a3186188486d47b1f804b9c1670b6eef2501a66b758ad6c3f3823b10b65dc7128579d +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b0c9231cceb48572296606fa1ec3b763 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/db16e2d9dab7d88c27b33218f075442fe4627f00ce32af2ebe3f1b65bd5da852f91047e97abe99e1e3541fb42e592b638d314300ec870646af823e29afe3bcab +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/16aefb3daade4fc7838c7f40b4a9717f +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/54e8c25636ebb8a86edef578d039fd4d23859ed2c012025b872af5d05f6b485b3825d88cf6fc8027dc44b881c84bbd911753d6d70a963facf7708b92109707e0 +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/4533027acd5733377fd24a971b1f0987 +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/8c4cfcf4f1961eccf3ecfa48ce55ae6d5b812afbb636dd59c1889a0cef60c59828e42b601528258584785fc0ca357e1b57ecd8cbcd2eef0aef9735b63ad7a778 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/eb1c74229fb43d9f8f5b3917222b37ee +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6fef5953f3d03e9b1172feb297c8cec7d487cf38dd7d88085ef6e41c719920d2fbe4c62a1fea175600fc3e839c719385d7ae6ae62c606e0294d5df0555d5cec0 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/86cf13ddeb512f99030957c9d223b411 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/59b701fa8b2dbf3afddbc846f6e4953f1b18abd74ac263cfbad815c997d57048217a672a9b34cafa9aaf6f87a636d400d008762d80ce35de8ddf61978d1bafae +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/73c8986b390269b2f2a6e14e1b60c7dd +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/af8338185d68774d690ab0b9b3725d5752ff26a32139a0f596780dccd7b124661c38561958bbe5b3c26258a5587eb7a40ed67aebbf0273d238b90d12020f8c65 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/4ca9c2962f62b45c82104dff43f7864b +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/63e580c4431d86925652374e25e00cb5d76fa72e48a2a1d6dcd99c2a6c7236382bf6ff696a12869f4d008ab2e9ffb10b84f7faab1b107816a8cf01dc1ef1b133 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/eba4632089f7cf8785e8aa122068eee2 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/adaf858e0362ac2813f555e46da82427e2366d90fa0e6b7b103cb57a98cda6586284923f58e91eb0d3f888fc0af17d0ccfa7a58c5432ba1f99cd90143febcac9 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c12059240e4f2d78f99a0e0e912bad66 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/c14865f4088e4dd31c89738f21cafa6f34122a82bb7c20c9ca8f184d3b9c76c2bba6dbb3c9caa1b1bb7183517bc46e23f070741f0083c8bbf7cd4f3d5bfe89d1 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/4c2e4479005f4767ed04b216926031a8 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/fbfdc5451a652bcedfdd6a870b550a0c9dbcd8feae81a10792922f1972420408d5e3d69d675768d45cb7b893098b46493982669af55e6892b45654d2dd759caa +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1c78eb7776082c7555ec61975df84e6d +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/5fc325293747d8bc4508256f016f5ac62342e1dcb58f3d47bfe9027dd127564dbb7c321f1ee689ed2983d24abeaf79909156167774c3ae20ca7697647f7d6387 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/6850d32e14a28529daded9a0a74b2f23 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/9c837940c27307e67fab0a57b5619a44429ac85edd0e04c4ea5fc2991312f2534f606c2a7b263dc5e82b67b8e33447e3e29081862208e2fb19998d9bf59769a9 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/ffe2beee649f7d2624a04616f7bdba37 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/e637f58309e61c568a2e3d28440330f0a8dca3a4005e171e9fe14808c1f25df498ee63d46de54a8827b99a78bf55b0323272bc1e82356227a26b2f86a5e8dcc2 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/fea1960f440c1f44161f7a2bee756073 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/97ab7fc9977fa3d4a7442554385218d56479f5a87420fc75a8827d9044db5ab52f3a46a9ad71769658ca08abac9e25df15f456d80c25d3c76afb98ab2855abf9 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/f71a246f402b1f4f5efe60e09e2f44af +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/568a39ccf791eb983b0d1a94a7188986477398308712425ac1007387a2fa0df99e00836046533d3891ff2b208ed4218289e0953ef0f1b4ec1f6127af76b3f066 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/b66193be08fbb0b07181e333a8857f08 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/17c9e77e907f40e799da4b2d21cb36e6e10bde6ceda6ff29c5941e5f671c8625401f6079f3be0de710b57475f4e7beccc1e7e64824c334286efad7b21919a7a8 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/122ab2567ca9d8824f94299b45e8b1f3 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/78420eaba62e18541443f4b1d846fecaf95b58e93ee7cc08c4818339d3a0705b0dd82dadae2024f6be65b3b2e26011269e319eaa9739af599b640486031b60ce +libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/030e6925d110a3e1e16e21b695aa2901 +libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/d1c083099ade0186a4efeec1ee402aadf511dd2d01ec79896446870e23f71e16a32a80623c487eadf4e4ba659a02fe8044af27de3011ede133ea891159fb686d +libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/4d29fe04a66e1c5b90396d2122f3a6e7 +libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/b1853885245f907985e51fa3c2052f2b9e7add0af722812f9f33175c3e6b5addb813283cc46b2e81baf32083a6f412c6cde5261b9dba8a54e2a37edbab236c32 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/8d049c7597a0cae940a28153584084c8 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/35bbff689a87dbf7eaed2c2c98e90ed448acb6ef518bb23501ed05a67de3df53978dc28104b1d2607786486466905dc399de9e858b141f394510bb72876fcad6 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/42bc10fcf6c6626e839e59e9688f2fad +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/74d95c89fa487f3f46191094d807d1fd258652548969492a0b37f2290a82eaea6dae2fffa77af1c5be5b80bef525e7a5601c432ce308bbd874e063fbf75d5db4 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/69331507403bcd85db7e0d3d4a0d7a2e +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/3f2d96f8beb7d95eb1485c00f047a5050e7994d12ae16422943ed9a4dc9530c949622539532af5db1e655fafb9d1183e3e878a9608f12a6ef9a6413498249c41 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/3ad71df03561774b42394270a7db1797 +libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/2b11da0f3f94e29d14a0bdb7364e61057f9ed1f82b8126a46f0c0f98888a4173bbe46f64d404b71e2a08056f2b550ac4f3188409e456506ee1fcf1367af416c8 +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/3ad1cdf720b752bf5d37cc20c442ae66 +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/e177c5f3bee869e2975849e2ebf2d139614774dbeaea95f423956afe9b86af9cdaeee13a03a42cd439402a5bf96fa0df67ffd54ad7f5a645fb9ebe837c9f011c +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/7e2990489b986c4e1dc411d3034420fe +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/bfa1fcec41c2dd985222d6dd9995d2ca7708d314e5e7fc7085022a47ac32a6322edd3b415026d82d761082c4716c23600c262906ac1f283cb0209a15c549b372 +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/60d9336b463cf0d953733b072f58ed7f +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/77da5e8ae89e3e943e9af4e24388bae4e1ec1f335b4e131abf678c44d641d447769516b4eff1f2908797b80fef12beaae3fc963c408d8cf95febe6e1c4cca130 +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/6836377f50152632a26b45bf3f5dfa7a +libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/7bdca756e9fd87b600320fb82c6f9cd60afe5dd534e136e25dfc6a144026f7b987233e2e33d336bfcfc94fbfd39b21e91ea9eeb5a0c63468c4037998c3c6ee33 +libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/2ced7dada1583104c0eda06ad59edb4f +libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/d7ffb0627e3012b4dab12418d105fe431530df8334a787304d31e74cd29094d5df04e17ec333e292eaaaa2c8b210db28f7a368df769a3af8d31c0c2908a88ec4 +libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/bdbfab4aecfa5519e4dacb4d2bda09d8 +libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/405c1d62d5fcf59cbfb02382ba7827089b6cce600b04df670781945f8e1e8c9bc9814f4dca55b956f73c068689a2f50d558453d05a0d4bdccba99e71aaa7261e +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/fd148b8b11498c0e6f1587355ac29f0c +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/f24dde988c7d9f42a5baf915e813d93e7f16810067b6d12967d34fcba18f06f7dde2375b093ba18a145ad82d6757a1a853bb3a1c68009ff59b4edafa58eb30a4 +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/f07035563a248217e9447e7dbd8f1e3f +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/e1f02f575cc977d13c9f093a621dfc11ea9c73cff2c0c2e52598a644866096885dcac3c0944bb7ea7fad4ddfc7ec588b850c20ba464c8ec193bfb05bdcd18d6a +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/03b30ad82bf9cf3b3ffb454452a6f147 +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1349100776b182949f9ef65fa23484dbd2dedcd51aad1e02eecc47c7d7fe556c43e2188c914d5a7d409e1d29eea07709d6d76718c7dcaab4fa8ca3d3866fbbf3 +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/57e3da5f04c49d0fb1bc74613aa9102c +libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/09f34661f8a27b2bf6f800640beff3ff4170e18f13f35ce97b6e809a653744ec59d08b6d90f38e2e5b0cdcd7157ee1541b96c270ad6aedb8578036fd63dccdf7 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/8a0b721da8e29b9a3f8c6430f6316df2 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/166fd97bea310c2e87c3b799693532d8f1ce8625889fa0201f114094140509d5e570fff87db6d216aa38c331d5f06c9de81b7235cd5b98c4404e3c7320beaf72 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/8a052f798dce9b93618890010cd1e06b +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/4d070d56d425788ec20aa0c258c845bdc7a42078ced5501aaa7bcae906b2eb43eac97fdf52e4ee955e1dfd24fa5c95915717d501977fd2d98ea82e04e708422c +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/69e67d41dd400119d67e632a63b8b852 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/bef7ec7fd19b205c1fb0fed09a82c7362cfebdd4e62005195f9c73ee3b6e31c4216e23f50328fde2ff96f26dd55d8dacf5c9f3e70270bb0c13a73f08b6ed65b1 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/5b6a1ef8bb57be87f2067ef9816ea782 +libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/e75ca64872b5b4794c0652f07f8494c83cc52545a7de3d7e7da3c2290a7c064006c434f82f979e7df5abb6a75f9612478244c6048fb135d1d2f1f3f28328e689 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/869483bfcd96dfebc20b468e5c4eec12 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/3f129d028f8642fa1b8218d9eaa7122c305b9f01e66fed9a0443575d7acdd4f23e45da673e5df37260911797c373d8daac7b37a40667b4907dabebccb0ec92d3 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/d01c44f20732eca87461aaba9bf90f92 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/193bc6c8e3bd6eb512c29da873f63777da9e9d6ddc0e96094c5828bc6048b8da96517a17989505d1ebc43bb018fc034af94c9c66a337e18e86a34809c5470a08 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/47c494ba09d7fea196aeeae3e13de6a3 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0beaf19797f536a1083a277c3b655972337d594315790722e7c970ce7ef8974479cd77d92d0b0e403139cd4b2ba33885c2604e6ab5e07d2ef15c682353829466 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/3257fb989b619b3fe07ae196f2b84ce3 +libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/fc93031f73fbe7883b318e1a26b85a354f93cbb1f32befb5399c0d89bb4dd93a36de6839d15465c00adbb5dc3fe29926adf9b390b0122ec34afa99b64bcce60b +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/0e17af3bf23a8626ff758e40ecf9d83c +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/8a57ff2efb34363e6a9698932f4b1265067ea9abbea3ec783ccce23cf3e6e3e81e31a5a01d7382487e3709124612d02b5b6e2ca5448add1e583d25bc62c8bf1e +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/a9c8dd8a055704b192980b295f5f8c69 +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/59a2af6f5970b0a1836734f336acd55ccc8f16eb1d894e0c97c7577a0a5d35d675f2138fe736110c1746ab63d0e10238d975538c98c4fe686b64dbc2f717d7e0 +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/23afe5ea859954e5672636a743fc1f6a +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/2c07b81805b2edd7ac86f0a63cadb607a4f854945828e6daea55e04b9fa2491ca41259b67e46fee0ed7309fdeab9921c27cddc97c5aa1bde41e739b80afb19d8 +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/6d6c815b0c1955148c800d17d1e9a5ef +libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/d4d57e0aac6c813bfec09ff4c28e39e474194b0c6ab1cc11dcdae3654c51dc9a729aac8297715563bcc3fb89502dd216b66a7d35eef76bff93effe8974b191ea +libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/6bcf33d477fe67e86e8190aeb29da648 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/dfd44f521064472ddba0f123820e6ad931e3a5420531c71c1ea165cef4d6355855cdba56b1ee4543b5e15d0e516c38e3a15e7e18b27ff5aefb4abb476faf7cf3 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/51ca42bcd058e87cd9bb146c1e705d34 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/fde60b8eddd37cec6e1dc3445c11b0ec8dc37f3638e99b209b55ff28d92b557b797a571d98ecae79e71d629f1be33edd4338df83da9379dab578fb7310ce0dd5 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/abb40a36c1a3d77f3f2eae4d88729a81 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1e48ac8aa697ed3827a9cef46a008bdf609f9e2aaf66981d760c5baff9a8d0d63eb2b6600544c8b8b6243fa62f5832a8fc1a9d77602c135336a4ff2bb570b170 +libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/c9bff27fe92407e5907852363c3eb6db +libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/ca0df65a0f2512984b264fe44c9edea65ee6e92a6bcb2390259da40d4d6a9b5c58320459693b73822da23ffe47fbab72b1919129336b63dc03d26530e801c1de +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/26c401ef44560569334008b754391973 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/dc5eb72deff4ba26e838dcc831bff32053c967450c8d0bb95ae4a25e73c864b6810f580d4f341e6aa47d0fa91892ff92aa4f45504415e70d958c466fec81e062 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/d5934a0ec319c4dd3d1043855284ba80 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/dff3bfb91daa61555e91a3120923cfa7642dd856c0d13c988273d4b135a419d22fa87c39d7552594077e05659c3958c8bf501e171d671bd395d72b4d0e5e99f5 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/f6734f3af26b223c67461753cf3711c7 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/ca807029d5f3d03b943bb2b18c43b17e8a776f30599ef65706f3ee71f21ab5d17225527ad3c7d48bd52ce2b4c36a56ef6424c68530fc9f414e643df257cbee99 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/5ce381da456a1f3b76b46c4d1af016e6 +libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/23a52fe4084a810a2afaa86d521041c493f6e010724b62d8a5cb206e82f02bd8498299716f634e5cf4359c3e59c1702aadaa48f3d9def41193c037cb12d6cfb8 +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/9a19b59e71c055a79d1b07d560c6325c +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/628adfe605ca296280cd2fa6771235bbc993e52c6fd5276441f1a0c715c14945865ee78e177afbc2f60e156db7349fe2aeccf656d8754cc8f8891de8b24713ef +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/8e8ffc26951228c6b8c4e4df786a6402 +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/378f60f7a8fdfdb51177f10730ec7d3a6afadb4466c2d2c5974580e1748dece6e07300b445db0f47e2a2067f73770d5da0b5fef443150e2987e38c27bc99ec7d +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/80f50299632de127179fa6a3cc907a9e +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/5b0855c283503d86c110bc37adaa2bd465be691517dbea08804b24051ecb3ab59921ac0e4c1e766d1d46a33a18f42bc87e5928765412caaa29cb8772995a5ee0 +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/c2c727419798d4bb3ccd32ebb81d1a2d +libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/8d6781c532b8c60c5f2c961cb076f4b303116753abecc1ef1656eb7ccfcad0c553cfbe52a0e10dd449a302d4383917f80ad79de6b39de6ee4ee865ecc94bc766 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/ea09093c412181d9e32a12406ede691d +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/d3bb655c4a276a2d16c0f4bbec67fb5cb252bb12850e57b302af7904e3b8bd708ef96d13d7b3b392e2ff63701b5abd8b50f3b7392442567188ec73a068c3d53f +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/740da054d26d889df5f898019c2f3c45 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/839d613cbca89fbbcfe8d4a5194b255b1376fdf6054c74189a62b0de0a4e0476cfed8e077b4c1b168c9d6ff8fe6a84a50912e6c19389aead718fffdf879868e3 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/53c1bf2aee12c342d0c7f7f26cae75a0 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/87778a19952829547ca0cfcc6d0acdf007858dc387d4e00b569a89ae2094e860cb64f4976ab393432d3bf80c184aebd007a26d4f1651aac9b20a1ff1f0c70da5 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/54990164de5d5284c221725ee492cd77 +libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/5442a878cda167e15c0994c6ecfde1cfc2cd94bbe9b27471f16dd2bf442e90e5e402536fb007b816800cdab7071b704a3fefe66ae6a38f6571e8f7a898ea494e +libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/dfcf38ade854f883ad0d453fbfcb146a +libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/9e6a9ccd8a2d397bbd0ff37db7044a476e1df136a7b79ba145bd767e8d1af7f902a7a6cc900592488268d331635ca84691f2d7f68c64d7bf806d3b9ddeb8f550 +libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/10ab73c8c0754f8bd4d6535fc540dd5c +libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/4a16cdb2258f013f8e149b435d6e86393c2ccda8729b95de7744109addb2e229b4b52b3128a4324f71d3e4ae4a616daf46f4fead9a5f445a5aaef8f8491e269a +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/67d4b8f59f623665f380fe5b9644bdc5 +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/05db739257d31805bc5d4577e8c0c35dcccce5fb7b7581001758fb596cb528d7ab376cffea53fac4f16adc7d9a941d56fd856095bca5c6b86a2c355303e81b3b +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/21da228049c12e06b50fb4b3caa2fcb0 +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/9f93640383d8064c45c101d87b99b4df3096a78a71bf89d40ade592fc3bd1d7235d01681a68bb14b5191a79aa06361c242bab3f31d9c5053a685759b2e522719 +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a31c99673c8d6e3be00f442b8f97e4a6 +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/0004186ad8d28e18e112e548fb291151f6efa74140cebdad92afd34c46425a61ffb33969495391387aacfa812b927f9784aac55c00c0d57d76e8c0ac3063616a +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/eea6f4fb592967f4c137007ad8693852 +libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/7ed4d32d5ba2f8582cba63ab992c3dc6c1c0002a533e437696b6cae2ac18f4867fe4d897e0c5ede64f58d88b82a87b355d1699e24171f2e1b84e1ec3d0e3a738 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/ea6487b87d7929e4de8a7ef858708ce4 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/8e4a6ecf3827d62482e8d0b0163f58057c18c5d9ac7c05c25db0e1f9f15cf1c0ba9ccc1acf2caa686792e5dad9ee5bceae0051fcbb17fbcb6bbfa79acb509c5c +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/b85c25315f0f7d5ad800ba822a48bf51 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/9965dc687164c11ea9d30ebb4bc0bccb5a9f9ac220b94a98e1088a938b25864e6321100efcb211816b89713081e4cde89f2ab2dd7030cb297cb7e3ad49b69be4 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/63a75f6c13445552424993b84658f3b4 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/1f07edd4d3456da3c3970cdf0d4892458bf6b12d004a1f823e56cd7344020648c1486bbfe02d4c0ecebb6ce32ebe6c124375ac3f9c13b618d0e6bf36aed3c84b +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/d9d3903abc3266d960e50f680b7497b6 +libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/dd570e511be6e63f29f2864fb620db75e489722c0bd41750164ab7689cbeb0a4d203634f457bf4c65e023cf7b33b2a071d78fe8909f8050102bed6d396da4017 +libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/1c389de1de869c3744bad21da4844cd8 +libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/6cda6f023f3bfd6f9330efe61405bd3ddc062e72918a95bb6d9737f54b1dab04def0736abb52a983669b7059744bc9ab7d030feac29c5335d82a42c9baea27cf +libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/e6504fb9ea87afc61baf598024f677b0 +libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/ea974e940095ec65d4d5f71fb0050651c3824bc18a794f47a83a22ac419d30fe2cc404ac352087169153274caf32f3014207f43e2afa6bef3e86c3d5867b006c +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/112367628e6c3fcccca6b1630f37b6cd +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/949be09b5d33dfd3a43cde05cb1d009bbbb3c5515e7bade0249398ce6dfe5fb678c0451f6c7bb109e94c9ca6bc5c07f2f763cdea4231ee00bb911749f4c78e9b +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/8057b884d6de98245e819463042d4e96 +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/ead00174a7a609900289a1c3fcb9a18bf7945a857292df7ab716bff42eca7cd83f9ccfec90a8605e01070407dbd59f8d638fd29203c62e5179c774eead5900c3 +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/715471fd07647a7198c9cc4a6e5cce15 +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/807c97e4788b958eb158579a537381831ad3f357be17e12a38acb0346dc8a7404afd0956a31d1d7d2ffcf7c641bbaf5e5712ab7df8ad2d7ebf32f4d18ee23095 +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/5d13196d2d8c5adaef257272e0f317f1 +libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/8162eee065749468457d5672011640b3cbf279288ee67647f439c03129a572077b3503becba6d43657d1fed0e685d832b481fee4b2bca66a1e731a7ddb630d9c +llvm-julia-20.1.2-0.tar.gz/md5/aebf84a1b2ea2384d8f7d89ae6e06a29 +llvm-julia-20.1.2-0.tar.gz/sha512/0023c0ba21df6f130b543e58c4934a4061c4fdba9525867da3b7f1dd7a9aa2189eca90c8600d93958da5694780cb4aa2c05683d19817d8298305ec81d88cdc4a +llvm-project-19.1.4.tar.xz/md5/1e13043b18558e4346ea3769094c9737 +llvm-project-19.1.4.tar.xz/sha512/a586f8a41dde5e0d9ca6d8c58e9ef2a2e59b70a86d2e2c46106dc31b5c096bb80af0cdbdb486179e9cc676a540099f49a1c2db9e5e84c50362db1f72e9af6906 diff --git a/deps/clang.version b/deps/clang.version index 48130a21d68a4..68dafab189f7e 100644 --- a/deps/clang.version +++ b/deps/clang.version @@ -3,4 +3,4 @@ ## jll artifact # Clang (paired with LLVM, only here as a JLL download) CLANG_JLL_NAME := Clang -CLANG_JLL_VER := 19.1.7+1 +CLANG_JLL_VER := 20.1.2+0 diff --git a/deps/lld.version b/deps/lld.version index 1b76dd46f5b46..023122efc4596 100644 --- a/deps/lld.version +++ b/deps/lld.version @@ -2,4 +2,4 @@ ## jll artifact LLD_JLL_NAME := LLD -LLD_JLL_VER := 19.1.7+1 +LLD_JLL_VER := 20.1.2+0 diff --git a/deps/llvm-tools.version b/deps/llvm-tools.version index e6740380b599a..285079ca412e5 100644 --- a/deps/llvm-tools.version +++ b/deps/llvm-tools.version @@ -3,5 +3,5 @@ ## jll artifact # LLVM_tools (downloads LLVM_jll to get things like `lit` and `opt`) LLVM_TOOLS_JLL_NAME := LLVM -LLVM_TOOLS_JLL_VER := 19.1.7+1 -LLVM_TOOLS_ASSERT_JLL_VER := 19.1.7+1 +LLVM_TOOLS_JLL_VER := 20.1.2+0 +LLVM_TOOLS_ASSERT_JLL_VER := 20.1.2+0 diff --git a/deps/llvm.version b/deps/llvm.version index 4881d8182e22e..f283c55d1f4e6 100644 --- a/deps/llvm.version +++ b/deps/llvm.version @@ -2,14 +2,14 @@ ## jll artifact LLVM_JLL_NAME := libLLVM -LLVM_ASSERT_JLL_VER := 19.1.7+2 +LLVM_ASSERT_JLL_VER := 20.1.2+0 ## source build # Version number of LLVM -LLVM_VER := 19.1.7 +LLVM_VER := 20.1.2 # Git branch name in `LLVM_GIT_URL` repository -LLVM_BRANCH=julia-19.1.7-2 +LLVM_BRANCH=julia-20.1.2-0 # Git ref in `LLVM_GIT_URL` repository -LLVM_SHA1=julia-19.1.7-2 +LLVM_SHA1=julia-20.1.2-0 ## Following options are used to automatically fetch patchset from Julia's fork. This is ## useful if you want to build an external LLVM while still applying Julia's patches. @@ -18,6 +18,6 @@ LLVM_APPLY_JULIA_PATCHES := 0 # GitHub repository to use for fetching the Julia patches to apply to LLVM source code. LLVM_JULIA_DIFF_GITHUB_REPO := https://github.com/llvm/llvm-project # Base GitHub ref for generating the diff. -LLVM_BASE_REF := llvm:llvmorg-19.1.7 +LLVM_BASE_REF := llvm:llvmorg-20.1.2 # Julia fork's GitHub ref for generating the diff. -LLVM_JULIA_REF := JuliaLang:julia-19.1.7-2 +LLVM_JULIA_REF := JuliaLang:julia-20.1.2-0 diff --git a/src/Makefile b/src/Makefile index 6a6f604f3c5fc..89e8051afcb03 100644 --- a/src/Makefile +++ b/src/Makefile @@ -29,7 +29,7 @@ endif JCFLAGS += -Wold-style-definition -Wstrict-prototypes -Wc++-compat ifeq ($(USECLANG),1) -FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment +FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment -Wno-nullability-extension -Wno-nullability-completeness # required to be allowed to use nullability extension and not be required to annotate all of everything endif ifeq (${USE_THIRD_PARTY_GC},mmtk) @@ -226,6 +226,12 @@ DEBUGFLAGS_GCC += $(FLAGS) $(ADDL_DEBUGFLAGS) SHIPFLAGS_CLANG += $(FLAGS) $(ADDL_SHIPFLAGS) DEBUGFLAGS_CLANG += $(FLAGS) $(ADDL_DEBUGFLAGS) +DEBUGFLAGS_CLANG += -Wno-nullability-extension -Wno-nullability-completeness # required to be allowed to use nullability extension and not be required to annotate all of everything +ifeq ($(USEGCC),1) # TODO: we currently set flags incorrectly for clang analyze, but mostly it works out okay +DEBUGFLAGS_CLANG += -Wno-unknown-warning-option +endif +DEBUGFLAGS_CLANG += -Wno-return-type-c-linkage # TODO: do we care about fixing this instead? (it is not a bug, just a nuisance) + ifeq ($(USE_CROSS_FLISP), 1) FLISPDIR := $(BUILDDIR)/flisp/host FLISP_EXECUTABLE_debug := $(FLISPDIR)/flisp-debug$(BUILD_EXE) diff --git a/src/genericmemory.c b/src/genericmemory.c index b455a2fb36274..3676333ddb982 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -56,7 +56,7 @@ jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t is { if (nel == 0) // zero-sized allocation optimization return (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; - size_t nbytes; + size_t nbytes = 0; // initialized to workaround clang sa bug on v20: https://github.com/llvm/llvm-project/issues/136292 int overflow = __builtin_mul_overflow(nel, elsz, &nbytes); if (isunion) { // an extra byte for each isbits union memory element, stored at m->ptr + m->length @@ -152,7 +152,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void if (((uintptr_t)data) & ((align > JL_HEAP_ALIGNMENT ? JL_HEAP_ALIGNMENT : align) - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); - size_t nbytes; + size_t nbytes = 0; // initialized to workaround clang sa bug on v20: https://github.com/llvm/llvm-project/issues/136292 int overflow = __builtin_mul_overflow(nel, elsz, &nbytes); if (isunion) { // an extra byte for each isbits union memory element, stored at m->ptr + m->length @@ -283,6 +283,9 @@ JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy_slice(jl_genericmemory_t memcpy(jl_genericmemory_typetagdata(new_mem), jl_genericmemory_typetagdata(mem) + (size_t)data, len); } else if (layout->first_ptr != -1) { + if (data == NULL) { + assert(len * elsz / sizeof(void*) == 0); // make static analyzer happy + } memmove_refs((_Atomic(void*)*)new_mem->ptr, (_Atomic(void*)*)data, len * elsz / sizeof(void*)); } else if (data != NULL) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 676b5aeab84a9..b0a1338ab81dd 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2008,17 +2008,14 @@ JuliaOJIT::JuliaOJIT() reinterpret_cast(static_cast(msan_workaround::MSanTLS::origin))), JITSymbolFlags::Exported}; cantFail(GlobalJD.define(orc::absoluteSymbols(msan_crt))); #endif -#if JL_LLVM_VERSION < 200000 #ifdef _COMPILER_ASAN_ENABLED_ // this is a hack to work around a bad assertion: // /workspace/srcdir/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp:3028: llvm::Error llvm::orc::ExecutionSession::OL_notifyResolved(llvm::orc::MaterializationResponsibility&, const SymbolMap&): Assertion `(KV.second.getFlags() & ~JITSymbolFlags::Common) == (I->second & ~JITSymbolFlags::Common) && "Resolving symbol with incorrect flags"' failed. - // hopefully fixed upstream by e7698a13e319a9919af04d3d693a6f6ea7168a44 static int64_t jl___asan_globals_registered; orc::SymbolMap asan_crt; asan_crt[mangle("___asan_globals_registered")] = {ExecutorAddr::fromPtr(&jl___asan_globals_registered), JITSymbolFlags::Common | JITSymbolFlags::Exported}; cantFail(JD.define(orc::absoluteSymbols(asan_crt))); #endif -#endif } JuliaOJIT::~JuliaOJIT() = default; diff --git a/src/julia.h b/src/julia.h index f696fff21271f..143ad9b556469 100644 --- a/src/julia.h +++ b/src/julia.h @@ -489,17 +489,17 @@ typedef struct _jl_abi_override_t { typedef struct { JL_DATA_TYPE - jl_sym_t *name; - jl_value_t *lb; // lower bound - jl_value_t *ub; // upper bound + jl_sym_t *JL_NONNULL name; + jl_value_t *JL_NONNULL lb; // lower bound + jl_value_t *JL_NONNULL ub; // upper bound } jl_tvar_t; // UnionAll type (iterated union over all values of a variable in certain bounds) // written `body where lb<:var<:ub` typedef struct { JL_DATA_TYPE - jl_tvar_t *var; - jl_value_t *body; + jl_tvar_t *JL_NONNULL var; + jl_value_t *JL_NONNULL body; } jl_unionall_t; // represents the "name" part of a DataType, describing the syntactic structure @@ -534,8 +534,8 @@ typedef struct { typedef struct { JL_DATA_TYPE - jl_value_t *a; - jl_value_t *b; + jl_value_t *JL_NONNULL a; + jl_value_t *JL_NONNULL b; } jl_uniontype_t; // in little-endian, isptr is always the first bit, avoiding the need for a branch in computing isptr diff --git a/src/llvm-expand-atomic-modify.cpp b/src/llvm-expand-atomic-modify.cpp index 7b7b3c8761c17..e4152bb45fe42 100644 --- a/src/llvm-expand-atomic-modify.cpp +++ b/src/llvm-expand-atomic-modify.cpp @@ -17,6 +17,7 @@ #include #include #include +#include "llvm/IR/MemoryModelRelaxationAnnotations.h" #include #include #include @@ -141,12 +142,28 @@ std::pair insertRMWCmpXchgLoop( } // from AtomicExpandImpl -struct ReplacementIRBuilder : IRBuilder { +// IRBuilder to be used for replacement atomic instructions. +struct ReplacementIRBuilder + : IRBuilder { + MDNode *MMRAMD = nullptr; + // Preserves the DebugLoc from I, and preserves still valid metadata. + // Enable StrictFP builder mode when appropriate. explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL) - : IRBuilder(I->getContext(), DL) { + : IRBuilder(I->getContext(), InstSimplifyFolder(DL), + IRBuilderCallbackInserter( + [this](Instruction *I) { addMMRAMD(I); })) { SetInsertPoint(I); this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections}); + if (BB->getParent()->getAttributes().hasFnAttr(Attribute::StrictFP)) + this->setIsFPConstrained(true); + + MMRAMD = I->getMetadata(LLVMContext::MD_mmra); + } + + void addMMRAMD(Instruction *I) { + if (canInstructionHaveMMRAs(*I)) + I->setMetadata(LLVMContext::MD_mmra, MMRAMD); } }; @@ -321,7 +338,6 @@ void expandAtomicModifyToCmpXchg(CallInst &Modify, Type *Ty = Modify.getFunctionType()->getReturnType()->getStructElementType(0); ReplacementIRBuilder Builder(&Modify, Modify.getModule()->getDataLayout()); - Builder.setIsFPConstrained(Modify.hasFnAttr(Attribute::StrictFP)); CallInst *ModifyOp; { @@ -366,7 +382,7 @@ void expandAtomicModifyToCmpXchg(CallInst &Modify, ModifyOp = cast(ValOp->getUser()); LoadedOp = ValOp; assert(LoadedOp->get() == RMW); - RMW->moveBefore(ModifyOp); // NewValInst is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + RMW->moveBeforePreserving(ModifyOp->getIterator()); // NewValInst is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) BinOp = false; if (++attempts > 3) break; @@ -383,7 +399,7 @@ void expandAtomicModifyToCmpXchg(CallInst &Modify, assert(isa(RMW->getOperand(1))); // RMW was previously being used as the placeholder for Val Value *Val; if (ValOp != nullptr) { - RMW->moveBefore(cast(ValOp->getUser())); // ValOp is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + RMW->moveBeforePreserving(cast(ValOp->getUser())->getIterator()); // ValOp is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) Val = ValOp->get(); } else if (RMWOp == AtomicRMWInst::Xchg) { Val = NewVal; @@ -411,7 +427,7 @@ void expandAtomicModifyToCmpXchg(CallInst &Modify, Builder, Ty, Ptr, *Alignment, Ordering, SSID, Modify, [&](IRBuilderBase &Builder, Value *Loaded) JL_NOTSAFEPOINT { LoadedOp->set(Loaded); - ModifyOp->moveBefore(*Builder.GetInsertBlock(), Builder.GetInsertPoint()); + ModifyOp->moveBeforePreserving(*Builder.GetInsertBlock(), Builder.GetInsertPoint()); return ModifyOp; }, CreateWeakCmpXchg); diff --git a/src/stackwalk.c b/src/stackwalk.c index 1f3c8d690c8ce..6f8f9c8b89db1 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -326,7 +326,11 @@ static void decode_backtrace(jl_bt_element_t *bt_data, size_t bt_size, bt = *btout = jl_alloc_array_1d(array_ptr_void_type, bt_size); static_assert(sizeof(jl_bt_element_t) == sizeof(void*), "jl_bt_element_t is presented as Ptr{Cvoid} on julia side"); - memcpy(jl_array_data(bt, jl_bt_element_t), bt_data, bt_size * sizeof(jl_bt_element_t)); + if (bt_data != NULL) { + memcpy(jl_array_data(bt, jl_bt_element_t), bt_data, bt_size * sizeof(jl_bt_element_t)); + } else { + assert(bt_size == 0); + } bt2 = *bt2out = jl_alloc_array_1d(jl_array_any_type, 0); // Scan the backtrace buffer for any gc-managed values for (size_t i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { diff --git a/src/subtype.c b/src/subtype.c index 3d50dc492e190..19e5b4bd222b2 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -65,9 +65,9 @@ typedef struct { // Most of the complexity is due to the "diagonal rule", requiring us to // identify which type vars range over only concrete types. typedef struct jl_varbinding_t { - jl_tvar_t *var; - jl_value_t *lb; - jl_value_t *ub; + jl_tvar_t *var; // store NULL to "delete" this from env (temporarily) + jl_value_t *JL_NONNULL lb; + jl_value_t *JL_NONNULL ub; int8_t right; // whether this variable came from the right side of `A <: B` int8_t occurs_inv; // occurs in invariant position int8_t occurs_cov; // # of occurrences in covariant position @@ -356,7 +356,7 @@ static void free_stenv(jl_stenv_t *e) JL_NOTSAFEPOINT static void restore_env(jl_stenv_t *e, jl_savedenv_t *se, int root) JL_NOTSAFEPOINT { - jl_value_t **roots = NULL; + jl_value_t *JL_NONNULL *roots = NULL; int nroots = 0; if (root) { if (se->gcframe.nroots == JL_GC_ENCODE_PUSHARGS(1)) { @@ -1182,12 +1182,14 @@ static int subtype_tuple_varargs( if (bxp1) { if (bxp1->intvalued == 0) bxp1->intvalued = 1; + assert(bxp1->lb); // make static analyzer happy if (jl_is_long(bxp1->lb)) xp1 = bxp1->lb; } if (byp1) { if (byp1->intvalued == 0) byp1->intvalued = 1; + assert(byp1->lb); // make static analyzer happy if (jl_is_long(byp1->lb)) yp1 = byp1->lb; } diff --git a/src/typemap.c b/src/typemap.c index 8c0e585601944..5b25389fe1cab 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -286,20 +286,20 @@ static _Atomic(jl_value_t*) *mtcache_hash_lookup_bp(jl_genericmemory_t *cache JL return pml; } -static void mtcache_hash_insert(_Atomic(jl_genericmemory_t*) *cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) +static void mtcache_hash_insert(_Atomic(jl_genericmemory_t*) *pcache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) { int inserted = 0; - jl_genericmemory_t *a = jl_atomic_load_relaxed(cache); + jl_genericmemory_t *a = jl_atomic_load_relaxed(pcache); if (a == (jl_genericmemory_t*)jl_an_empty_memory_any) { a = jl_alloc_memory_any(16); - jl_atomic_store_release(cache, a); + jl_atomic_store_release(pcache, a); if (parent) jl_gc_wb(parent, a); } a = jl_eqtable_put(a, key, val, &inserted); assert(inserted); - if (a != jl_atomic_load_relaxed(cache)) { - jl_atomic_store_release(cache, a); + if (a != jl_atomic_load_relaxed(pcache)) { + jl_atomic_store_release(pcache, a); if (parent) jl_gc_wb(parent, a); } @@ -1293,9 +1293,10 @@ static void jl_typemap_memory_insert_( static jl_value_t *jl_method_convert_list_to_cache( jl_typemap_t *map, jl_typemap_entry_t *ml, int8_t tparam, int8_t offs, int8_t doublesplit) { - jl_value_t *cache = doublesplit ? jl_an_empty_memory_any : (jl_value_t*)jl_new_typemap_level(); + _Atomic(jl_genericmemory_t*) dblcache = (jl_genericmemory_t*)jl_an_empty_memory_any; + jl_typemap_level_t *cache = doublesplit ? NULL : jl_new_typemap_level(); jl_typemap_entry_t *next = NULL; - JL_GC_PUSH3(&cache, &next, &ml); + JL_GC_PUSH4(&cache, &dblcache, &next, &ml); while (ml != (void*)jl_nothing) { next = jl_atomic_load_relaxed(&ml->next); jl_atomic_store_relaxed(&ml->next, (jl_typemap_entry_t*)jl_nothing); @@ -1316,14 +1317,14 @@ static jl_value_t *jl_method_convert_list_to_cache( assert(jl_is_type_type(key)); key = jl_tparam0(key); } - jl_typemap_memory_insert_(map, (_Atomic(jl_genericmemory_t*)*)&cache, key, ml, NULL, 0, offs, NULL); + jl_typemap_memory_insert_(map, &dblcache, key, ml, NULL, 0, offs, NULL); } else - jl_typemap_level_insert_(map, (jl_typemap_level_t*)cache, ml, offs); + jl_typemap_level_insert_(map, cache, ml, offs); ml = next; } JL_GC_POP(); - return cache; + return doublesplit ? (jl_value_t*)jl_atomic_load_relaxed(&dblcache) : (jl_value_t*)cache; } static void jl_typemap_list_insert_( diff --git a/stdlib/LLD_jll/Project.toml b/stdlib/LLD_jll/Project.toml index c7041ac4a2577..f1ebf691ff3a9 100644 --- a/stdlib/LLD_jll/Project.toml +++ b/stdlib/LLD_jll/Project.toml @@ -1,6 +1,6 @@ name = "LLD_jll" uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" -version = "19.1.7+1" +version = "20.1.2+0" [deps] Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" @@ -10,7 +10,7 @@ Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] julia = "1.13" -libLLVM_jll = "19.1.7" +libLLVM_jll = "20.1.2" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/libLLVM_jll/Project.toml b/stdlib/libLLVM_jll/Project.toml index 87eb4263ecac9..512f8cc4e4dd9 100644 --- a/stdlib/libLLVM_jll/Project.toml +++ b/stdlib/libLLVM_jll/Project.toml @@ -1,6 +1,6 @@ name = "libLLVM_jll" uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "19.1.7+2" +version = "20.1.2+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/clangsa/GCPushPop.cpp b/test/clangsa/GCPushPop.cpp index 6736d3e181118..79cad28f4b9a5 100644 --- a/test/clangsa/GCPushPop.cpp +++ b/test/clangsa/GCPushPop.cpp @@ -8,15 +8,15 @@ void missingPop() { jl_value_t *x = NULL; JL_GC_PUSH1(&x); // expected-note{{GC frame changed here}} -} // expected-warning{{Non-popped GC frame present at end of function}} - // expected-note@-1{{Non-popped GC frame present at end of function}} +} // expected-warning@-1{{Non-popped GC frame present at end of function}} + // expected-note@-2{{Non-popped GC frame present at end of function}} void missingPop2() { jl_value_t **x; JL_GC_PUSHARGS(x, 2); // expected-note{{GC frame changed here}} -} // expected-warning{{Non-popped GC frame present at end of function}} - // expected-note@-1{{Non-popped GC frame present at end of function}} +} // expected-warning@-1{{Non-popped GC frame present at end of function}} + // expected-note@-2{{Non-popped GC frame present at end of function}} void superfluousPop() { JL_GC_POP(); // expected-warning{{JL_GC_POP without corresponding push}} From 58daba4765a46ef13a7f1b04d05e2e277aaf60ae Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 May 2025 10:06:32 -0400 Subject: [PATCH 266/662] fix isconst definition/accessor issues with binding partitions (#58261) --- src/codegen.cpp | 4 ++-- src/gf.c | 2 +- src/julia.h | 6 +++--- src/julia_internal.h | 4 ++++ src/method.c | 2 +- src/module.c | 36 ++++++++++++++++++------------------ src/rtutils.c | 4 ++-- src/toplevel.c | 2 +- test/syntax.jl | 2 +- 9 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index e5600115b3a7d..2a937fc626c0b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3198,7 +3198,7 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * if (!jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { return emit_globalref_runtime(ctx, bnd, mod, name); } - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { + if (jl_bkind_is_real_constant(rkp.kind) || rkp.kind == PARTITION_KIND_UNDEF_CONST) { if (rkp.maybe_depwarn) { Value *bp = julia_binding_gv(ctx, bnd); ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); @@ -3788,7 +3788,7 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ jl_binding_t *bnd = allow_import ? jl_get_binding(modu, name) : jl_get_module_binding(modu, name, 0); struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (allow_import && jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { - if (jl_bkind_is_some_constant(rkp.kind) && rkp.restriction) + if (jl_bkind_is_real_constant(rkp.kind)) return mark_julia_const(ctx, jl_true); if (rkp.kind == PARTITION_KIND_GLOBAL) { Value *bp = julia_binding_gv(ctx, rkp.binding_if_global); diff --git a/src/gf.c b/src/gf.c index 483a18d13682c..436b4c84347dc 100644 --- a/src/gf.c +++ b/src/gf.c @@ -771,7 +771,7 @@ int foreach_mtable_in_module( if ((void*)b == jl_nothing) break; jl_sym_t *name = b->globalref->name; - jl_value_t *v = jl_get_binding_value_if_const(b); + jl_value_t *v = jl_get_latest_binding_value_if_const(b); if (v) { jl_value_t *uw = jl_unwrap_unionall(v); if (jl_is_datatype(uw)) { diff --git a/src/julia.h b/src/julia.h index 143ad9b556469..4cb4f18d30c8e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1933,9 +1933,9 @@ JL_DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, size_t len); JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void); JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b JL_PROPAGATES_ROOT); JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world); -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_latest_resolved_and_const_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_and_const_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name); JL_DLLEXPORT jl_method_t *jl_method_def(jl_svec_t *argdata, jl_methtable_t *mt, jl_code_info_t *f, jl_module_t *module); JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo, size_t world, jl_code_instance_t **cache); diff --git a/src/julia_internal.h b/src/julia_internal.h index 9436e19fb26b4..938e87935578c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -987,6 +987,10 @@ STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_N return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST; } +STATIC_INLINE int jl_bkind_is_real_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT; +} + JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b JL_PROPAGATES_ROOT, jl_binding_partition_t *previous_part, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b JL_PROPAGATES_ROOT, size_t min_world, size_t max_world) JL_GLOBALLY_ROOTED; diff --git a/src/method.c b/src/method.c index 675293089de44..f71f09ceb83bc 100644 --- a/src/method.c +++ b/src/method.c @@ -224,7 +224,7 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_sym_t *fe_sym = jl_globalref_name(fe); // look at some known called functions jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); - if (jl_get_binding_value_if_const(b) == BUILTIN(tuple)) { + if (jl_get_latest_binding_value_if_const(b) == BUILTIN(tuple)) { size_t j; for (j = 1; j < nargs; j++) { if (!jl_is_quotenode(jl_exprarg(e, j))) diff --git a/src/module.c b/src/module.c index 0271a645aa4ed..272748edbadb2 100644 --- a/src/module.c +++ b/src/module.c @@ -463,7 +463,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_bindin struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (!jl_get_binding_leaf_partitions_restriction_kind(b, &rkp, min_world, max_world)) return NULL; - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { + if (jl_bkind_is_real_constant(rkp.kind)) { *maybe_depwarn = rkp.maybe_depwarn; return rkp.restriction; } @@ -581,7 +581,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( for (;;) { enum jl_partition_kind prev_kind = jl_binding_kind(prev_bpart); if (jl_bkind_is_some_constant(prev_kind) || prev_kind == PARTITION_KIND_GLOBAL || - (jl_bkind_is_some_import(prev_kind))) { + jl_bkind_is_some_import(prev_kind)) { need_backdate = 0; break; } @@ -923,22 +923,23 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) return jl_atomic_load(&b->value); } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_const(jl_binding_t *b) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + // See note below. Note that this is for some deprecated uses, and should not be added to new code. + size_t world = jl_atomic_load_relaxed(&jl_world_counter); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; - if (!jl_bkind_is_some_constant(kind)) + if (!jl_bkind_is_real_constant(kind)) return NULL; - check_backdated_binding(b, kind); return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_latest_resolved_and_const_debug_only(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_and_const_debug_only(jl_binding_t *b) { - // Unlike jl_get_binding_value_if_const this doesn't try to allocate new binding partitions if they + // Unlike jl_get_latest_binding_value_if_const this doesn't try to allocate new binding partitions if they // don't already exist, making this JL_NOTSAFEPOINT. However, as a result, this may fail to return // a value - even if one does exist. It should only be used for reflection/debugging when the integrity // of the runtime is not guaranteed. @@ -948,18 +949,17 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_latest_resolved_and_const_debug if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (jl_atomic_load_relaxed(&bpart->min_world) > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (max_world != ~(size_t)0) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; - if (!jl_bkind_is_some_constant(kind)) + if (!jl_bkind_is_real_constant(kind)) return NULL; - check_backdated_binding(b, kind); return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_debug_only(jl_binding_t *b) { // See note above. Use for debug/reflection purposes only. if (!b) @@ -968,7 +968,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_ if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (jl_atomic_load_relaxed(&bpart->min_world) > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (max_world != ~(size_t)0) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -976,7 +976,6 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_debug_only(jl_binding_ if (jl_bkind_is_some_import(kind)) return NULL; if (jl_bkind_is_some_constant(kind)) { - check_backdated_binding(b, kind); return bpart->restriction; } return jl_atomic_load_relaxed(&b->value); @@ -1011,6 +1010,7 @@ static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b) // along the way. JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b, size_t new_world) { + assert(new_world > jl_atomic_load_relaxed(&jl_world_counter)); jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_constant(kind) && kind != PARTITION_KIND_IMPLICIT_CONST) @@ -1032,7 +1032,7 @@ JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b, size_t new_w check_safe_newbinding(b->globalref->mod, b->globalref->name); return NULL; } - jl_module_t *from = jl_binding_dbgmodule(b);\ + jl_module_t *from = jl_binding_dbgmodule(b); assert(from); // Can only be NULL if implicit, which we excluded above jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", jl_module_debug_name(b->globalref->mod), jl_module_debug_name(from), jl_symbol_name(b->globalref->name)); @@ -1728,7 +1728,7 @@ JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr) b = jl_get_module_binding(gr->mod, gr->name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); - return jl_bkind_is_some_constant(jl_binding_kind(bpart)); + return jl_bkind_is_real_constant(jl_binding_kind(bpart)); } JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) @@ -1757,7 +1757,7 @@ JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) jl_binding_t *b = jl_get_binding(m, var); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); - return b && jl_bkind_is_some_constant(jl_binding_kind(bpart)); + return b && jl_bkind_is_real_constant(jl_binding_kind(bpart)); } // set the deprecated flag for a binding: diff --git a/src/rtutils.c b/src/rtutils.c index 5966497ec331c..4baf0ee5e6e9c 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -579,7 +579,7 @@ JL_DLLEXPORT jl_value_t *jl_stderr_obj(void) JL_NOTSAFEPOINT if (jl_base_module == NULL) return NULL; jl_binding_t *stderr_obj = jl_get_module_binding(jl_base_module, jl_symbol("stderr"), 0); - return stderr_obj ? jl_get_binding_value_if_resolved_debug_only(stderr_obj) : NULL; + return stderr_obj ? jl_get_latest_binding_value_if_resolved_debug_only(stderr_obj) : NULL; } // toys for debugging --------------------------------------------------------- @@ -674,7 +674,7 @@ static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; if (globname && dv->name->module) { jl_binding_t *b = jl_get_module_binding(dv->name->module, globname, 0); - jl_value_t *bv = jl_get_binding_value_if_latest_resolved_and_const_debug_only(b); + jl_value_t *bv = jl_get_latest_binding_value_if_resolved_and_const_debug_only(b); if (bv && ((jl_value_t*)dv == v ? jl_typeof(bv) == v : bv == v)) return 1; } diff --git a/src/toplevel.c b/src/toplevel.c index 826b9195da059..739eb8024ece0 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -416,7 +416,7 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int jl_module_t *mod = jl_globalref_mod(f); jl_sym_t *name = jl_globalref_name(f); jl_binding_t *b = jl_get_binding(mod, name); - called = jl_get_binding_value_if_const(b); + called = jl_get_latest_binding_value_if_const(b); } else if (jl_is_quotenode(f)) { called = jl_quotenode_value(f); diff --git a/test/syntax.jl b/test/syntax.jl index 27abea4c57a66..2321d8cf9266f 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1939,7 +1939,7 @@ end # eval'ing :const exprs eval(Expr(:const, :_var_30877)) @test !isdefined(@__MODULE__, :_var_30877) -@test isconst(@__MODULE__, :_var_30877) +@test !isconst(@__MODULE__, :_var_30877) # anonymous kw function in value position at top level f30926 = function (;k=0) From 229a6984ee142283d81955d8d53d7985fd5736ca Mon Sep 17 00:00:00 2001 From: Laine Taffin Altman Date: Wed, 14 May 2025 09:53:37 -0700 Subject: [PATCH 267/662] Add JuliaLang/JuliaSyntax.jl#525 to NEWS.md, flisp parser, and REPL (#57143) Now that JuliaLang/JuliaSyntax.jl#525 has been merged, also add it to the flisp parser and REPL, and document it! --- NEWS.md | 3 +++ src/flisp/julia_extensions.c | 7 ++++++- src/julia-parser.scm | 2 +- stdlib/REPL/src/latex_symbols.jl | 1 + test/syntax.jl | 5 +++++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index d2eb96214beec..fcf94b340ad66 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,9 @@ New language features --------------------- - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) + - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, + is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. + ([JuliaLang/JuliaSyntax.jl#525], [#57143]) Language changes ---------------- diff --git a/src/flisp/julia_extensions.c b/src/flisp/julia_extensions.c index 07d074e1fb80b..c39c2edfe0f37 100644 --- a/src/flisp/julia_extensions.c +++ b/src/flisp/julia_extensions.c @@ -130,6 +130,9 @@ JL_DLLEXPORT int jl_id_start_char(uint32_t wc) return 1; if (wc < 0xA1 || wc > 0x10ffff) return 0; + // "Rightwards Arrow with Lower Hook" + if (wc == 0x1f8b2) + return 1; return is_wc_cat_id_start(wc, utf8proc_category((utf8proc_int32_t) wc)); } @@ -147,7 +150,9 @@ JL_DLLEXPORT int jl_id_char(uint32_t wc) cat == UTF8PROC_CATEGORY_SK || cat == UTF8PROC_CATEGORY_ME || cat == UTF8PROC_CATEGORY_NO || // primes (single, double, triple, their reverses, and quadruple) - (wc >= 0x2032 && wc <= 0x2037) || (wc == 0x2057)) + (wc >= 0x2032 && wc <= 0x2037) || (wc == 0x2057) || + // "Rightwards Arrow with Lower Hook" + wc == 0x1f8b2) return 1; return 0; } diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 4415dc8686065..1a11494b5c8e3 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -10,7 +10,7 @@ ;; comma - higher than assignment outside parentheses, lower when inside (define prec-pair (add-dots '(=>))) (define prec-conditional '(?)) -(define prec-arrow (add-dots '(← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⥷ ⭄ ⥺ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ← → ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ --> <-- <-->))) +(define prec-arrow (add-dots '(← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⥷ ⭄ ⥺ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ← → ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ --> <-- <--> 🢲))) (define prec-lazy-or (add-dots '(|\|\||))) (define prec-lazy-and (add-dots '(&&))) (define prec-comparison diff --git a/stdlib/REPL/src/latex_symbols.jl b/stdlib/REPL/src/latex_symbols.jl index 9f5b7e3e864ed..b739accc72499 100644 --- a/stdlib/REPL/src/latex_symbols.jl +++ b/stdlib/REPL/src/latex_symbols.jl @@ -517,6 +517,7 @@ const latex_symbols = Dict( "\\mapsto" => "↦", "\\hookleftarrow" => "↩", "\\hookrightarrow" => "↪", + "\\hookunderrightarrow" => "🢲", "\\looparrowleft" => "↫", "\\looparrowright" => "↬", "\\leftrightsquigarrow" => "↭", diff --git a/test/syntax.jl b/test/syntax.jl index 2321d8cf9266f..4116bdef23a52 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2287,6 +2287,11 @@ end @test Meta.parse("a ⥷ b") == Expr(:call, :⥷, :a, :b) end +# issue 57143 +@testset "binary 🢲" begin + @test Meta.parse("a 🢲 b") == Expr(:call, :🢲, :a, :b) +end + # only allow certain characters after interpolated vars (#25231) @test_parseerror("\"\$x෴ \"", "interpolated variable \$x ends with invalid character \"෴\"; use \"\$(x)\" instead.") From 7df60f480df8c6aed874b35eb7c8a26fc769a4cc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 14 May 2025 17:36:33 -0400 Subject: [PATCH 268/662] fix `hasmethod` with kwargs to exclude positional arg names (#58410) --- base/reflection.jl | 2 +- test/reflection.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/reflection.jl b/base/reflection.jl index 3b0d522bfd0c2..2e976add50190 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1035,11 +1035,11 @@ function hasmethod(f, t, kwnames::Tuple{Vararg{Symbol}}; world::UInt=get_world_c match = ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) match === nothing && return false kws = ccall(:jl_uncompress_argnames, Array{Symbol,1}, (Any,), (match::Method).slot_syms) + kws = kws[((match::Method).nargs + 1):end] # remove positional arguments isempty(kws) && return true # some kwfuncs simply forward everything directly for kw in kws endswith(String(kw), "...") && return true end - kwnames = collect(kwnames) return issubset(kwnames, kws) end diff --git a/test/reflection.jl b/test/reflection.jl index 25b61945aefac..6da64ae7d7031 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -936,6 +936,7 @@ f(x::Int; y=3) = x + y @test hasmethod(f, Tuple{Int}) @test hasmethod(f, Tuple{Int}, ()) @test hasmethod(f, Tuple{Int}, (:y,)) +@test !hasmethod(f, Tuple{Int}, (:x,)) @test !hasmethod(f, Tuple{Int}, (:jeff,)) @test !hasmethod(f, Tuple{Int}, (:y,), world=typemin(UInt)) g(; b, c, a) = a + b + c From a87b05665fa58dd0360b2a69e4468404a552cd00 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 May 2025 22:54:15 -0400 Subject: [PATCH 269/662] [REPL] fix type confusion resulting in nonsensical errors (#58414) --- stdlib/REPL/src/REPL.jl | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 81272ac971d40..169b6a71858ef 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -452,8 +452,8 @@ function repl_backend_loop(backend::REPLBackend, get_module::Function) try ret = f() put!(backend.response_channel, Pair{Any, Bool}(ret, false)) - catch err - put!(backend.response_channel, Pair{Any, Bool}(err, true)) + catch + put!(backend.response_channel, Pair{Any, Bool}(current_exceptions(), true)) end else ast = ast_or_func @@ -594,11 +594,11 @@ function print_response(errio::IO, response, backend::Union{REPLBackendRef,Nothi if val !== nothing && show_value val2, iserr = if specialdisplay === nothing # display calls may require being run on the main thread - eval_with_backend(backend) do + call_on_backend(backend) do Base.invokelatest(display, val) end else - eval_with_backend(backend) do + call_on_backend(backend) do Base.invokelatest(display, specialdisplay, val) end end @@ -715,7 +715,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef) (isa(ast,Expr) && ast.head === :incomplete) || break end if !isempty(line) - response = eval_with_backend(ast, backend) + response = eval_on_backend(ast, backend) print_response(repl, response, !ends_with_semicolon(line), false) end write(repl.terminal, '\n') @@ -1166,21 +1166,23 @@ find_hist_file() = get(ENV, "JULIA_HISTORY", backend(r::AbstractREPL) = hasproperty(r, :backendref) ? r.backendref : nothing -function eval_with_backend(ast::Expr, backend::REPLBackendRef) +function eval_on_backend(ast, backend::REPLBackendRef) put!(backend.repl_channel, (ast, 1)) # (f, show_value) return take!(backend.response_channel) # (val, iserr) end -function eval_with_backend(f, backend::REPLBackendRef) +function call_on_backend(f, backend::REPLBackendRef) + applicable(f) || error("internal error: f is not callable") put!(backend.repl_channel, (f, 2)) # (f, show_value) 2 indicates function (rather than ast) return take!(backend.response_channel) # (val, iserr) end # if no backend just eval (used by tests) -function eval_with_backend(f, backend::Nothing) +eval_on_backend(ast, backend::Nothing) = error("no backend for eval ast") +function call_on_backend(f, backend::Nothing) try ret = f() return (ret, false) # (val, iserr) - catch err - return (err, true) + catch + return (current_exceptions(), true) end end @@ -1196,7 +1198,7 @@ function respond(f, repl, main; pass_empty::Bool = false, suppress_on_semicolon: local response try ast = Base.invokelatest(f, line) - response = eval_with_backend(ast, backend(repl)) + response = eval_on_backend(ast, backend(repl)) catch response = Pair{Any, Bool}(current_exceptions(), true) end @@ -1803,7 +1805,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef) if have_color print(repl.stream, Base.color_normal) end - response = eval_with_backend(ast, backend) + response = eval_on_backend(ast, backend) print_response(repl, response, !ends_with_semicolon(line), have_color) end end From 18f7fe9618d435c3cb35744e9a61dd5491f8bb53 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 15 May 2025 16:55:44 +0900 Subject: [PATCH 270/662] change the Compiler.jl stdlib version to 0.1.0 (#58420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In JuliaRegistries/General#130304 I proposed naming the placeholder version of the Compiler stdlib "v0.0.0" and made changes accordingly. However, after further discussions around adjusting Pkg.jl (c.f. JuliaLang/PKg.jl#4233), we decided to call it "v0.1.0" instead since the idea may sound to be an abuse of semver and that versioning wouldn't be able to handle cases when any changes to the implementation of that special version are needed in the future. As a result, this commit changes the version of the Compiler.jl stdlib implementation maintained in the base from v0.0.0 to v0.1.0. Since BaseCompiler.jl is a very internal, special package that doesn’t yet follow proper versioning, there’s no need to worry about ecosystem impact from this change. --- Compiler/Project.toml | 2 +- Compiler/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Compiler/Project.toml b/Compiler/Project.toml index 7113a60318984..f44b45a810708 100644 --- a/Compiler/Project.toml +++ b/Compiler/Project.toml @@ -1,6 +1,6 @@ name = "Compiler" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.0.0" +version = "0.1.0" [compat] julia = "1.10" diff --git a/Compiler/README.md b/Compiler/README.md index aa1f6a0c92827..5e58152519678 100644 --- a/Compiler/README.md +++ b/Compiler/README.md @@ -16,13 +16,13 @@ your `Project.toml` as follows: Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" [compat] -Compiler = "0" +Compiler = "0.1" ``` -With the setup above, [the special placeholder version (v0.0.0)](https://github.com/JuliaLang/BaseCompiler.jl) +With the setup above, [the special placeholder version (v0.1.0)](https://github.com/JuliaLang/BaseCompiler.jl) will be installed by default.[^1] -[^1]: Currently, only version v0.0.0 is registered in the [General](https://github.com/JuliaRegistries/General) registry. +[^1]: Currently, only version v0.1.0 is registered in the [General](https://github.com/JuliaRegistries/General) registry. If needed, you can switch to a custom implementation of the `Compiler` module by running ```julia-repl From d54f9b67ea7ae75eded8fc175464c6edcba3d961 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Fri, 16 May 2025 01:13:28 +1200 Subject: [PATCH 271/662] Bump Documenter to v1.11.3 (#58421) --- doc/Manifest.toml | 4 ++-- doc/make.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 879476c7d0bd9..cb2508030ece4 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -44,9 +44,9 @@ version = "0.9.4" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "7745f07eaf6454c15caa21c5ecaebef3afad32eb" +git-tree-sha1 = "6f8730fd1bdf974009ef296bd81afb2728854fc0" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.11.1" +version = "1.11.3" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] diff --git a/doc/make.jl b/doc/make.jl index f149e4b37a9e9..ac158afffcaa6 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -450,6 +450,7 @@ const devurl = "v$(VERSION.major).$(VERSION.minor)-dev" # Hack to make rc docs visible in the version selector struct Versions versions end +Documenter.determine_deploy_subfolder(deploy_decision, ::Versions) = deploy_decision.subfolder function Documenter.Writers.HTMLWriter.expand_versions(dir::String, v::Versions) # Find all available docs available_folders = readdir(dir) From 94570e1bac838faa6e5a900769b4562dd047a57c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 15 May 2025 10:14:43 -0400 Subject: [PATCH 272/662] [REPL] more reliable extension loading (#58415) A weird edge case of loading, if REPL is loaded explicitly and does not come from a require_stdlib call, it should not be getting REPLExt from a require_stdlib call either. This is a convoluted bit of hackery specific to work around problems with the way Pkg's REPLExt is designed to mutate the REPL in unsafe ways. No other stdlib should ever want to access an extension, particularly of a different module, as that is a private API violation on multiple counts, so this only needs to be made to work specifically for that REPL-Pkg scenario, even if it looks seemingly more general. Refs https://github.com/JuliaLang/julia/issues/58373 Reproducer: ``` JULIA_DEPOT_PATH=tmpdir/.julia ./julia --hist=no -qie 'using REPL' ] status ``` --- base/loading.jl | 39 ++++++++++++++++++++++--------- stdlib/REPL/src/Pkg_beforeload.jl | 4 +++- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 48aed8461f01b..d8b7d2156cf30 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2715,7 +2715,7 @@ end # load a serialized file directly from append_bundled_depot_path for uuidkey without stalechecks """ - require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=nothing) + require_stdlib(package_uuidkey::PkgId, [ext::String, from::Module]) !!! warning "May load duplicate copies of stdlib packages." @@ -2754,7 +2754,8 @@ end [1] https://github.com/JuliaLang/Pkg.jl/issues/4017#issuecomment-2377589989 [2] https://github.com/JuliaLang/StyledStrings.jl/issues/91#issuecomment-2379602914 """ -function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=nothing) +require_stdlib(package_uuidkey::PkgId) = require_stdlib(package_uuidkey, nothing, Base) +function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, from::Module) if generating_output(#=incremental=#true) # Otherwise this would lead to awkward dependency issues by loading a package that isn't in the Project/Manifest error("This interactive function requires a stdlib to be loaded, and package code should instead use it directly from that stdlib.") @@ -2766,15 +2767,29 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=noth newm = start_loading(this_uuidkey, UInt128(0), true) newm === nothing || return newm try - # first since this is a stdlib, try to look there directly first - if ext === nothing - sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl") - else - sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext) - end depot_path = append_bundled_depot_path!(empty(DEPOT_PATH)) - set_pkgorigin_version_path(this_uuidkey, sourcepath) - newm = _require_search_from_serialized(this_uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + from_stdlib = true # set to false if `from` is a normal package so we do not want the internal loader for the extension either + if ext isa String + from_uuid = PkgId(from) + from_m = get(loaded_modules, from_uuid, nothing) + if from_m === from + # if from_uuid is either nothing or points to something else, assume we should use require_stdlib + # otherwise check cachepath for from to see if it looks like it is from depot_path, since try_build_ids + cachepath = get(PkgOrigin, pkgorigins, from_uuid).cachepath + entrypath, entryfile = cache_file_entry(from_uuid) + from_stdlib = any(x -> startswith(entrypath, x), depot_path) + end + end + if from_stdlib + # first since this is a stdlib, try to look there directly first + if ext === nothing + sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl") + else + sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext) + end + set_pkgorigin_version_path(this_uuidkey, sourcepath) + newm = _require_search_from_serialized(this_uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + end finally end_loading(this_uuidkey, newm) end @@ -2784,10 +2799,12 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=noth run_package_callbacks(this_uuidkey) else # if the user deleted their bundled depot, next try to load it completely normally + # if it is an extension, we first need to indicate where to find its parant via EXT_PRIMED + ext isa String && (EXT_PRIMED[this_uuidkey] = PkgId[package_uuidkey]) newm = _require_prelocked(this_uuidkey) end return newm - end + end # release lock end # relative-path load diff --git a/stdlib/REPL/src/Pkg_beforeload.jl b/stdlib/REPL/src/Pkg_beforeload.jl index 86b5cd35abd2f..e51cf7550bce5 100644 --- a/stdlib/REPL/src/Pkg_beforeload.jl +++ b/stdlib/REPL/src/Pkg_beforeload.jl @@ -1,7 +1,9 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + ## Pkg stuff needed before Pkg has loaded const Pkg_pkgid = Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg") -load_pkg() = Base.require_stdlib(Pkg_pkgid, "REPLExt") +load_pkg() = Base.require_stdlib(Pkg_pkgid, "REPLExt", REPL) ## Below here copied/tweaked from Pkg Types.jl so that the dummy Pkg prompt # can populate the env correctly before Pkg loads From 34070fa3be828c282d74d0e67ee907fd3e2f2a07 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 15 May 2025 10:42:51 -0400 Subject: [PATCH 273/662] add a warning to the docstring for `deepcopy` (#58416) This function is surprisingly popular and we often need to warn people against using it. Somehow such a warning never made it into the docs, so here it is. --- base/deepcopy.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base/deepcopy.jl b/base/deepcopy.jl index f60ce2043dd5a..58c753705a61f 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -23,6 +23,11 @@ where `T` is the type to be specialized for, and `dict` keeps track of objects c so far within the recursion. Within the definition, `deepcopy_internal` should be used in place of `deepcopy`, and the `dict` variable should be updated as appropriate before returning. + +!!! warning + It is better to avoid this function in favor of custom `copy` methods or use-case-specific + copying functions. `deepcopy` is slow and can easily copy too many objects, or generate an + object that violates invariants, since it does not respect abstraction boundaries. """ function deepcopy(@nospecialize x) isbitstype(typeof(x)) && return x From 52e14ccc01dc4cfeba43e2c56b2fec51e94752f1 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Thu, 15 May 2025 17:09:24 -0700 Subject: [PATCH 274/662] Don't create a type parameter in the closure for captured @nospecialize arguments (#58426) When we capture a variable without boxing, we always generate a type parameter for the closure. This probably isn't what the user wants if the captured variable is an argument marked `@nospecialize`. Before: ``` julia> K(@nospecialize(x)) = @nospecialize(y) -> x K (generic function with 1 method) julia> f, g = K(1), K("a") (var"#K##0#K##1"{Int64}(1), var"#K##0#K##1"{String}("a")) julia> f(2), g(2) (1, "a") julia> methods(f)[1].specializations svec(MethodInstance for (::var"#K##0#K##1"{Int64})(::Any), MethodInstance for (::var"#K##0#K##1"{String})(::Any), nothing, nothing, nothing, nothing, nothing) julia> fieldtypes(typeof(f)), fieldtypes(typeof(g)) ((Int64,), (String,)) ``` After: ``` julia> K(@nospecialize(x)) = @nospecialize(y) -> x K (generic function with 1 method) julia> f, g = K(1), K("a") (var"#K##0#K##1"(1), var"#K##0#K##1"("a")) julia> f(2), g(2) (1, "a") julia> methods(f)[1].specializations MethodInstance for (::var"#K##0#K##1")(::Any) julia> fieldtypes(typeof(f)), fieldtypes(typeof(g)) ((Any,), (Any,)) ``` --- src/ast.scm | 2 ++ src/julia-syntax.scm | 44 ++++++++++++++++++-------------------------- test/syntax.jl | 9 +++++++++ 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/ast.scm b/src/ast.scm index a20f8f87f955c..15e55fc616041 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -493,6 +493,7 @@ (define (vinfo:never-undef v) (< 0 (logand (caddr v) 4))) (define (vinfo:read v) (< 0 (logand (caddr v) 8))) (define (vinfo:sa v) (< 0 (logand (caddr v) 16))) +(define (vinfo:nospecialize v) (< 0 (logand (caddr v) 128))) (define (set-bit x b val) (if val (logior x b) (logand x (lognot b)))) ;; record whether var is captured (define (vinfo:set-capt! v c) (set-car! (cddr v) (set-bit (caddr v) 1 c))) @@ -507,6 +508,7 @@ ;; occurs undef: mask 32 ;; whether var is called (occurs in function call head position) (define (vinfo:set-called! v a) (set-car! (cddr v) (set-bit (caddr v) 64 a))) +(define (vinfo:set-nospecialize! v c) (set-car! (cddr v) (set-bit (caddr v) 128 c))) (define var-info-for assq) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 8f0bcd55ac194..cf2c964cf2374 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3492,10 +3492,15 @@ (define (analyze-vars e env captvars sp tab) (if (or (atom? e) (quoted? e)) (begin - (if (symbol? e) - (let ((vi (get tab e #f))) - (if vi - (vinfo:set-read! vi #t)))) + (cond + ((symbol? e) + (let ((vi (get tab e #f))) + (if vi + (vinfo:set-read! vi #t)))) + ((nospecialize-meta? e) + (let ((vi (get tab (caddr e) #f))) + (if vi + (vinfo:set-nospecialize! vi #t))))) e) (case (car e) ((local-def) ;; a local that we know has an assignment that dominates all usages @@ -3594,21 +3599,6 @@ f(x) = yt(x) (call (core _typebody!) (false) ,s (call (core svec) ,@types)) (return (null))))))))) -(define (type-for-closure name fields super) - (let ((s (make-ssavalue))) - `((thunk ,(linearize `(lambda () - (() () 0 ()) - (block (global ,name) - (= ,s (call (core _structtype) (thismodule) (inert ,name) (call (core svec)) - (call (core svec) ,@(map quotify fields)) - (call (core svec)) - (false) ,(length fields))) - (call (core _setsuper!) ,s ,super) - (const (globalref (thismodule) ,name) ,s) - (call (core _typebody!) (false) ,s - (call (core svec) ,@(map (lambda (v) '(core Box)) fields))) - (return (null))))))))) - ;; better versions of above, but they get handled wrong in many places ;; need to fix that in order to handle #265 fully (and use the definitions) @@ -4022,6 +4012,10 @@ f(x) = yt(x) (let ((cv (assq v (cadr (lam:vinfo lam))))) (and cv (vinfo:asgn cv) (vinfo:capt cv))))) +(define (is-var-nospecialize? v lam) + (let ((vi (assq v (car (lam:vinfo lam))))) + (and vi (vinfo:nospecialize vi)))) + (define (toplevel-preserving? e) (and (pair? e) (memq (car e) '(if elseif block trycatch tryfinally trycatchelse)))) @@ -4313,16 +4307,14 @@ f(x) = yt(x) (closure-param-syms (map (lambda (s) (make-ssavalue)) closure-param-names)) (typedef ;; expression to define the type (let* ((fieldtypes (map (lambda (v) - (if (is-var-boxed? v lam) - '(core Box) - (make-ssavalue))) + (cond ((is-var-boxed? v lam) '(core Box)) + ((is-var-nospecialize? v lam) (vinfo:type (assq v (car (lam:vinfo lam))))) + (else (make-ssavalue)))) capt-vars)) (para (append closure-param-syms (filter ssavalue? fieldtypes))) (fieldnames (append closure-param-names (filter (lambda (v) (not (is-var-boxed? v lam))) capt-vars)))) - (if (null? para) - (type-for-closure type-name capt-vars '(core Function)) - (type-for-closure-parameterized type-name para fieldnames capt-vars fieldtypes '(core Function))))) + (type-for-closure-parameterized type-name para fieldnames capt-vars fieldtypes '(core Function)))) (mk-method ;; expression to make the method (if short '() (let* ((iskw ;; TODO jb/functions need more robust version of this @@ -4352,7 +4344,7 @@ f(x) = yt(x) (P (append closure-param-names (filter identity (map (lambda (v ve) - (if (is-var-boxed? v lam) + (if (or (is-var-boxed? v lam) (is-var-nospecialize? v lam)) #f `(call (core _typeof_captured_variable) ,ve))) capt-vars var-exprs))))) diff --git a/test/syntax.jl b/test/syntax.jl index 4116bdef23a52..0c56b6b74b167 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4330,3 +4330,12 @@ let ex = @Meta.lower function return_my_method(); 1; end code[end] = Core.ReturnNode(Core.SSAValue(idx)) @test isa(Core.eval(@__MODULE__, ex), Method) end + +# Capturing a @nospecialize argument should result in an Any field in the closure +module NoSpecClosure + K(@nospecialize(x)) = y -> x +end +let f = NoSpecClosure.K(1) + @test f(2) == 1 + @test typeof(f).parameters == Core.svec() +end From 43ead4774248b1cf9cb84a349e9e90b3595c9ecd Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Thu, 15 May 2025 21:33:30 -0400 Subject: [PATCH 275/662] reflection: Label "dynamic invoke" in `code_typed` (#58411) --- Compiler/src/abstractinterpretation.jl | 4 ++-- Compiler/src/ssair/show.jl | 19 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index dae80bec55693..bff75c63681cd 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2215,7 +2215,7 @@ function abstract_call_unionall(interp::AbstractInterpreter, argtypes::Vector{An return CallMeta(ret, Any, Effects(EFFECTS_TOTAL; nothrow), call.info) end -function ci_abi(ci::CodeInstance) +function get_ci_abi(ci::CodeInstance) def = ci.def isa(def, ABIOverride) && return def.abi (def::MethodInstance).specTypes @@ -2238,7 +2238,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt if isa(method_or_ci, CodeInstance) our_world = sv.world.this argtype = argtypes_to_type(pushfirst!(argtype_tail(argtypes, 4), ft)) - specsig = ci_abi(method_or_ci) + specsig = get_ci_abi(method_or_ci) defdef = get_ci_mi(method_or_ci).def exct = method_or_ci.exctype if !hasintersect(argtype, specsig) diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index 0688c02eb6440..1b6eeca57f3f9 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -12,7 +12,8 @@ using .Compiler: ALWAYS_FALSE, ALWAYS_TRUE, argextype, BasicBlock, block_for_ins CachedMethodTable, CFG, compute_basic_blocks, DebugInfoStream, Effects, EMPTY_SPTYPES, getdebugidx, IncrementalCompact, InferenceResult, InferenceState, InvalidIRError, IRCode, LimitedAccuracy, NativeInterpreter, scan_ssa_use!, - singleton_type, sptypes_from_meth_instance, StmtRange, Timings, VarState, widenconst + singleton_type, sptypes_from_meth_instance, StmtRange, Timings, VarState, widenconst, + get_ci_mi, get_ci_abi @nospecialize @@ -95,16 +96,14 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co elseif isexpr(stmt, :invoke) && length(stmt.args) >= 2 && isa(stmt.args[1], Union{MethodInstance,CodeInstance}) stmt = stmt::Expr # TODO: why is this here, and not in Base.show_unquoted - printstyled(io, " invoke "; color = :light_black) - mi = stmt.args[1] - if !(mi isa Core.MethodInstance) - mi = (mi::Core.CodeInstance).def - end - if isa(mi, Core.ABIOverride) - abi = mi.abi - mi = mi.def + ci = stmt.args[1] + if ci isa Core.CodeInstance + printstyled(io, " invoke "; color = :light_black) + mi = get_ci_mi(ci) + abi = get_ci_abi(ci) else - abi = mi.specTypes + printstyled(io, "dynamic invoke "; color = :yellow) + abi = (ci::Core.MethodInstance).specTypes end show_unquoted(io, stmt.args[2], indent) print(io, "(") From 7ca634adc0d9bc674b05663a7e24250f3bd5fc33 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Fri, 16 May 2025 06:07:40 -0700 Subject: [PATCH 276/662] Fix MethodError in IR validator (#58425) --- Compiler/src/validation.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Compiler/src/validation.jl b/Compiler/src/validation.jl index d5faf51a89356..067d9460b52ec 100644 --- a/Compiler/src/validation.jl +++ b/Compiler/src/validation.jl @@ -35,8 +35,6 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :aliasscope => 0:0, :popaliasscope => 0:0, :new_opaque_closure => 5:typemax(Int), - :import => 1:typemax(Int), - :using => 1:typemax(Int), :export => 1:typemax(Int), :public => 1:typemax(Int), :latestworld => 0:0, @@ -72,11 +70,13 @@ function maybe_validate_code(mi::MethodInstance, src::CodeInfo, kind::String) if !isempty(errors) for e in errors if mi.def isa Method - println(stderr, "WARNING: Encountered invalid ", kind, " code for method ", - mi.def, ": ", e) + println(Core.stderr, + "WARNING: Encountered invalid ", kind, + " code for method ", mi.def, ": ", e) else - println(stderr, "WARNING: Encountered invalid ", kind, " code for top level expression in ", - mi.def, ": ", e) + println(Core.stderr, + "WARNING: Encountered invalid ", kind, + " code for top level expression in ", mi.def, ": ", e) end end error("") From eebda6c040f7285d2ee1eb69be8f5a7a203b43dc Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Sat, 17 May 2025 01:09:32 +1200 Subject: [PATCH 277/662] Bump Documenter to v1.11.4 (#58431) --- doc/Makefile | 4 ++++ doc/Manifest.toml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 4469a40f74248..207adb850a49b 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -57,3 +57,7 @@ deploy: deps @echo "Deploying HTML documentation." $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) -- deploy $(DOCUMENTER_OPTIONS) @echo "Build & deploy of docs finished." + +update-documenter: + @echo "Updating Documenter." + JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project --color=yes -e 'using Pkg; Pkg.update("Documenter")' diff --git a/doc/Manifest.toml b/doc/Manifest.toml index cb2508030ece4..86a9a130d49f1 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -44,9 +44,9 @@ version = "0.9.4" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "6f8730fd1bdf974009ef296bd81afb2728854fc0" +git-tree-sha1 = "6c182d0bd94142d7cbc3ae8a1e74668f15d0dd65" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.11.3" +version = "1.11.4" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] From ee7e8147902996ac8c563672c666c787126613ca Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 16 May 2025 15:52:16 -0400 Subject: [PATCH 278/662] stacktraces: Add an extension point for printing custom-owner CIs (#58430) On Julia master, it is possible to add custom CodeInstances with non `nothing` owner to the runtime system (e.g. by invoking them or with custom AbstractInterpreters). However, the backtrace printing for these custom code instances was not in any way distinguished from the backtrace printing for an ordinary code instance of the same method instance, making it sometimes hard to determine what code instance a particular backtrace was referring to (if the downstream compiler was making many different code instances for a particular MethodInstance). Remidy this by adding an explicit extension point that can be overwritten to modify the printing of these custom-owner CodeInstances. --- Compiler/src/Compiler.jl | 4 +++- Compiler/src/abstractinterpretation.jl | 6 ------ base/invalidation.jl | 2 +- base/runtime_internals.jl | 9 +++++++++ base/stacktraces.jl | 12 +++++++++++- base/staticdata.jl | 11 +---------- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 9c3581677132e..68eab073b7e2d 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -65,7 +65,9 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout, uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal, _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized, - partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using + partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using, + get_ci_mi + using Base.Order import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!, diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index bff75c63681cd..a16bc06dca103 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2221,12 +2221,6 @@ function get_ci_abi(ci::CodeInstance) (def::MethodInstance).specTypes end -function get_ci_mi(ci::CodeInstance) - def = ci.def - isa(def, ABIOverride) && return def.def - return def::MethodInstance -end - function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState) argtypes = arginfo.argtypes ft′ = argtype_by_index(argtypes, 2) diff --git a/base/invalidation.jl b/base/invalidation.jl index bb312931a567b..f26e7968d8a2a 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -188,7 +188,7 @@ invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPar invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance}) - meth = isa(edge, Method) ? edge : Compiler.get_ci_mi(edge).def + meth = isa(edge, Method) ? edge : get_ci_mi(edge).def ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth) return nothing end diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index db8fd114a3493..dc78d35a0bc0d 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1478,6 +1478,15 @@ end _uncompressed_ir(codeinst::CodeInstance, s::String) = ccall(:jl_uncompress_ir, Ref{CodeInfo}, (Any, Any, Any), codeinst.def.def::Method, codeinst, s) +function get_ci_mi(codeinst::CodeInstance) + def = codeinst.def + if def isa Core.ABIOverride + return def.def + else + return def::MethodInstance + end +end + """ Base.generating_output([incremental::Bool])::Bool diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 9e454e430e2b6..653419d2148fb 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -273,8 +273,12 @@ function show_spec_linfo(io::IO, frame::StackFrame) if linfo isa Union{MethodInstance, CodeInstance} def = frame_method_or_module(frame) if def isa Module - Base.show_mi(io, linfo, #=from_stackframe=#true) + Base.show_mi(io, linfo::MethodInstance, #=from_stackframe=#true) + elseif linfo isa CodeInstance && linfo.owner !== nothing + show_custom_spec_sig(io, linfo.owner, linfo, frame) else + # Equivalent to the default implementation of `show_custom_spec_sig` + # for `linfo isa CodeInstance`, but saves an extra dynamic dispatch. show_spec_sig(io, def, frame_mi(frame).specTypes) end else @@ -284,6 +288,12 @@ function show_spec_linfo(io::IO, frame::StackFrame) end end +# Can be extended by compiler packages to customize backtrace display of custom code instance frames +function show_custom_spec_sig(io::IO, @nospecialize(owner), linfo::CodeInstance, frame::StackFrame) + mi = get_ci_mi(linfo) + return show_spec_sig(io, mi.def, mi.specTypes) +end + function show_spec_sig(io::IO, m::Method, @nospecialize(sig::Type)) if get(io, :limit, :false)::Bool if !haskey(io, :displaysize) diff --git a/base/staticdata.jl b/base/staticdata.jl index 0b65d97750194..928372154653f 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -3,22 +3,13 @@ module StaticData using .Core: CodeInstance, MethodInstance -using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable +using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1 const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) debug_method_invalidation(onoff::Bool) = _jl_debug_method_invalidation[] = onoff ? Any[] : nothing -function get_ci_mi(codeinst::CodeInstance) - def = codeinst.def - if def isa Core.ABIOverride - return def.def - else - return def::MethodInstance - end -end - # Restore backedges to external targets # `edges` = [caller1, ...], the list of worklist-owned code instances internally # `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally From a483d90adc6176e5a3639a329a3a1fd309411b81 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 16 May 2025 15:52:54 -0400 Subject: [PATCH 279/662] codegen: Unify printing of and add some more internal IR validity errors (#58429) What should codegen do when it detects invalid IR? There are a few reasonable options. One is to just assert - this is relatively straightforward but also not very friendly because it immediately takes down your session, so you can't inspect what values may have caused the issue. Additionally, often allow invalid IR in dead code (because otherwise transformation passes would have to check whether the transformation that they're doing makes some code dead, which can be expensive or not necessarily visible). Thus we generally try to keep codegen going, trying to bail back to valid code as quickly as possible. In a few places, we additionally insert an unmodeled `emit_error("Internal Error")`. These are a bit weird because the earlier stages of the compiler do not know that they can exist, but in practice they mostly work fine (although they can cause additional crashes if there are try/catches in the way). If we don't, then we generally just stop codegening, so if execution ever were to reach there, it'll just run into whatever code comes after, likely crashing fairly soon. Because of this consideration, the `emit_error` is generally preferred. However, the tradeoff is that it increases the size of the code and LLVM can no longer optimize under the assumption that a particular code branch doesn't happen. We thus need to be judicious in where we use it. The general guidance is that it's fine to use in situations where the IR itself is obviously invalid, but should not be arbitrarily added to all instances of `unreachable` (putting behind NDEBUG is fine). This PR cleans this up a bit by changing all these error locations to print `INTERNAL ERROR - IR Validity`, as well as adding a new one to `emit_invoke` when there's a manifest mismatch in type of the arguments and the signature. This new case is particularly useful after the recent addition of ABIOverride, as that makes it more likely that external AbstractInterpreters are doing ABI shenanigans. --- src/codegen.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2a937fc626c0b..c0aeb5bac1915 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4990,8 +4990,10 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } jl_value_t *jt = jl_nth_slot_type(specTypes, i); jl_cgval_t arg = update_julia_type(ctx, argv[i], jt); - if (arg.typ == jl_bottom_type) + if (arg.typ == jl_bottom_type) { + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Argument type mismatch in Expr(:invoke)"); return jl_cgval_t(); + } if (is_uniquerep_Type(jt)) { continue; } @@ -5028,7 +5030,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos Value *val = emit_unbox(ctx, et, arg, jt); if (!val) { // There was a type mismatch of some sort - exit early - CreateTrap(ctx.builder); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Argument type mismatch in Expr(:invoke)"); return jl_cgval_t(); } argvals[idx] = val; @@ -5295,7 +5297,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } if (result.typ == jl_bottom_type) { #ifndef JL_NDEBUG - emit_error(ctx, "(Internal Error - IR Validity): Returned from function we expected not to."); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Returned from function we expected not to."); #endif CreateTrap(ctx.builder); } @@ -5647,7 +5649,7 @@ static jl_cgval_t emit_local(jl_codectx_t &ctx, jl_value_t *slotload) if (sym == jl_unused_sym) { // This shouldn't happen in well-formed input, but let's be robust, // since we otherwise cause undefined behavior here. - emit_error(ctx, "(INTERNAL ERROR): Tried to use `#undef#` argument."); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Tried to use `#undef#` argument."); return jl_cgval_t(); } return emit_varinfo(ctx, vi, sym); @@ -6494,7 +6496,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ if (source.constant == NULL) { // For now, we require non-constant source to be handled by using // eval. This should probably be a verifier error and an abort here. - emit_error(ctx, "(internal error) invalid IR: opaque closure source must be constant"); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): opaque closure source must be constant"); return jl_cgval_t(); } bool can_optimize = argt.constant != NULL && lb.constant != NULL && ub.constant != NULL && @@ -9380,7 +9382,7 @@ static jl_llvm_functions_t // Probably dead code, but let's be loud about it in case it isn't, so we fail // at the point of the miscompile, rather than later when something attempts to // read the scope. - emit_error(ctx, "(INTERNAL ERROR): Attempted to execute EnterNode with bad scope"); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Attempted to execute EnterNode with bad scope"); find_next_stmt(-1); continue; } From fc456bdd04d6d391381554e5c8e5d9a1e3ca6ce3 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 16 May 2025 19:15:46 -0400 Subject: [PATCH 280/662] Add some precompiles to help loading time (#58436) --- contrib/generate_precompile.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 2292eb52fa2b3..cc91f577959e9 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -103,6 +103,7 @@ precompile(Base.CoreLogging.current_logger_for_env, (Base.CoreLogging.LogLevel, precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module)) precompile(Base.StackTraces.lookup, (Ptr{Nothing},)) precompile(Tuple{typeof(Base.run_module_init), Module, Int}) +precompile(Tuple{Type{Base.VersionNumber}, Int32, Int32, Int32}) # Presence tested in the tests precompile(Tuple{typeof(Base.print), Base.IOStream, String}) @@ -140,6 +141,9 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter()) end empty!(Set()) push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) +get!(ENV, "___DUMMY", "") +ENV["___DUMMY"] +delete!(ENV, "___DUMMY") (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] @@ -209,6 +213,9 @@ if Artifacts !== nothing end dlopen("libjulia$(Base.isdebugbuild() ? "-debug" : "")", RTLD_LAZY | RTLD_DEEPBIND) """ + hardcoded_precompile_statements *= """ + precompile(Tuple{typeof(Artifacts._artifact_str), Module, String, Base.SubString{String}, String, Base.Dict{String, Any}, Base.SHA1, Base.BinaryPlatforms.Platform, Base.Val{Artifacts}}) + """ end FileWatching = get(Base.loaded_modules, From de090a92b3d564179d1fdeed7455d91c356accfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Sat, 17 May 2025 07:34:13 +0100 Subject: [PATCH 281/662] [Compiler] Add more tests for non-power-of-two primitive types (#58437) The issue reported in #42326 (inconsistent sizes for structs involving primitive types with non-power-of-two sizes) is already fixed since upgrading to LLVM 18, this PR adds regressions tests. I verified they already pass on x86_64, i686 and aarch64. Close #42326. --- Compiler/test/codegen.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 3ff9790e29cf8..373d1d91cd912 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -951,6 +951,16 @@ for (T, StructName) in ((Int128, :Issue55558), (UInt128, :UIssue55558)) end end +# Issue #42326 +primitive type PadAfter64_42326 448 end +mutable struct CheckPadAfter64_42326 + a::UInt64 + pad::PadAfter64_42326 + b::UInt64 +end +@test fieldoffset(CheckPadAfter64_42326, 3) == 80 +@test sizeof(CheckPadAfter64_42326) == 96 + @noinline Base.@nospecializeinfer f55768(@nospecialize z::UnionAll) = z === Vector @test f55768(Vector) @test f55768(Vector{T} where T) From 18d5ccd6c30eff16aca186dc65a42c39714020b0 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sat, 17 May 2025 07:56:33 -0400 Subject: [PATCH 282/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?SparseArrays=20stdlib=20from=20f3610c0=20to=206d072a8=20(#58446?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: SparseArrays URL: https://github.com/JuliaSparse/SparseArrays.jl.git Stdlib branch: main Julia branch: master Old commit: f3610c0 New commit: 6d072a8 Julia version: 1.13.0-DEV SparseArrays version: 1.12.0(Does not match) Bump invoked by: @IanButterworth Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaSparse/SparseArrays.jl/compare/f3610c07fe0403792743d9c9802a25642a5f2d18...6d072a81fca5f4394f88a012f4ce914c70769303 ``` $ git log --oneline f3610c0..6d072a8 6d072a8 Lazily init cholmod (#626) 8ff11da fix libsuitesparseconfig bug (#625) fac1548 lazily init cholmod 75eaa18 fix libsuitesparseconfig bug 7b6e810 Use `libsuitesparseconfig` from JLL (#620) b225a85 Clarify pros, cons and limitations of Cholesky and LDLt (#621) 3d42644 Update index.md (#623) 16bbcbc 5-term mul! with Diagonal (#603) d050b1b Relax `eltype` in `Diagonal` `ldiv!`/`rdiv!` (#616) 4968cff `ldiv!` for `Diagonal` and a sparse vector (#613) 5062034 Fix `issymmetric` for matrices with empty columns (#606) 9a46561 Replace `v == zero(v)` with `_iszero` (#610) ``` Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 delete mode 100644 deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 new file mode 100644 index 0000000000000..09c3bafa19e6a --- /dev/null +++ b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 @@ -0,0 +1 @@ +ec80bedb86483002a78de7859f524621 diff --git a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 new file mode 100644 index 0000000000000..979950d9080f4 --- /dev/null +++ b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 @@ -0,0 +1 @@ +563e706de71b7147f44fa28ec1a729e08172bdf27bdf741f0702a451717f561562b2211d74ca2d01218fd9da05268436dc795fe7aeef7d34b46bbd966c23f71e diff --git a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 b/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 deleted file mode 100644 index 541d74f9eee17..0000000000000 --- a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ab4ff8bd5749c1c5915ecf9ac1ae82a3 diff --git a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 b/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 deleted file mode 100644 index 46bcc3ea9668f..0000000000000 --- a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0586606d86bca352f6610ba2baeecbe997b8c90ee798c9ab89f42bf3c94c5f23c56ea75fe49d8ccc39d8b25f02592db879c95affa30f98cb45024b177157249a diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index 9498fbe4943f9..9230b5c3e6d94 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = f3610c07fe0403792743d9c9802a25642a5f2d18 +SPARSEARRAYS_SHA1 = 6d072a81fca5f4394f88a012f4ce914c70769303 SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From d04724a6de9a4ad9a17c1b3a3d96ef94b264be4e Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Fri, 25 Apr 2025 09:02:20 +0200 Subject: [PATCH 283/662] Implement takestring! This function creates a string from an IOBuffer or a Vector, emptying the input object and reusing the memory where possible. --- NEWS.md | 1 + base/exports.jl | 1 + base/iobuffer.jl | 74 +++++++++++++++++++++++++++++++++++++++++ base/strings/string.jl | 20 ++++++++++- doc/src/base/strings.md | 1 + src/genericmemory.c | 1 - test/iobuffer.jl | 30 +++++++++++++++++ test/strings/basic.jl | 18 ++++++++++ 8 files changed, 144 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index fcf94b340ad66..9057e012b0cef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -50,6 +50,7 @@ New library features * `fieldoffset` now also accepts the field name as a symbol as `fieldtype` already did ([#58100]). * `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). * `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). +* `takestring!(::IOBuffer)` removes the content from the buffer, returning the content as a `String`. Standard library changes ------------------------ diff --git a/base/exports.jl b/base/exports.jl index 36b9bba3d8358..53f6152ea55f2 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -682,6 +682,7 @@ export split, string, strip, + takestring!, textwidth, thisind, titlecase, diff --git a/base/iobuffer.jl b/base/iobuffer.jl index a28fe5376755b..bca1012ad2394 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -783,6 +783,80 @@ function take!(io::IOBuffer) return data end +"Internal method. This method can be faster than takestring!, because it does not +reset the buffer to a usable state, and it does not check for io.reinit. +Using the buffer after calling unsafe_takestring! may cause undefined behaviour. +This function is meant to be used when the buffer is only used as a temporary +string builder, which is discarded after the string is built." +function unsafe_takestring!(io::IOBuffer) + used_span = get_used_span(io) + nbytes = length(used_span) + from = first(used_span) + isempty(used_span) && return "" + # The C function can only copy from the start of the memory. + # Fortunately, in most cases, the offset will be zero. + return if isone(from) + ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), io.data, nbytes) + else + mem = StringMemory(nbytes % UInt) + unsafe_copyto!(mem, 1, io.data, from, nbytes) + unsafe_takestring(mem) + end +end + +""" + takestring!(io::IOBuffer) -> String + +Return the content of `io` as a `String`, resetting the buffer to its initial +state. +This is preferred over calling `String(take!(io))` to create a string from +an `IOBuffer`. + +# Examples +```jldoctest +julia> io = IOBuffer(); + +julia> write(io, [0x61, 0x62, 0x63]); + +julia> s = takestring!(io) +"abc" + +julia> isempty(take!(io)) # io is now empty +true +``` + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. +""" +function takestring!(io::IOBuffer) + # If the buffer has been used up and needs to be replaced, there are no bytes, and + # we can return an empty string without interacting with the buffer at all. + io.reinit && return "" + + # If the iobuffer is writable, taking will remove the buffer from `io`. + # So, we reset the iobuffer, and directly unsafe takestring. + return if io.writable + s = unsafe_takestring!(io) + io.reinit = true + io.mark = -1 + io.ptr = 1 + io.size = 0 + io.offset_or_compacted = 0 + s + else + # If the buffer is not writable, taking will NOT remove the buffer, + # so if we just converted the buffer to a string, garbage collecting + # the string would free the memory underneath the iobuffer + used_span = get_used_span(io) + mem = StringMemory(length(used_span)) + unsafe_copyto!(mem, 1, io.data, first(used_span), length(used_span)) + unsafe_takestring(mem) + end +end + +# Fallback methods +takestring!(io::GenericIOBuffer) = String(take!(io)) + """ _unsafe_take!(io::IOBuffer) diff --git a/base/strings/string.jl b/base/strings/string.jl index c2aa95690e99d..bf6e40c9ea372 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -62,8 +62,8 @@ In other cases, `Vector{UInt8}` data may be copied, but `v` is truncated anyway to guarantee consistent behavior. """ String(v::AbstractVector{UInt8}) = unsafe_takestring(copyto!(StringMemory(length(v)), v)) + function String(v::Vector{UInt8}) - #return ccall(:jl_array_to_string, Ref{String}, (Any,), v) len = length(v) len == 0 && return "" ref = v.ref @@ -84,6 +84,24 @@ function unsafe_takestring(m::Memory{UInt8}) isempty(m) ? "" : ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), m, length(m)) end +""" + takestring!(x) -> String + +Create a string from the content of `x`, emptying `x`. + +# Examples +```jldoctest +julia> v = [0x61, 0x62, 0x63]; + +julia> s = takestring!(v) +"abc" + +julia> isempty(v) +true +``` +""" +takestring!(v::Vector{UInt8}) = String(v) + """ unsafe_string(p::Ptr{UInt8}, [length::Integer]) diff --git a/doc/src/base/strings.md b/doc/src/base/strings.md index a9637a1a7be3a..15a7c0531de4a 100644 --- a/doc/src/base/strings.md +++ b/doc/src/base/strings.md @@ -29,6 +29,7 @@ Base.SubstitutionString Base.@s_str Base.@raw_str Base.@b_str +Base.takestring! Base.Docs.@html_str Base.Docs.@text_str Base.isvalid(::Any) diff --git a/src/genericmemory.c b/src/genericmemory.c index 3676333ddb982..a81a4a199956c 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -197,7 +197,6 @@ JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_ } int how = jl_genericmemory_how(m); size_t mlength = m->length; - m->length = 0; if (how != 0) { jl_value_t *o = jl_genericmemory_data_owner_field(m); jl_genericmemory_data_owner_field(m) = NULL; diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 428e259f225fc..6163e59beb567 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -321,6 +321,36 @@ end @test_throws ArgumentError seek(io, 0) end +@testset "takestring!" begin + buf = IOBuffer() + write(buf, "abcø") + s = takestring!(buf) + @test isempty(takestring!(buf)) + @test s == "abcø" + write(buf, "xyz") + @test takestring!(buf) == "xyz" + buf = IOBuffer() + + # Test with a nonzero offset in the buffer + v = rand(UInt8, 8) + for i in 1:8 + pushfirst!(v, rand(UInt8)) + end + buf = IOBuffer(v) + s = String(copy(v)) + @test takestring!(buf) == s + + # Test with a non-writable IOBuffer + buf = IOBuffer(b"abcdef") + read(buf, UInt8) + @test takestring!(buf) == "abcdef" + + buf = new_unseekable_buffer() + write(buf, "abcde") + read(buf, UInt16) + @test takestring!(buf) == "cde" +end + @testset "Read/write readonly IOBuffer" begin io = IOBuffer("hamster\nguinea pig\nturtle") @test position(io) == 0 diff --git a/test/strings/basic.jl b/test/strings/basic.jl index c3e0bcc501070..b18e2830e06d8 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -49,6 +49,24 @@ using Random end end +@testset "takestring!" begin + v = [0x61, 0x62, 0x63] + old_mem = v.ref.mem + @test takestring!(v) == "abc" + @test isempty(v) + @test v.ref.mem !== old_mem # memory is changed + for v in [ + UInt8[], + [0x01, 0x02, 0x03], + collect(codeunits("æøå")) + ] + cp = copy(v) + s = takestring!(v) + @test isempty(v) + @test codeunits(s) == cp + end +end + @testset "{starts,ends}with" begin @test startswith("abcd", 'a') @test startswith('a')("abcd") From 6cfa52fb9aed8d24cce2dd1649789667b90280e1 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Fri, 25 Apr 2025 09:03:23 +0200 Subject: [PATCH 284/662] Use takestring! in Base and docs This commit switches the usage of the pattern `String(take!(::IOBuffer))` to use the new `takestring!` or `unsafe_takestring!` functions across Base, and in doc- strings. Not all occurrences in e.g. tests are switched over, as this would consistute a lot of code churn for no real purpose. --- Compiler/src/ssair/show.jl | 2 +- base/errorshow.jl | 4 ++-- base/filesystem.jl | 2 +- base/indices.jl | 2 +- base/int.jl | 2 +- base/io.jl | 21 +++++++++---------- base/iobuffer.jl | 4 ++-- base/iostream.jl | 6 +++--- base/logging/ConsoleLogger.jl | 2 +- base/pkgid.jl | 2 +- base/show.jl | 10 ++++----- base/stat.jl | 2 +- base/strings/annotated.jl | 4 ++-- base/strings/basic.jl | 2 +- base/strings/io.jl | 16 +++++++------- base/strings/unicode.jl | 2 +- base/strings/util.jl | 4 ++-- base/task.jl | 2 +- base/toml_parser.jl | 2 +- base/util.jl | 4 ++-- doc/src/devdocs/functions.md | 2 +- doc/src/manual/getting-started.md | 2 +- stdlib/Base64/src/encode.jl | 4 ++-- stdlib/FileWatching/src/pidfile.jl | 7 ++++--- .../InteractiveUtils/src/InteractiveUtils.jl | 2 +- stdlib/LibGit2/src/utils.jl | 2 +- stdlib/Markdown/src/Common/block.jl | 8 +++---- stdlib/Markdown/src/GitHub/GitHub.jl | 4 ++-- stdlib/Markdown/src/parse/parse.jl | 4 ++-- stdlib/Markdown/src/parse/util.jl | 4 ++-- stdlib/REPL/src/LineEdit.jl | 10 ++++----- stdlib/REPL/src/REPL.jl | 8 +++---- 32 files changed, 76 insertions(+), 76 deletions(-) diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index 1b6eeca57f3f9..de58018866274 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -328,7 +328,7 @@ function compute_ir_line_annotations(code::IRCode) loc_method = string(" "^printing_depth, loc_method) last_stack = stack end - push!(loc_annotations, String(take!(buf))) + push!(loc_annotations, takestring!(buf)) push!(loc_lineno, (lineno != 0 && lineno != last_lineno) ? string(lineno) : "") push!(loc_methods, loc_method) (lineno != 0) && (last_lineno = lineno) diff --git a/base/errorshow.jl b/base/errorshow.jl index 384b459f99972..4a8003ad82b6f 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -307,7 +307,7 @@ function showerror(io::IO, ex::MethodError) iob = IOContext(buf, io) # for type abbreviation as in #49795; some, like `convert(T, x)`, should not abbreviate show_signature_function(iob, Core.Typeof(f)) show_tuple_as_call(iob, :function, arg_types; hasfirst=false, kwargs = isempty(kwargs) ? nothing : kwargs) - str = String(take!(buf)) + str = takestring!(buf) str = type_limited_string_from_context(io, str) print(io, str) end @@ -598,7 +598,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) m = parentmodule_before_main(method) modulecolor = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, m) print_module_path_file(iob, m, string(file), line; modulecolor, digit_align_width = 3) - push!(lines, String(take!(buf))) + push!(lines, takestring!(buf)) push!(line_score, -(right_matches * 2 + (length(arg_types_param) < 2 ? 1 : 0))) end end diff --git a/base/filesystem.jl b/base/filesystem.jl index 2934fc15e392f..a5f1327b5cff1 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -141,7 +141,7 @@ import .Base: bytesavailable, position, read, read!, readbytes!, readavailable, seek, seekend, show, skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, _uv_error, setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, - isexecutable, isreadable, iswritable, MutableDenseArrayType, truncate + isexecutable, isreadable, iswritable, MutableDenseArrayType, truncate, unsafe_takestring! import .Base.RefValue diff --git a/base/indices.jl b/base/indices.jl index 45f3495e51191..88e48a2b331ee 100644 --- a/base/indices.jl +++ b/base/indices.jl @@ -132,7 +132,7 @@ function throw_promote_shape_mismatch(a::Tuple, b::Union{Nothing,Tuple}, i = not if i ≢ nothing print(msg, ", mismatch at dim ", i) end - throw(DimensionMismatch(String(take!(msg)))) + throw(DimensionMismatch(takestring!(msg))) end function promote_shape(a::Tuple{Int,}, b::Tuple{Int,}) diff --git a/base/int.jl b/base/int.jl index dfb4cc5d99b06..6fdcc413c04a6 100644 --- a/base/int.jl +++ b/base/int.jl @@ -722,7 +722,7 @@ macro big_str(s::String) is_prev_dot = (c == '.') end print(bf, s[end]) - s = String(take!(bf)) + s = unsafe_takestring!(bf) end n = tryparse(BigInt, s) n === nothing || return n diff --git a/base/io.jl b/base/io.jl index 2fb8ce0c95f06..c0e1740c5bd11 100644 --- a/base/io.jl +++ b/base/io.jl @@ -277,13 +277,13 @@ julia> io = IOBuffer(); julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.") 56 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization. It has many members." julia> write(io, "Sometimes those members") + write(io, " write documentation.") 44 -julia> String(take!(io)) +julia> takestring!(io) "Sometimes those members write documentation." ``` User-defined plain-data types without `write` methods can be written when wrapped in a `Ref`: @@ -544,7 +544,7 @@ julia> rm("my_file.txt") """ readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String) readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) -readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = String(_unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...))) +readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) readuntil(stream::IO, delim::T; keep::Bool=false) where T = _copyuntil(Vector{T}(), stream, delim, keep) @@ -566,10 +566,10 @@ Similar to [`readuntil`](@ref), which returns a `String`; in contrast, ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); -julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", 'L'))) +julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", 'L')) "Julia" -julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true))) +julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true)) "JuliaLang is a GitHub organization." julia> rm("my_file.txt") @@ -616,8 +616,7 @@ Logan """ readline(filename::AbstractString; keep::Bool=false) = open(io -> readline(io; keep), filename) -readline(s::IO=stdin; keep::Bool=false) = - String(_unsafe_take!(copyline(IOBuffer(sizehint=16), s; keep))) +readline(s::IO=stdin; keep::Bool=false) = takestring!(copyline(IOBuffer(sizehint=16), s; keep)) """ copyline(out::IO, io::IO=stdin; keep::Bool=false) @@ -642,10 +641,10 @@ See also [`copyuntil`](@ref) for reading until more general delimiters. ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); -julia> String(take!(copyline(IOBuffer(), "my_file.txt"))) +julia> takestring!(copyline(IOBuffer(), "my_file.txt")) "JuliaLang is a GitHub organization." -julia> String(take!(copyline(IOBuffer(), "my_file.txt", keep=true))) +julia> takestring!(copyline(IOBuffer(), "my_file.txt", keep=true)) "JuliaLang is a GitHub organization.\\n" julia> rm("my_file.txt") @@ -1290,7 +1289,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) empty!(chunks) # will cause next iteration to terminate seekend(r.itr.stream) # reposition to end of stream for isdone - s = String(_unsafe_take!(buf)) + s = unsafe_takestring!(buf) else # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline] if ichunk == jchunk # common case: current and previous newline in same chunk @@ -1307,7 +1306,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) end write(buf, view(chunks[jchunk], 1:jnewline)) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) - s = String(_unsafe_take!(buf)) + s = unsafe_takestring!(buf) # overwrite obsolete chunks (ichunk+1:jchunk) i = jchunk diff --git a/base/iobuffer.jl b/base/iobuffer.jl index bca1012ad2394..b2e51618a8835 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -198,7 +198,7 @@ julia> io = IOBuffer(); julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.") 56 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization. It has many members." julia> io = IOBuffer(b"JuliaLang is a GitHub organization.") @@ -216,7 +216,7 @@ IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=fa julia> write(io, "JuliaLang is a GitHub organization.") 34 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization" julia> length(read(IOBuffer(b"data", read=true, truncate=false))) diff --git a/base/iostream.jl b/base/iostream.jl index 7a06e7d1e237b..8847c1b2d8ccd 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -111,7 +111,7 @@ julia> write(io, "JuliaLang is a GitHub organization.") julia> truncate(io, 15) IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=15, maxsize=Inf, ptr=16, mark=-1) -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a " julia> io = IOBuffer(); @@ -120,7 +120,7 @@ julia> write(io, "JuliaLang is a GitHub organization."); julia> truncate(io, 40); -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization.\\0\\0\\0\\0\\0" ``` """ @@ -469,7 +469,7 @@ function readuntil_string(s::IOStream, delim::UInt8, keep::Bool) end readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) = isascii(delim) ? readuntil_string(s, delim % UInt8, keep) : - String(_unsafe_take!(copyuntil(IOBuffer(sizehint=70), s, delim; keep))) + takestring!(copyuntil(IOBuffer(sizehint=70), s, delim; keep)) function readline(s::IOStream; keep::Bool=false) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2) diff --git a/base/logging/ConsoleLogger.jl b/base/logging/ConsoleLogger.jl index 8766d0ae56331..6521bf49b3e66 100644 --- a/base/logging/ConsoleLogger.jl +++ b/base/logging/ConsoleLogger.jl @@ -147,7 +147,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module for (key, val) in kwargs key === :maxlog && continue showvalue(valio, val) - vallines = split(String(take!(valbuf)), '\n') + vallines = split(takestring!(valbuf), '\n') if length(vallines) == 1 push!(msglines, (indent=2, msg=SubString("$key = $(vallines[1])"))) else diff --git a/base/pkgid.jl b/base/pkgid.jl index 577529bbe7f63..7ef7c58eee4cc 100644 --- a/base/pkgid.jl +++ b/base/pkgid.jl @@ -32,7 +32,7 @@ function binpack(pkg::PkgId) uuid = pkg.uuid write(io, uuid === nothing ? UInt128(0) : UInt128(uuid)) write(io, pkg.name) - return String(take!(io)) + return unsafe_takestring!(io) end function binunpack(s::String) diff --git a/base/show.jl b/base/show.jl index 879916d794f96..9864fc7e5dec8 100644 --- a/base/show.jl +++ b/base/show.jl @@ -390,12 +390,12 @@ julia> io = IOBuffer(); julia> printstyled(IOContext(io, :color => true), "string", color=:red) -julia> String(take!(io)) +julia> takestring!(io) "\\e[31mstring\\e[39m" julia> printstyled(io, "string", color=:red) -julia> String(take!(io)) +julia> takestring!(io) "string" ``` @@ -2649,7 +2649,7 @@ function show_tuple_as_call(out::IO, name::Symbol, sig::Type; end print_within_stacktrace(io, ")", bold=true) show_method_params(io, tv) - str = String(take!(buf)) + str = takestring!(buf) str = type_limited_string_from_context(out, str) print(out, str) nothing @@ -2758,7 +2758,7 @@ function type_depth_limit(str::String, n::Int; maxdepth = nothing) end prev = di end - return String(take!(output)) + return unsafe_takestring!(output) end function print_type_bicolor(io, type; kwargs...) @@ -3193,7 +3193,7 @@ summary(io::IO, x) = print(io, typeof(x)) function summary(x) io = IOBuffer() summary(io, x) - String(take!(io)) + takestring!(io) end ## `summary` for AbstractArrays diff --git a/base/stat.jl b/base/stat.jl index 8753c18f3ea38..fbab5126d39bc 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -320,7 +320,7 @@ function filemode_string(mode) end complete && write(str, "-") end - return String(take!(str)) + return unsafe_takestring!(str) end """ diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 0da6c26751618..cb2d09f5b7054 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -272,7 +272,7 @@ function annotatedstring(xs...) print(s, x) end end - str = String(take!(buf)) + str = takestring!(buf) AnnotatedString(str, annotations) end @@ -457,7 +457,7 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin stop_offset = last(offsets[findlast(<=(stop) ∘ first, offsets)::Int]) push!(annots, setindex(annot, (start + start_offset):(stop + stop_offset), :region)) end - AnnotatedString(String(take!(outstr)), annots) + AnnotatedString(takestring!(outstr), annots) end struct RegionIterator{S <: AbstractString} diff --git a/base/strings/basic.jl b/base/strings/basic.jl index b92c177b0ec56..314903898b92a 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -678,7 +678,7 @@ function filter(f, s::AbstractString) for c in s f(c) && write(out, c) end - String(_unsafe_take!(out)) + takestring!(out) end ## string first and last ## diff --git a/base/strings/io.jl b/base/strings/io.jl index 89a5a992cbfd0..c9c5de1d791d5 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -25,7 +25,7 @@ julia> io = IOBuffer(); julia> print(io, "Hello", ' ', :World!) -julia> String(take!(io)) +julia> takestring!(io) "Hello World!" ``` """ @@ -70,7 +70,7 @@ julia> io = IOBuffer(); julia> println(io, "Hello", ',', " world.") -julia> String(take!(io)) +julia> takestring!(io) "Hello, world.\\n" ``` """ @@ -112,7 +112,7 @@ function sprint(f::Function, args...; context=nothing, sizehint::Integer=0) else f(s, args...) end - String(_unsafe_take!(s)) + takestring!(s) end function _str_sizehint(x) @@ -146,7 +146,7 @@ function print_to_string(xs...) for x in xs print(s, x) end - String(_unsafe_take!(s)) + takestring!(s) end setfield!(typeof(print_to_string).name.mt, :max_args, 10, :monotonic) @@ -164,7 +164,7 @@ function string_with_env(env, xs...) for x in xs print(env_io, x) end - String(_unsafe_take!(s)) + takestring!(s) end """ @@ -294,10 +294,10 @@ Create a read-only `IOBuffer` on the data underlying the given string. ```jldoctest julia> io = IOBuffer("Haho"); -julia> String(take!(io)) +julia> takestring!(io) "Haho" -julia> String(take!(io)) +julia> takestring!(io) "Haho" ``` """ @@ -774,7 +774,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) print(buf, ' ') end end - String(take!(buf)) + takestring!(buf) end function String(a::AbstractVector{Char}) diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index eb03fd0eb2837..5d59b0fc3bff4 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -702,7 +702,7 @@ function titlecase(s::AbstractString; wordsep::Function = !isletter, strict::Boo end c0 = c end - return String(take!(b)) + return takestring!(b) end # TODO: improve performance characteristics, room for a ~10x improvement. diff --git a/base/strings/util.jl b/base/strings/util.jl index b8a2f7882c252..957be7b914c02 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -1051,7 +1051,7 @@ function _replace_(str, pat_repl::NTuple{N, Pair}, count::Int) where N return String(str) end out = IOBuffer(sizehint=floor(Int, 1.2sizeof(str))) - return String(take!(_replace_finish(out, str, count, e1, patterns, replaces, rs))) + return takestring!(_replace_finish(out, str, count, e1, patterns, replaces, rs)) end """ @@ -1286,5 +1286,5 @@ function Base.rest(s::AbstractString, st...) for c in Iterators.rest(s, st...) print(io, c) end - return String(take!(io)) + return takestring!(io) end diff --git a/base/task.jl b/base/task.jl index cddf1fc854f4c..bc20ff8ac320a 100644 --- a/base/task.jl +++ b/base/task.jl @@ -95,7 +95,7 @@ function show_task_exception(io::IO, t::Task; indent = true) else show_exception_stack(IOContext(b, io), stack) end - str = String(take!(b)) + str = takestring!(b) if indent str = replace(str, "\n" => "\n ") end diff --git a/base/toml_parser.jl b/base/toml_parser.jl index ddacacee5ae0e..26c690253cb8a 100644 --- a/base/toml_parser.jl +++ b/base/toml_parser.jl @@ -315,7 +315,7 @@ function point_to_line(str::AbstractString, a::Int, b::Int, context) c == '\n' && break print(io1, c) end - return String(take!(io1.io)), String(take!(io2.io)) + return takestring!(io1.io), takestring!(io2.io) end function Base.showerror(io::IO, err::ParserError) diff --git a/base/util.jl b/base/util.jl index a05858305223a..fa01f1bcde498 100644 --- a/base/util.jl +++ b/base/util.jl @@ -77,7 +77,7 @@ function with_output_color(@nospecialize(f::Function), color::Union{Int, Symbol} iscolor = get(io, :color, false)::Bool try f(IOContext(buf, io), args...) finally - str = String(take!(buf)) + str = takestring!(buf) if !iscolor print(io, str) else @@ -109,7 +109,7 @@ function with_output_color(@nospecialize(f::Function), color::Union{Int, Symbol} isempty(line) && continue print(buf, enable_ansi, line, disable_ansi) end - print(io, String(take!(buf))) + print(io, takestring!(buf)) end end end diff --git a/doc/src/devdocs/functions.md b/doc/src/devdocs/functions.md index 777afaa56348d..fb67dfd17c3e2 100644 --- a/doc/src/devdocs/functions.md +++ b/doc/src/devdocs/functions.md @@ -117,7 +117,7 @@ function lines(words) n += length(w)+1 end end - String(take!(io)) + takestring!(io) end import Markdown [string(n) for n in names(Core;all=true) diff --git a/doc/src/manual/getting-started.md b/doc/src/manual/getting-started.md index b0299a4563f98..502fbd59166f7 100644 --- a/doc/src/manual/getting-started.md +++ b/doc/src/manual/getting-started.md @@ -13,7 +13,7 @@ known as a read-eval-print loop or "REPL") by double-clicking the Julia executab using REPL io = IOBuffer() REPL.banner(io) -banner = String(take!(io)) +banner = takestring!(io) import Markdown Markdown.parse("```\n\$ julia\n\n$(banner)\njulia> 1 + 2\n3\n\njulia> ans\n3\n```") ``` diff --git a/stdlib/Base64/src/encode.jl b/stdlib/Base64/src/encode.jl index 588b49aa28d97..d55421c52ff55 100644 --- a/stdlib/Base64/src/encode.jl +++ b/stdlib/Base64/src/encode.jl @@ -24,7 +24,7 @@ julia> write(iob64_encode, "Hello!") julia> close(iob64_encode); -julia> str = String(take!(io)) +julia> str = takestring!(io) "SGVsbG8h" julia> String(base64decode(str)) @@ -211,6 +211,6 @@ function base64encode(f::Function, args...; context=nothing) f(IOContext(b, context), args...) end close(b) - return String(take!(s)) + return takestring!(s) end base64encode(args...; context=nothing) = base64encode(write, args...; context=context) diff --git a/stdlib/FileWatching/src/pidfile.jl b/stdlib/FileWatching/src/pidfile.jl index 6862aaa9f8453..3a3ac7e754817 100644 --- a/stdlib/FileWatching/src/pidfile.jl +++ b/stdlib/FileWatching/src/pidfile.jl @@ -5,7 +5,8 @@ export mkpidlock, trymkpidlock using Base: IOError, UV_EEXIST, UV_ESRCH, UV_ENOENT, - Process + Process, + unsafe_takestring using Base.Filesystem: File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL, @@ -304,12 +305,12 @@ function open_exclusive(path::String; end function _rand_filename(len::Int=4) # modified from Base.Libc - slug = Base.StringVector(len) + slug = Base.StringMemory(len) chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" for i = 1:len slug[i] = chars[(Libc.rand() % length(chars)) + 1] end - return String(slug) + return unsafe_takestring(slug) end function tryrmopenfile(path::String) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 6b75a228b2761..d9d00851dbfa2 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -148,7 +148,7 @@ function versioninfo(io::IO=stdout; verbose::Bool=false) if verbose cpuio = IOBuffer() # print cpu_summary with correct alignment Sys.cpu_summary(cpuio) - for (i, line) in enumerate(split(chomp(String(take!(cpuio))), "\n")) + for (i, line) in enumerate(split(chomp(takestring!(cpuio)), "\n")) prefix = i == 1 ? " CPU: " : " " println(io, prefix, line) end diff --git a/stdlib/LibGit2/src/utils.jl b/stdlib/LibGit2/src/utils.jl index eb8b878d30484..6fc4acbd6bf77 100644 --- a/stdlib/LibGit2/src/utils.jl +++ b/stdlib/LibGit2/src/utils.jl @@ -162,7 +162,7 @@ function git_url(; end seekstart(io) - return String(take!(io)) + return takestring!(io) end function credential_identifier(scheme::AbstractString, host::AbstractString) diff --git a/stdlib/Markdown/src/Common/block.jl b/stdlib/Markdown/src/Common/block.jl index 247c894769f15..59ab0e58cf65b 100644 --- a/stdlib/Markdown/src/Common/block.jl +++ b/stdlib/Markdown/src/Common/block.jl @@ -114,7 +114,7 @@ function indentcode(stream::IO, block::MD) break end end - code = String(take!(buffer)) + code = takestring!(buffer) !isempty(code) && (push!(block, Code(rstrip(code))); return true) return false end @@ -178,7 +178,7 @@ function blockquote(stream::IO, block::MD) end empty && return false - md = String(take!(buffer)) + md = takestring!(buffer) push!(block, BlockQuote(parse(md, flavor = config(block)).content)) return true end @@ -236,7 +236,7 @@ function admonition(stream::IO, block::MD) end end # Parse the nested block as markdown and create a new Admonition block. - nested = parse(String(take!(buffer)), flavor = config(block)) + nested = parse(takestring!(buffer), flavor = config(block)) push!(block, Admonition(category, title, nested.content)) return true end @@ -326,7 +326,7 @@ function list(stream::IO, block::MD) return true end end -pushitem!(list, buffer) = push!(list.items, parse(String(take!(buffer))).content) +pushitem!(list, buffer) = push!(list.items, parse(takestring!(buffer)).content) # –––––––––––––– # HorizontalRule diff --git a/stdlib/Markdown/src/GitHub/GitHub.jl b/stdlib/Markdown/src/GitHub/GitHub.jl index 2ac75ea43cd19..de18b367988d9 100644 --- a/stdlib/Markdown/src/GitHub/GitHub.jl +++ b/stdlib/Markdown/src/GitHub/GitHub.jl @@ -21,9 +21,9 @@ function fencedcode(stream::IO, block::MD) if startswith(stream, string(ch) ^ n) if !startswith(stream, string(ch)) if flavor == "math" - push!(block, LaTeX(String(take!(buffer)) |> chomp)) + push!(block, LaTeX(takestring!(buffer) |> chomp)) else - push!(block, Code(flavor, String(take!(buffer)) |> chomp)) + push!(block, Code(flavor, takestring!(buffer) |> chomp)) end return true else diff --git a/stdlib/Markdown/src/parse/parse.jl b/stdlib/Markdown/src/parse/parse.jl index b38cc23b37dc0..8d691a281f218 100644 --- a/stdlib/Markdown/src/parse/parse.jl +++ b/stdlib/Markdown/src/parse/parse.jl @@ -63,7 +63,7 @@ function parseinline(stream::IO, md::MD, config::Config) char = peek(stream, Char) if haskey(config.inner, char) && (inner = parseinline(stream, md, config.inner[char])) !== nothing - c = String(take!(buffer)) + c = takestring!(buffer) !isempty(c) && push!(content, c) buffer = IOBuffer() push!(content, inner) @@ -71,7 +71,7 @@ function parseinline(stream::IO, md::MD, config::Config) write(buffer, read(stream, Char)) end end - c = String(take!(buffer)) + c = takestring!(buffer) !isempty(c) && push!(content, c) return content end diff --git a/stdlib/Markdown/src/parse/util.jl b/stdlib/Markdown/src/parse/util.jl index aabfcbb3ddc62..cd8158780bd6d 100644 --- a/stdlib/Markdown/src/parse/util.jl +++ b/stdlib/Markdown/src/parse/util.jl @@ -141,7 +141,7 @@ function readuntil(stream::IO, delimiter; newlines = false, match = nothing) while !eof(stream) if startswith(stream, delimiter) if count == 0 - return String(take!(buffer)) + return takestring!(buffer) else count -= 1 write(buffer, delimiter) @@ -187,7 +187,7 @@ function parse_inline_wrapper(stream::IO, delimiter::AbstractString; rep = false if !(char in whitespace || char == '\n' || char in delimiter) && startswith(stream, delimiter^n) trailing = 0 while startswith(stream, delimiter); trailing += 1; end - trailing == 0 && return String(take!(buffer)) + trailing == 0 && return takestring!(buffer) write(buffer, delimiter ^ (n + trailing)) end end diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 53e497a6548ee..ba4a8155b13bf 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -169,7 +169,7 @@ region_active(s::PromptState) = s.region_active region_active(s::ModeState) = :off -input_string(s::PromptState) = String(take!(copy(s.input_buffer)))::String +input_string(s::PromptState) = takestring!(copy(s.input_buffer))::String input_string_newlines(s::PromptState) = count(c->(c == '\n'), input_string(s)) function input_string_newlines_aftercursor(s::PromptState) @@ -1526,7 +1526,7 @@ function edit_input(s, f = (filename, line, column) -> InteractiveUtils.edit(fil end buf = buffer(s) pos = position(buf) - str = String(take!(buf)) + str = takestring!(buf) lines = readlines(IOBuffer(str); keep=true) # Compute line @@ -1760,7 +1760,7 @@ function normalize_key(key::Union{String,SubString{String}}) write(buf, c) end end - return String(take!(buf)) + return takestring!(buf) end function normalize_keys(keymap::Union{Dict{Char,Any},AnyDict}) @@ -2100,7 +2100,7 @@ function history_set_backward(s::SearchState, backward::Bool) nothing end -input_string(s::SearchState) = String(take!(copy(s.query_buffer))) +input_string(s::SearchState) = takestring!(copy(s.query_buffer)) function reset_state(s::SearchState) if s.query_buffer.size != 0 @@ -2188,7 +2188,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, return ias end -input_string(s::PrefixSearchState) = String(take!(copy(s.response_buffer))) +input_string(s::PrefixSearchState) = takestring!(copy(s.response_buffer)) write_prompt(terminal, s::PrefixSearchState, color::Bool) = write_prompt(terminal, s.histprompt.parent_prompt, color) prompt_string(s::PrefixSearchState) = prompt_string(s.histprompt.parent_prompt.prompt) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 169b6a71858ef..f742c0cc6cfda 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -923,7 +923,7 @@ function hist_from_file(hp::REPLHistoryProvider, path::String) end function add_history(hist::REPLHistoryProvider, s::PromptState) - str = rstrip(String(take!(copy(s.input_buffer)))) + str = rstrip(takestring!(copy(s.input_buffer))) isempty(strip(str)) && return mode = mode_idx(hist, LineEdit.mode(s)) !isempty(hist.history) && @@ -1056,7 +1056,7 @@ function history_move_prefix(s::LineEdit.PrefixSearchState, prefix::AbstractString, backwards::Bool, cur_idx::Int = hist.cur_idx) - cur_response = String(take!(copy(LineEdit.buffer(s)))) + cur_response = takestring!(copy(LineEdit.buffer(s))) # when searching forward, start at last_idx if !backwards && hist.last_idx > 0 cur_idx = hist.last_idx @@ -1098,7 +1098,7 @@ function history_search(hist::REPLHistoryProvider, query_buffer::IOBuffer, respo qpos = position(query_buffer) qpos > 0 || return true searchdata = beforecursor(query_buffer) - response_str = String(take!(copy(response_buffer))) + response_str = takestring!(copy(response_buffer)) # Alright, first try to see if the current match still works a = position(response_buffer) + 1 # position is zero-indexed @@ -1155,7 +1155,7 @@ end LineEdit.reset_state(hist::REPLHistoryProvider) = history_reset_state(hist) function return_callback(s) - ast = Base.parse_input_line(String(take!(copy(LineEdit.buffer(s)))), depwarn=false) + ast = Base.parse_input_line(takestring!(copy(LineEdit.buffer(s))), depwarn=false) return !(isa(ast, Expr) && ast.head === :incomplete) end From 1b0b02869daef1d2a1b8e8105075132357ee6897 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Mon, 19 May 2025 12:15:46 -0300 Subject: [PATCH 285/662] Fix layout flags for types that have oddly sized primitive type fields (#58435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is caused because for LLVMs sake we have to say that the oddly typed field is smaller than it actually is. (I wonder if we could represent it as an iN field in a struct and have it work but the result would be the same for now) Fix #58434, fix #49318, close #49362. --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> --- src/datatype.c | 6 ++++++ test/core.jl | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/datatype.c b/src/datatype.c index cbaf6e98993d9..0ccdd0b61d06f 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -702,6 +702,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) // Should never happen throw_ovf(should_malloc, desc, st, fsz); desc[i].isptr = 0; + if (jl_is_uniontype(fld)) { fsz += 1; // selector byte zeroinit = 1; @@ -709,6 +710,11 @@ void jl_compute_field_offsets(jl_datatype_t *st) isbitsegal = 0; } else { + if (fsz > jl_datatype_size(fld)) { + // We have to pad the size to integer size class, but it means this has some padding + isbitsegal = 0; + haspadding = 1; + } uint32_t fld_npointers = ((jl_datatype_t*)fld)->layout->npointers; if (((jl_datatype_t*)fld)->layout->flags.haspadding) haspadding = 1; diff --git a/test/core.jl b/test/core.jl index f24eeb23909c8..c65a2a821e275 100644 --- a/test/core.jl +++ b/test/core.jl @@ -8534,3 +8534,9 @@ module GlobalBindingMulti using .M.C end @test GlobalBindingMulti.S === GlobalBindingMulti.M.C.S + +#58434 bitsegal comparison of oddly sized fields +primitive type ByteString58434 (18 * 8) end + +@test Base.datatype_isbitsegal(Tuple{ByteString58434}) == false +@test Base.datatype_haspadding(Tuple{ByteString58434}) == (length(Base.padding(Tuple{ByteString58434})) > 0) From 864aac0d0d0215ccfb97d0c97c2f914501069534 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 19 May 2025 18:40:10 +0200 Subject: [PATCH 286/662] improve the `unsafe_takestring` doc string (#58457) --- base/strings/string.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/base/strings/string.jl b/base/strings/string.jl index c2aa95690e99d..e7fbad01cf94d 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -78,8 +78,13 @@ function String(v::Vector{UInt8}) return str end -"Create a string re-using the memory, if possible. -Mutating or reading the memory after calling this function is undefined behaviour." +""" + unsafe_takestring(m::Memory{UInt8})::String + +Create a `String` from `m`, changing the interpretation of the contents of `m`. +This is done without copying, if possible. Thus, any access to `m` after +calling this function, either to read or to write, is undefined behaviour. +""" function unsafe_takestring(m::Memory{UInt8}) isempty(m) ? "" : ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), m, length(m)) end From 4bcd7c9ca3dfb54837eaf768c3f6b898e5420b31 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 19 May 2025 16:48:58 -0400 Subject: [PATCH 287/662] ir/verify: Give more correct errors in two places (#58443) This adjusts two error checks that were supposed to catch and give informative errors for out-of-bounds access, but instead fell through to a BoundsError instead. In one case, this was an off-by-one, in another a missing range check for the new nodes array. Not a correctness issue, because we do get the error, but let's make sure we get the correct one. --- Compiler/src/ssair/ir.jl | 2 +- Compiler/src/ssair/verify.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index 8a317d2d8cc0d..2af4c8d304592 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -810,7 +810,7 @@ end types(ir::Union{IRCode, IncrementalCompact}) = TypesView(ir) function getindex(compact::IncrementalCompact, ssa::SSAValue) - (1 ≤ ssa.id ≤ compact.result_idx) || throw(InvalidIRError()) + (1 ≤ ssa.id < compact.result_idx) || throw(InvalidIRError()) return compact.result[ssa.id] end diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index 2b8f89173911a..f9a9bb674a536 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -32,6 +32,10 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, if isa(op, SSAValue) op.id > 0 || @verify_error "Def ($(op.id)) is invalid in final IR" if op.id > length(ir.stmts) + if op.id - length(ir.stmts) > length(ir.new_nodes.info) + @verify_error "Def ($(op.id)) points to non-existent new node" + raise_error() + end def_bb = block_for_inst(ir.cfg, ir.new_nodes.info[op.id - length(ir.stmts)].pos) else def_bb = block_for_inst(ir.cfg, op.id) From 2e2fac5beafd5596cbb3933728c12bd141349c5e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 19 May 2025 16:49:16 -0400 Subject: [PATCH 288/662] avoid deadlock if crashing inside profile_wr_lock (#58452) The rd/wr lock distinction here was supposed to help prevent deadlocks by allowing recursion (even over signals), but did not account for crashes causing recursion while holding the wr lock. Make these lock acquires fail-able if they would cause deadlock. --- src/debuginfo.cpp | 18 +++++++++-------- src/julia_internal.h | 5 ++--- src/signal-handling.c | 45 ++++++++++++++++++++++++++++++++++++------- src/threading.c | 8 +++++--- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 53c07c978f72a..e114dccfe5772 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -145,8 +145,8 @@ struct unw_table_entry template static void jl_profile_atomic(T f) JL_NOTSAFEPOINT { - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); #ifndef _OS_WINDOWS_ sigset_t sset; sigset_t oset; @@ -157,7 +157,8 @@ static void jl_profile_atomic(T f) JL_NOTSAFEPOINT #ifndef _OS_WINDOWS_ pthread_sigmask(SIG_SETMASK, &oset, NULL); #endif - jl_unlock_profile_wr(); + if (havelock) + jl_unlock_profile_wr(); } @@ -464,8 +465,8 @@ static int lookup_pointer( // DWARFContext/DWARFUnit update some internal tables during these queries, so // a lock is needed. - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + if (!jl_lock_profile_wr()) + return lookup_pointer(object::SectionRef(), NULL, frames, pointer, slide, demangle, noInline); auto inlineInfo = context->getInliningInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); jl_unlock_profile_wr(); @@ -490,7 +491,8 @@ static int lookup_pointer( info = inlineInfo.getFrame(i); } else { - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; info = context->getLineInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); jl_unlock_profile_wr(); } @@ -1198,8 +1200,8 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT { int found = 0; - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + if (!jl_lock_profile_wr()) + return 0; if (symsize) *symsize = 0; diff --git a/src/julia_internal.h b/src/julia_internal.h index 938e87935578c..d4167d3e8a5b9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -210,10 +210,9 @@ JL_DLLEXPORT double jl_get_profile_peek_duration(void); JL_DLLEXPORT void jl_set_profile_peek_duration(double); JL_DLLEXPORT void jl_init_profile_lock(void); -JL_DLLEXPORT uintptr_t jl_lock_profile_rd_held(void) JL_NOTSAFEPOINT; -JL_DLLEXPORT void jl_lock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; +JL_DLLEXPORT int jl_lock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; JL_DLLEXPORT void jl_unlock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE; -JL_DLLEXPORT void jl_lock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; +JL_DLLEXPORT int jl_lock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; JL_DLLEXPORT void jl_unlock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE; void jl_with_stackwalk_lock(void (*f)(void*) JL_NOTSAFEPOINT, void *ctx) JL_NOTSAFEPOINT; diff --git a/src/signal-handling.c b/src/signal-handling.c index ff073cc82a0a5..6e19028ca7940 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -102,7 +102,7 @@ void jl_init_profile_lock(void) #endif } -uintptr_t jl_lock_profile_rd_held(void) +static uintptr_t jl_lock_profile_rd_held(void) JL_NOTSAFEPOINT { #ifndef _OS_WINDOWS_ return (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); @@ -111,38 +111,69 @@ uintptr_t jl_lock_profile_rd_held(void) #endif } -void jl_lock_profile(void) +int jl_lock_profile(void) { uintptr_t held = jl_lock_profile_rd_held(); - if (held++ == 0) + if (held == -1) + return 0; + if (held == 0) { + held = -1; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_rdlock(&debuginfo_asyncsafe); + held = 0; + } + held++; #ifndef _OS_WINDOWS_ pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); #else TlsSetValue(debuginfo_asyncsafe_held, (void*)held); #endif + return 1; } JL_DLLEXPORT void jl_unlock_profile(void) { uintptr_t held = jl_lock_profile_rd_held(); - assert(held); - if (--held == 0) - uv_rwlock_rdunlock(&debuginfo_asyncsafe); + assert(held && held != -1); + held--; #ifndef _OS_WINDOWS_ pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); #else TlsSetValue(debuginfo_asyncsafe_held, (void*)held); #endif + if (held == 0) + uv_rwlock_rdunlock(&debuginfo_asyncsafe); } -void jl_lock_profile_wr(void) +int jl_lock_profile_wr(void) { + uintptr_t held = jl_lock_profile_rd_held(); + if (held) + return 0; + held = -1; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_wrlock(&debuginfo_asyncsafe); + return 1; } void jl_unlock_profile_wr(void) { + uintptr_t held = jl_lock_profile_rd_held(); + assert(held == -1); + held = 0; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_wrunlock(&debuginfo_asyncsafe); } diff --git a/src/threading.c b/src/threading.c index 4256115214fc2..97b66d2e2068f 100644 --- a/src/threading.c +++ b/src/threading.c @@ -559,18 +559,20 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER // this here by blocking. This also synchronizes our read of `current_task` // (which is the flag we currently use to check the liveness state of a thread). #ifdef _OS_WINDOWS_ - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; #elif defined(JL_DISABLE_LIBUNWIND) // nothing #elif defined(__APPLE__) - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; #else pthread_mutex_lock(&in_signal_lock); #endif jl_atomic_store_relaxed(&ptls->current_task, NULL); // indicate dead // finally, release all of the locks we had grabbed #ifdef _OS_WINDOWS_ - jl_unlock_profile_wr(); + if (havelock) jl_unlock_profile_wr(); #elif defined(JL_DISABLE_LIBUNWIND) // nothing #elif defined(__APPLE__) From bb28ea8f56e1f1ad46d12d3d5cd5a1dd08caf1f0 Mon Sep 17 00:00:00 2001 From: Sebastian Stock <42280794+sostock@users.noreply.github.com> Date: Tue, 20 May 2025 14:05:15 +0200 Subject: [PATCH 289/662] Add notes that doc lookup requires REPL (#52065) Documentation lookup with `@doc`, `apropos`, and `Base.Docs.doc` is exported by `Base`, but the actual implementation lives in the REPL. Since the REPL is no longer included in the sysimg, these functions only work if the REPL stdlib is loaded explicitly. This PR adds compat admonitions in several places. --- base/docs/Docs.jl | 10 ++++++++++ doc/src/manual/documentation.md | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index ae3891e218824..13a5b35a115da 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -51,6 +51,10 @@ You can retrieve docs for functions, macros and other objects as follows: @doc @time @doc md"" +!!! compat "Julia 1.11" + In Julia 1.11 and newer, retrieving documentation with the `@doc` macro requires that + the `REPL` stdlib is loaded. + ## Functions & Methods Placing documentation before a method definition (e.g. `function foo() ...` or `foo() = ...`) will cause that specific method to be documented, as opposed to the whole function. Method @@ -787,6 +791,9 @@ When `pattern` is a string, case is ignored. Results are printed to `io`. ``` help?> "pattern" ``` + +!!! compat "Julia 1.11" + In Julia 1.11 and newer, `apropos` requires that the `REPL` stdlib is loaded. """ function apropos end @@ -797,6 +804,9 @@ Return all documentation that matches both `binding` and `sig`. If `getdoc` returns a non-`nothing` result on the value of the binding, then a dynamic docstring is returned instead of one based on the binding itself. + +!!! compat "Julia 1.11" + In Julia 1.11 and newer, `Docs.doc` requires that the `REPL` stdlib is loaded. """ function doc end diff --git a/doc/src/manual/documentation.md b/doc/src/manual/documentation.md index aa58e5b600f49..2c32b57fad38d 100644 --- a/doc/src/manual/documentation.md +++ b/doc/src/manual/documentation.md @@ -331,6 +331,10 @@ documentation between different versions of a function: @doc (@doc foo!) foo ``` +!!! compat "Julia 1.11" + In Julia 1.11 and newer, retrieving documentation with the `@doc` macro requires that + the `REPL` stdlib is loaded. + Or for use with Julia's metaprogramming functionality: ```julia From f4d17506bcc8cffa36cdef4a8c354a44cba83bb0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 20 May 2025 12:31:29 -0400 Subject: [PATCH 290/662] ir: Consider Argument a useref (#58467) A common usecase for `userefs` is to replace an embedded `Argument` by something else (e.g. for inlining like operations). This allows this to be written as a single `userefs` loop rather than adding an additional if case for `Argument`. --- Compiler/src/ssair/ir.jl | 4 ++-- Compiler/test/ssair.jl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index 2af4c8d304592..87c8e7f0bc00d 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -609,7 +609,7 @@ end elseif isa(stmt, EnterNode) op == 1 || throw(BoundsError()) stmt = EnterNode(stmt.catch_dest, v) - elseif isa(stmt, Union{AnySSAValue, GlobalRef}) + elseif isa(stmt, Union{AnySSAValue, Argument, GlobalRef}) op == 1 || throw(BoundsError()) stmt = v elseif isa(stmt, UpsilonNode) @@ -640,7 +640,7 @@ end function userefs(@nospecialize(x)) relevant = (isa(x, Expr) && is_relevant_expr(x)) || isa(x, GotoIfNot) || isa(x, ReturnNode) || isa(x, SSAValue) || isa(x, OldSSAValue) || isa(x, NewSSAValue) || - isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) || isa(x, EnterNode) + isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) || isa(x, EnterNode) || isa(x, Argument) return UseRefIterator(x, relevant) end diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 0ae7c99fc3a62..a855439877f1b 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -459,7 +459,7 @@ let @test stmt.cond === v elseif isa(stmt, ReturnNode) || isa(stmt, UpsilonNode) @test stmt.val === v - elseif isa(stmt, SSAValue) || isa(stmt, NewSSAValue) + elseif isa(stmt, SSAValue) || isa(stmt, NewSSAValue) || isa(stmt, Argument) @test stmt === v elseif isa(stmt, PiNode) @test stmt.val === v && stmt.typ === typeof(stmt) @@ -508,6 +508,7 @@ let GotoNode(5), SSAValue(7), NewSSAValue(9), + Argument(1), ReturnNode(SSAValue(11)), ] From 4d96cb43ab1efafa323256a545a63a7b51a2c32f Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 20 May 2025 15:53:25 -0500 Subject: [PATCH 291/662] Add some loading precompiles (#58473) --- contrib/generate_precompile.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index cc91f577959e9..20aa0079d749c 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -34,13 +34,22 @@ hardcoded_precompile_statements = """ precompile(Base.unsafe_string, (Ptr{UInt8},)) precompile(Base.unsafe_string, (Ptr{Int8},)) -# loading.jl +# loading.jl - without these each precompile worker would precompile these because they're hit before pkgimages are loaded precompile(Base.__require, (Module, Symbol)) precompile(Base.__require, (Base.PkgId,)) precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) precompile(Tuple{typeof(Base.Threads.atomic_add!), Base.Threads.Atomic{Int}, Int}) precompile(Tuple{typeof(Base.Threads.atomic_sub!), Base.Threads.Atomic{Int}, Int}) +precompile(Tuple{Type{Pair{A, B} where B where A}, Base.PkgId, UInt128}) +precompile(Tuple{typeof(Base.in!), Tuple{Module, String, UInt64, UInt32, Float64}, Base.Set{Any}}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Nothing}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Base.Compiler.VolatileInferenceResult}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Base.getindex), Type{Pair{Base.PkgId, UInt128}}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Vararg{Pair{Base.PkgId, UInt128}}}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}, Core.SimpleVector}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{Base.IncludeInto, RelocatableFolders.Path}) # LazyArtifacts (but more generally helpful) precompile(Tuple{Type{Base.Val{x} where x}, Module}) @@ -215,6 +224,7 @@ if Artifacts !== nothing """ hardcoded_precompile_statements *= """ precompile(Tuple{typeof(Artifacts._artifact_str), Module, String, Base.SubString{String}, String, Base.Dict{String, Any}, Base.SHA1, Base.BinaryPlatforms.Platform, Base.Val{Artifacts}}) + precompile(Tuple{typeof(Base.tryparse), Type{Base.BinaryPlatforms.Platform}, String}) """ end From f03e9c33c18afa73004c08901a0a8fa0c49926ac Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 20 May 2025 16:54:18 -0400 Subject: [PATCH 292/662] [deps] enable zstd support (#58344) --- Make.inc | 9 +- Makefile | 117 +++++++----- THIRDPARTY.md | 1 + contrib/refresh_checksums.mk | 2 +- deps/Makefile | 9 +- deps/checksums/llvm | 240 ++++++++++++------------- deps/checksums/zstd | 38 ++++ deps/csl.mk | 2 + deps/llvm.mk | 11 +- deps/llvm.version | 2 +- deps/p7zip.mk | 30 +++- deps/sanitizers.mk | 7 +- deps/tools/bb-install.mk | 4 +- deps/tools/common.mk | 2 +- deps/zstd.mk | 60 +++++++ deps/zstd.version | 8 + julia.spdx.json | 19 +- src/Makefile | 12 +- src/debuginfo.cpp | 20 ++- src/llvm-Compression.cpp | 245 ++++++++++++++++++++++++++ src/llvm-Compression.h | 136 ++++++++++++++ stdlib/Makefile | 2 +- stdlib/Manifest.toml | 9 +- stdlib/Zstd_jll/Project.toml | 15 ++ stdlib/Zstd_jll/src/Zstd_jll.jl | 73 ++++++++ stdlib/Zstd_jll/test/runtests.jl | 7 + stdlib/libLLVM_jll/Project.toml | 8 +- stdlib/libLLVM_jll/src/libLLVM_jll.jl | 2 +- stdlib/p7zip_jll/src/p7zip_jll.jl | 67 ++----- stdlib/stdlib.mk | 2 +- 30 files changed, 919 insertions(+), 240 deletions(-) create mode 100644 deps/checksums/zstd create mode 100644 deps/zstd.mk create mode 100644 deps/zstd.version create mode 100644 src/llvm-Compression.cpp create mode 100644 src/llvm-Compression.h create mode 100644 stdlib/Zstd_jll/Project.toml create mode 100644 stdlib/Zstd_jll/src/Zstd_jll.jl create mode 100644 stdlib/Zstd_jll/test/runtests.jl diff --git a/Make.inc b/Make.inc index da7dfcdd35a05..f14714044b6fb 100644 --- a/Make.inc +++ b/Make.inc @@ -59,6 +59,7 @@ USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 USE_SYSTEM_LIBWHICH:=0 USE_SYSTEM_ZLIB:=0 +USE_SYSTEM_ZSTD:=0 USE_SYSTEM_P7ZIP:=0 USE_SYSTEM_LLD:=0 @@ -373,11 +374,15 @@ $(1)_rel_eval = $(call rel_path,$(2),$($(1))) $(1)_rel = $$(call hit_cache,$(1)_rel_eval) endef $(foreach D,libdir private_libdir datarootdir libexecdir private_libexecdir docdir sysconfdir includedir,$(eval $(call cache_rel_path,$(D),$(bindir)))) -$(foreach D,build_libdir build_private_libdir,$(eval $(call cache_rel_path,$(D),$(build_bindir)))) +$(foreach D,build_libdir build_private_libdir ,$(eval $(call cache_rel_path,$(D),$(build_bindir)))) # Save a special one: reverse_private_libdir_rel: usually just `../`, but good to be general: reverse_private_libdir_rel_eval = $(call rel_path,$(private_libdir),$(libdir)) reverse_private_libdir_rel = $(call hit_cache,reverse_private_libdir_rel_eval) +reverse_private_libexecdir_rel_eval = $(call rel_path,$(private_libexecdir),$(private_libdir)) +reverse_private_libexecdir_rel = $(call hit_cache,reverse_private_libexecdir_rel_eval) +reverse_build_private_libexecdir_rel_eval = $(call rel_path,$(build_private_libexecdir),$(build_libdir)) +reverse_build_private_libexecdir_rel = $(call hit_cache,reverse_build_private_libexecdir_rel_eval) INSTALL_F := $(JULIAHOME)/contrib/install.sh 644 INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 @@ -1417,7 +1422,7 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\. # Note: we explicitly _do not_ define `CSL` here, since it requires some more # advanced techniques to decide whether it should be installed from a BB source # or not. See `deps/csl.mk` for more detail. -BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT BOLT +BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT BOLT ifeq (${USE_THIRD_PARTY_GC},mmtk) BB_PROJECTS += MMTK_JULIA diff --git a/Makefile b/Makefile index 7df749bd74541..b3aeb4fb56286 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" all: debug release # sort is used to remove potential duplicates -DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir)) +DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_private_libexecdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/gcext test/llvmpasses) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk $(BUILDROOT)/pkgimage.mk @@ -203,6 +203,12 @@ endif # private libraries, that are installed in $(prefix)/lib/julia JL_PRIVATE_LIBS-0 := libccalltest libccalllazyfoo libccalllazybar libllvmcalltest +JL_PRIVATE_LIBS-1 := # libraries from USE_SYSTEM=1 +JL_PRIVATE_EXES := 7z +ifeq ($(OS),WINNT) +JL_PRIVATE_EXES += 7z.dll +endif +JL_PRIVATE_TOOLS := ifeq ($(JULIA_BUILD_MODE),release) JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen else ifeq ($(JULIA_BUILD_MODE),debug) @@ -232,9 +238,12 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib else JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz endif +JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libzstd +JL_PRIVATE_EXES += zstd$(EXE) zstdmt$(EXE) ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME) endif +JL_PRIVATE_TOOLS += lld$(EXE) dsymutil$(EXE) JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBUNWIND) += libunwind ifeq ($(USE_SYSTEM_LIBM),0) @@ -313,37 +322,33 @@ install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index $(INSTALL_M) $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) $(DESTDIR)$(bindir)/ ifeq ($(OS),WINNT) - -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ + $(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ ifeq ($(JULIA_BUILD_MODE),release) - -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ else ifeq ($(JULIA_BUILD_MODE),debug) - -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif - -$(INSTALL_M) $(wildcard $(build_private_libdir)/*.a) $(DESTDIR)$(private_libdir)/ - -rm -f $(DESTDIR)$(private_libdir)/sys-o.a + $(INSTALL_M) $(filter-out %-bc.a %-o.a,$(wildcard $(build_private_libdir)/lib*.a)) $(DESTDIR)$(private_libdir)/ - # We have a single exception; we want 7z.dll to live in private_libexecdir, - # not bindir, so that 7z.exe can find it. - -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(private_libexecdir)/ - -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(OS),Darwin) ifeq ($(JULIA_BUILD_MODE),release) - -cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) - -cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) + cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) else ifeq ($(JULIA_BUILD_MODE),debug) - -cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) - -cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) + cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) endif endif @@ -351,7 +356,7 @@ endif for suffix in $(JL_TARGETS) ; do \ for lib in $(build_libdir)/lib$${suffix}.*$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(libdir) || exit 1; \ fi \ done \ done @@ -371,26 +376,24 @@ endif for suffix in $(JL_PRIVATE_LIBS-0) ; do \ for lib in $(build_libdir)/$${suffix}.*$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \ fi \ done \ done for suffix in $(JL_PRIVATE_LIBS-1) ; do \ for lib in $(build_private_libdir)/$${suffix}.$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \ fi \ done \ done endif - # Install `7z` into private_libexecdir - $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(private_libexecdir)/ - - # Install `lld` into private_libexecdir - $(INSTALL_M) $(build_depsbindir)/lld$(EXE) $(DESTDIR)$(private_libexecdir)/ - - # Install `dsymutil` into private_libexecdir/ - $(INSTALL_M) $(build_depsbindir)/dsymutil$(EXE) $(DESTDIR)$(private_libexecdir)/ + for exe in $(JL_PRIVATE_EXES) ; do \ + $(INSTALL_M) $(build_private_libexecdir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \ + done + for exe in $(JL_PRIVATE_TOOLS) ; do \ + $(INSTALL_M) $(build_depsbindir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \ + done # Copy public headers cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia @@ -442,13 +445,13 @@ ifneq ($(private_libdir_rel),$(build_private_libdir_rel)) ifeq ($(OS), Darwin) ifneq ($(DARWIN_FRAMEWORK),1) for j in $(JL_TARGETS) ; do \ - install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j; \ - install_name_tool -add_rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j; \ + install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \ + install_name_tool -rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \ done endif else ifneq (,$(findstring $(OS),Linux FreeBSD)) for j in $(JL_TARGETS) ; do \ - $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j || exit 1; \ done endif @@ -471,11 +474,11 @@ endif ifeq ($(OS), Darwin) ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(JULIA_BUILD_MODE),release) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT) else ifeq ($(JULIA_BUILD_MODE),debug) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) endif endif else ifneq (,$(findstring $(OS),Linux FreeBSD)) @@ -486,11 +489,43 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) endif +endif + +ifeq ($(OS), Darwin) +ifneq ($(DARWIN_FRAMEWORK),1) + for j in $(JL_PRIVATE_TOOLS) ; do \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_private_libexecdir_rel) $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in $(JL_PRIVATE_TOOLS) ; do \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_private_libexecdir_rel)' $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif + +ifneq ($(reverse_private_libexecdir_rel),$(reverse_build_private_libexecdir_rel)) +ifeq ($(OS), Darwin) +ifneq ($(DARWIN_FRAMEWORK),1) + for j in $(JL_PRIVATE_EXES) ; do \ + [ $$j = 7z ] && continue; \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @executable_path/$(reverse_private_libexecdir_rel) $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in $(JL_PRIVATE_EXES) ; do \ + [ $$j = 7z ] && continue; \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_private_libexecdir_rel)' $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif endif # Fix rpaths for dependencies. This should be fixed in BinaryBuilder later. ifeq ($(OS), Linux) - -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT) + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT) endif ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS)) # Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS) @@ -511,7 +546,7 @@ ifeq ($(OS),FreeBSD) # don't set libgfortran's RPATH, it won't be able to find its friends on systems # that don't have the exact GCC port installed used for the build. for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \ - $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib || exit 1; \ done endif diff --git a/THIRDPARTY.md b/THIRDPARTY.md index f3f59ca4ff3f7..06973b0163e5e 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -67,6 +67,7 @@ Julia bundles the following external programs and libraries: - [7-Zip](https://www.7-zip.org/license.txt) - [ZLIB](https://zlib.net/zlib_license.html) +- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE) On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index 9b373e0d7ab62..77921858f2b6e 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia +BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps diff --git a/deps/Makefile b/deps/Makefile index 4ad4df9ec4ba4..303520fb45e80 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -72,10 +72,12 @@ endif endif endif +PATCHELF_MANIFEST := ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) ifeq ($(USE_SYSTEM_PATCHELF), 0) DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf +PATCHELF_MANIFEST:=$(build_prefix)/manifest/patchelf else PATCHELF:=patchelf endif @@ -157,6 +159,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) DEP_LIBS += zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +DEP_LIBS += zstd +endif + ifeq ($(USE_SYSTEM_P7ZIP), 0) DEP_LIBS += p7zip endif @@ -204,7 +210,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS) # list all targets DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \ openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \ - objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \ + objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \ sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \ terminfo mmtk_julia DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL) @@ -256,6 +262,7 @@ include $(SRCDIR)/openblas.mk include $(SRCDIR)/utf8proc.mk include $(SRCDIR)/libsuitesparse.mk include $(SRCDIR)/zlib.mk +include $(SRCDIR)/zstd.mk include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk diff --git a/deps/checksums/llvm b/deps/checksums/llvm index efefff6d50a97..552904b4cc204 100644 --- a/deps/checksums/llvm +++ b/deps/checksums/llvm @@ -118,126 +118,126 @@ LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/b6619 LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/17c9e77e907f40e799da4b2d21cb36e6e10bde6ceda6ff29c5941e5f671c8625401f6079f3be0de710b57475f4e7beccc1e7e64824c334286efad7b21919a7a8 LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/122ab2567ca9d8824f94299b45e8b1f3 LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/78420eaba62e18541443f4b1d846fecaf95b58e93ee7cc08c4818339d3a0705b0dd82dadae2024f6be65b3b2e26011269e319eaa9739af599b640486031b60ce -libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/030e6925d110a3e1e16e21b695aa2901 -libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/d1c083099ade0186a4efeec1ee402aadf511dd2d01ec79896446870e23f71e16a32a80623c487eadf4e4ba659a02fe8044af27de3011ede133ea891159fb686d -libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/4d29fe04a66e1c5b90396d2122f3a6e7 -libLLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/b1853885245f907985e51fa3c2052f2b9e7add0af722812f9f33175c3e6b5addb813283cc46b2e81baf32083a6f412c6cde5261b9dba8a54e2a37edbab236c32 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/8d049c7597a0cae940a28153584084c8 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/35bbff689a87dbf7eaed2c2c98e90ed448acb6ef518bb23501ed05a67de3df53978dc28104b1d2607786486466905dc399de9e858b141f394510bb72876fcad6 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/42bc10fcf6c6626e839e59e9688f2fad -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/74d95c89fa487f3f46191094d807d1fd258652548969492a0b37f2290a82eaea6dae2fffa77af1c5be5b80bef525e7a5601c432ce308bbd874e063fbf75d5db4 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/69331507403bcd85db7e0d3d4a0d7a2e -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/3f2d96f8beb7d95eb1485c00f047a5050e7994d12ae16422943ed9a4dc9530c949622539532af5db1e655fafb9d1183e3e878a9608f12a6ef9a6413498249c41 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/3ad71df03561774b42394270a7db1797 -libLLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/2b11da0f3f94e29d14a0bdb7364e61057f9ed1f82b8126a46f0c0f98888a4173bbe46f64d404b71e2a08056f2b550ac4f3188409e456506ee1fcf1367af416c8 -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/3ad1cdf720b752bf5d37cc20c442ae66 -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/e177c5f3bee869e2975849e2ebf2d139614774dbeaea95f423956afe9b86af9cdaeee13a03a42cd439402a5bf96fa0df67ffd54ad7f5a645fb9ebe837c9f011c -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/7e2990489b986c4e1dc411d3034420fe -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/bfa1fcec41c2dd985222d6dd9995d2ca7708d314e5e7fc7085022a47ac32a6322edd3b415026d82d761082c4716c23600c262906ac1f283cb0209a15c549b372 -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/60d9336b463cf0d953733b072f58ed7f -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/77da5e8ae89e3e943e9af4e24388bae4e1ec1f335b4e131abf678c44d641d447769516b4eff1f2908797b80fef12beaae3fc963c408d8cf95febe6e1c4cca130 -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/6836377f50152632a26b45bf3f5dfa7a -libLLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/7bdca756e9fd87b600320fb82c6f9cd60afe5dd534e136e25dfc6a144026f7b987233e2e33d336bfcfc94fbfd39b21e91ea9eeb5a0c63468c4037998c3c6ee33 -libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/2ced7dada1583104c0eda06ad59edb4f -libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/d7ffb0627e3012b4dab12418d105fe431530df8334a787304d31e74cd29094d5df04e17ec333e292eaaaa2c8b210db28f7a368df769a3af8d31c0c2908a88ec4 -libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/bdbfab4aecfa5519e4dacb4d2bda09d8 -libLLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/405c1d62d5fcf59cbfb02382ba7827089b6cce600b04df670781945f8e1e8c9bc9814f4dca55b956f73c068689a2f50d558453d05a0d4bdccba99e71aaa7261e -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/fd148b8b11498c0e6f1587355ac29f0c -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/f24dde988c7d9f42a5baf915e813d93e7f16810067b6d12967d34fcba18f06f7dde2375b093ba18a145ad82d6757a1a853bb3a1c68009ff59b4edafa58eb30a4 -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/f07035563a248217e9447e7dbd8f1e3f -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/e1f02f575cc977d13c9f093a621dfc11ea9c73cff2c0c2e52598a644866096885dcac3c0944bb7ea7fad4ddfc7ec588b850c20ba464c8ec193bfb05bdcd18d6a -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/03b30ad82bf9cf3b3ffb454452a6f147 -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1349100776b182949f9ef65fa23484dbd2dedcd51aad1e02eecc47c7d7fe556c43e2188c914d5a7d409e1d29eea07709d6d76718c7dcaab4fa8ca3d3866fbbf3 -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/57e3da5f04c49d0fb1bc74613aa9102c -libLLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/09f34661f8a27b2bf6f800640beff3ff4170e18f13f35ce97b6e809a653744ec59d08b6d90f38e2e5b0cdcd7157ee1541b96c270ad6aedb8578036fd63dccdf7 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/8a0b721da8e29b9a3f8c6430f6316df2 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/166fd97bea310c2e87c3b799693532d8f1ce8625889fa0201f114094140509d5e570fff87db6d216aa38c331d5f06c9de81b7235cd5b98c4404e3c7320beaf72 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/8a052f798dce9b93618890010cd1e06b -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/4d070d56d425788ec20aa0c258c845bdc7a42078ced5501aaa7bcae906b2eb43eac97fdf52e4ee955e1dfd24fa5c95915717d501977fd2d98ea82e04e708422c -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/69e67d41dd400119d67e632a63b8b852 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/bef7ec7fd19b205c1fb0fed09a82c7362cfebdd4e62005195f9c73ee3b6e31c4216e23f50328fde2ff96f26dd55d8dacf5c9f3e70270bb0c13a73f08b6ed65b1 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/5b6a1ef8bb57be87f2067ef9816ea782 -libLLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/e75ca64872b5b4794c0652f07f8494c83cc52545a7de3d7e7da3c2290a7c064006c434f82f979e7df5abb6a75f9612478244c6048fb135d1d2f1f3f28328e689 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/869483bfcd96dfebc20b468e5c4eec12 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/3f129d028f8642fa1b8218d9eaa7122c305b9f01e66fed9a0443575d7acdd4f23e45da673e5df37260911797c373d8daac7b37a40667b4907dabebccb0ec92d3 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/d01c44f20732eca87461aaba9bf90f92 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/193bc6c8e3bd6eb512c29da873f63777da9e9d6ddc0e96094c5828bc6048b8da96517a17989505d1ebc43bb018fc034af94c9c66a337e18e86a34809c5470a08 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/47c494ba09d7fea196aeeae3e13de6a3 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0beaf19797f536a1083a277c3b655972337d594315790722e7c970ce7ef8974479cd77d92d0b0e403139cd4b2ba33885c2604e6ab5e07d2ef15c682353829466 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/3257fb989b619b3fe07ae196f2b84ce3 -libLLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/fc93031f73fbe7883b318e1a26b85a354f93cbb1f32befb5399c0d89bb4dd93a36de6839d15465c00adbb5dc3fe29926adf9b390b0122ec34afa99b64bcce60b -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/0e17af3bf23a8626ff758e40ecf9d83c -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/8a57ff2efb34363e6a9698932f4b1265067ea9abbea3ec783ccce23cf3e6e3e81e31a5a01d7382487e3709124612d02b5b6e2ca5448add1e583d25bc62c8bf1e -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/a9c8dd8a055704b192980b295f5f8c69 -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/59a2af6f5970b0a1836734f336acd55ccc8f16eb1d894e0c97c7577a0a5d35d675f2138fe736110c1746ab63d0e10238d975538c98c4fe686b64dbc2f717d7e0 -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/23afe5ea859954e5672636a743fc1f6a -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/2c07b81805b2edd7ac86f0a63cadb607a4f854945828e6daea55e04b9fa2491ca41259b67e46fee0ed7309fdeab9921c27cddc97c5aa1bde41e739b80afb19d8 -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/6d6c815b0c1955148c800d17d1e9a5ef -libLLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/d4d57e0aac6c813bfec09ff4c28e39e474194b0c6ab1cc11dcdae3654c51dc9a729aac8297715563bcc3fb89502dd216b66a7d35eef76bff93effe8974b191ea -libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/6bcf33d477fe67e86e8190aeb29da648 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/dfd44f521064472ddba0f123820e6ad931e3a5420531c71c1ea165cef4d6355855cdba56b1ee4543b5e15d0e516c38e3a15e7e18b27ff5aefb4abb476faf7cf3 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/51ca42bcd058e87cd9bb146c1e705d34 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/fde60b8eddd37cec6e1dc3445c11b0ec8dc37f3638e99b209b55ff28d92b557b797a571d98ecae79e71d629f1be33edd4338df83da9379dab578fb7310ce0dd5 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/abb40a36c1a3d77f3f2eae4d88729a81 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1e48ac8aa697ed3827a9cef46a008bdf609f9e2aaf66981d760c5baff9a8d0d63eb2b6600544c8b8b6243fa62f5832a8fc1a9d77602c135336a4ff2bb570b170 -libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/c9bff27fe92407e5907852363c3eb6db -libLLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/ca0df65a0f2512984b264fe44c9edea65ee6e92a6bcb2390259da40d4d6a9b5c58320459693b73822da23ffe47fbab72b1919129336b63dc03d26530e801c1de -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/26c401ef44560569334008b754391973 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/dc5eb72deff4ba26e838dcc831bff32053c967450c8d0bb95ae4a25e73c864b6810f580d4f341e6aa47d0fa91892ff92aa4f45504415e70d958c466fec81e062 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/d5934a0ec319c4dd3d1043855284ba80 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/dff3bfb91daa61555e91a3120923cfa7642dd856c0d13c988273d4b135a419d22fa87c39d7552594077e05659c3958c8bf501e171d671bd395d72b4d0e5e99f5 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/f6734f3af26b223c67461753cf3711c7 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/ca807029d5f3d03b943bb2b18c43b17e8a776f30599ef65706f3ee71f21ab5d17225527ad3c7d48bd52ce2b4c36a56ef6424c68530fc9f414e643df257cbee99 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/5ce381da456a1f3b76b46c4d1af016e6 -libLLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/23a52fe4084a810a2afaa86d521041c493f6e010724b62d8a5cb206e82f02bd8498299716f634e5cf4359c3e59c1702aadaa48f3d9def41193c037cb12d6cfb8 -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/9a19b59e71c055a79d1b07d560c6325c -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/628adfe605ca296280cd2fa6771235bbc993e52c6fd5276441f1a0c715c14945865ee78e177afbc2f60e156db7349fe2aeccf656d8754cc8f8891de8b24713ef -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/8e8ffc26951228c6b8c4e4df786a6402 -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/378f60f7a8fdfdb51177f10730ec7d3a6afadb4466c2d2c5974580e1748dece6e07300b445db0f47e2a2067f73770d5da0b5fef443150e2987e38c27bc99ec7d -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/80f50299632de127179fa6a3cc907a9e -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/5b0855c283503d86c110bc37adaa2bd465be691517dbea08804b24051ecb3ab59921ac0e4c1e766d1d46a33a18f42bc87e5928765412caaa29cb8772995a5ee0 -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/c2c727419798d4bb3ccd32ebb81d1a2d -libLLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/8d6781c532b8c60c5f2c961cb076f4b303116753abecc1ef1656eb7ccfcad0c553cfbe52a0e10dd449a302d4383917f80ad79de6b39de6ee4ee865ecc94bc766 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/ea09093c412181d9e32a12406ede691d -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/d3bb655c4a276a2d16c0f4bbec67fb5cb252bb12850e57b302af7904e3b8bd708ef96d13d7b3b392e2ff63701b5abd8b50f3b7392442567188ec73a068c3d53f -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/740da054d26d889df5f898019c2f3c45 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/839d613cbca89fbbcfe8d4a5194b255b1376fdf6054c74189a62b0de0a4e0476cfed8e077b4c1b168c9d6ff8fe6a84a50912e6c19389aead718fffdf879868e3 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/53c1bf2aee12c342d0c7f7f26cae75a0 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/87778a19952829547ca0cfcc6d0acdf007858dc387d4e00b569a89ae2094e860cb64f4976ab393432d3bf80c184aebd007a26d4f1651aac9b20a1ff1f0c70da5 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/54990164de5d5284c221725ee492cd77 -libLLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/5442a878cda167e15c0994c6ecfde1cfc2cd94bbe9b27471f16dd2bf442e90e5e402536fb007b816800cdab7071b704a3fefe66ae6a38f6571e8f7a898ea494e -libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/dfcf38ade854f883ad0d453fbfcb146a -libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/9e6a9ccd8a2d397bbd0ff37db7044a476e1df136a7b79ba145bd767e8d1af7f902a7a6cc900592488268d331635ca84691f2d7f68c64d7bf806d3b9ddeb8f550 -libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/10ab73c8c0754f8bd4d6535fc540dd5c -libLLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/4a16cdb2258f013f8e149b435d6e86393c2ccda8729b95de7744109addb2e229b4b52b3128a4324f71d3e4ae4a616daf46f4fead9a5f445a5aaef8f8491e269a -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/67d4b8f59f623665f380fe5b9644bdc5 -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/05db739257d31805bc5d4577e8c0c35dcccce5fb7b7581001758fb596cb528d7ab376cffea53fac4f16adc7d9a941d56fd856095bca5c6b86a2c355303e81b3b -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/21da228049c12e06b50fb4b3caa2fcb0 -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/9f93640383d8064c45c101d87b99b4df3096a78a71bf89d40ade592fc3bd1d7235d01681a68bb14b5191a79aa06361c242bab3f31d9c5053a685759b2e522719 -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a31c99673c8d6e3be00f442b8f97e4a6 -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/0004186ad8d28e18e112e548fb291151f6efa74140cebdad92afd34c46425a61ffb33969495391387aacfa812b927f9784aac55c00c0d57d76e8c0ac3063616a -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/eea6f4fb592967f4c137007ad8693852 -libLLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/7ed4d32d5ba2f8582cba63ab992c3dc6c1c0002a533e437696b6cae2ac18f4867fe4d897e0c5ede64f58d88b82a87b355d1699e24171f2e1b84e1ec3d0e3a738 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/ea6487b87d7929e4de8a7ef858708ce4 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/8e4a6ecf3827d62482e8d0b0163f58057c18c5d9ac7c05c25db0e1f9f15cf1c0ba9ccc1acf2caa686792e5dad9ee5bceae0051fcbb17fbcb6bbfa79acb509c5c -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/b85c25315f0f7d5ad800ba822a48bf51 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/9965dc687164c11ea9d30ebb4bc0bccb5a9f9ac220b94a98e1088a938b25864e6321100efcb211816b89713081e4cde89f2ab2dd7030cb297cb7e3ad49b69be4 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/63a75f6c13445552424993b84658f3b4 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/1f07edd4d3456da3c3970cdf0d4892458bf6b12d004a1f823e56cd7344020648c1486bbfe02d4c0ecebb6ce32ebe6c124375ac3f9c13b618d0e6bf36aed3c84b -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/d9d3903abc3266d960e50f680b7497b6 -libLLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/dd570e511be6e63f29f2864fb620db75e489722c0bd41750164ab7689cbeb0a4d203634f457bf4c65e023cf7b33b2a071d78fe8909f8050102bed6d396da4017 -libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/1c389de1de869c3744bad21da4844cd8 -libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/6cda6f023f3bfd6f9330efe61405bd3ddc062e72918a95bb6d9737f54b1dab04def0736abb52a983669b7059744bc9ab7d030feac29c5335d82a42c9baea27cf -libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/e6504fb9ea87afc61baf598024f677b0 -libLLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/ea974e940095ec65d4d5f71fb0050651c3824bc18a794f47a83a22ac419d30fe2cc404ac352087169153274caf32f3014207f43e2afa6bef3e86c3d5867b006c -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/112367628e6c3fcccca6b1630f37b6cd -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/949be09b5d33dfd3a43cde05cb1d009bbbb3c5515e7bade0249398ce6dfe5fb678c0451f6c7bb109e94c9ca6bc5c07f2f763cdea4231ee00bb911749f4c78e9b -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/8057b884d6de98245e819463042d4e96 -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/ead00174a7a609900289a1c3fcb9a18bf7945a857292df7ab716bff42eca7cd83f9ccfec90a8605e01070407dbd59f8d638fd29203c62e5179c774eead5900c3 -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/715471fd07647a7198c9cc4a6e5cce15 -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/807c97e4788b958eb158579a537381831ad3f357be17e12a38acb0346dc8a7404afd0956a31d1d7d2ffcf7c641bbaf5e5712ab7df8ad2d7ebf32f4d18ee23095 -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/5d13196d2d8c5adaef257272e0f317f1 -libLLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/8162eee065749468457d5672011640b3cbf279288ee67647f439c03129a572077b3503becba6d43657d1fed0e685d832b481fee4b2bca66a1e731a7ddb630d9c +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/291723bdec338b312764cc6e33a76007 +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/105282bd4f63d3c6fbf4e45f5a72a52f1a6154e46e6ba595c70aeb763afe27dea688884e6448d4ef502cc4598bb377f10f172747cfc05b42495ba7ab4222307c +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/289aede4d44649a0d265abba59041ec4 +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/76f215d8e45bff747109a6aac5338ac09c5c89ab4af9b82c1575bc96908ba9a4675fc18a034bb222435a362f244b69abfb4282d0bc0c71f84ee9f715f281ec7f +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/267645d41e71d3f3a5a19cc70c63e573 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/14d79aa893acca711baa5a20354bedec51e8c77a90f52639279d9b4453efb670b647dd0d1d60820b9c49033e848f90efdc25e87cb5aa65a9ac35aa8f6f71ac54 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/2f03e683eb1631503e28117ffd6b0bfd +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/d0fafe8643cf0e1b702d4022fcc3c4a0c35389a65cca9490770d1a053a9ad8102dbb51e8a01fea46ef47d222074fefac66fdfdd4a34c2805a7761f43281acc0d +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/7729756ba2186da8578b5247e1ece99c +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/134058fecbdb9150435f1b3a9c6d208838562e0af8b3aac5c519502644b37b0d2093c2171a21ebd653ace9d51ca9bd4825343a60d80678496a5ccf876c951b71 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/55e222b181a94df519477c3d9450ef1f +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6958f4972db0871b6005d24a8dc426b86504aa604a75a3cb18472dc4000477284e5c76c9477490252b521e40b17b9f587a7c6c3657e02e562858cbaae601260f +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/70deee22fe83f5f7234e5b2849579522 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/24e956e1fc8431838d2f4d6b1d12bd90250e30acef64bda2330e542753ea7c60f7233fed0110a6166cd7510d65a0512b0115edead427924e5509f3443632716a +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/1082d39fd25ed971e1ed3f43882cdf77 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/2fde64b294b1144af2754a543b5a9f63d3d161fe7de5c50061160ac845e4b80f8c447618a90768cc0a239546196a67e89f1a3295bc9cce6fd3ece36749cab871 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/279ab853b812c2ae99aefbd301699e49 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/f1f6ff89876bef3d2da28fcb230399378bab1df532a2ee31b83ae4cde8e5689e1bff2d44bb87b61c649055823fa8477fe8bb3445c8c8ed12a0b6cddffba5f26b +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1ba2e81cc47c6600a25a507286bc74ae +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/f8e88b9064b14e7565063d93630f67da9a927945e12b4234fce2335fcbc4c365528273058fde86cd046e2b63e0edc666641d5db6123a94865e1450f298468966 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/b722ae106708f785689f294f061105c8 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/82bda3b9668775da095c9078680042926e76c30ec3c18c893d16319cc740b7cc8e2f52f3a0c2f8f59ee84637be0c80bb11eefde531d5ae88d1d51d14f387d0b0 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/ee1c39eb3fd4ab6b6cf81dec58cd0f1e +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/4500d7b795dace2ab6c9279fd62c9a68376fc9c1cb000450dc63547d8c7f2b4a41f523130ab824923ad73efe890fe5497a486748d9efa6feb523b29640693b88 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/87a0be634e7f789b04aab50580b8c670 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/442fa7f7702c2c6e80682d943bef8df3fd4867e0f4d871a3bbb9ad015b53fce4c1f9d38f8fe80a7805d3955060031efe569c0e909ea326969923dcf923180a86 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/e50d255bfbec40a6d4e97cc810aa4497 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/9108628863586f4ff4099d04b45aea75ca51d6540191d4471910a32f2ccb89f4fda388968ee7011932affbfc1cf8d33a02e28e81d572ecdc6c96f2e2fe982a62 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88260fa90d1a3f9bf0b3f7fe37af52ba +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/b2455517b2c5a698b465b04305b547352e641ce96079d42542278d3b4f92ca9e181585ab6fa42f2bb4b5407b4c72249af5a5fab131efa74fc79a043d975cc102 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/78cf5b861f6f159ca48335b6a927e759 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b11274b7ae94edba3e12004df5a95d24004d9ef81e5af7037741caeaa44b3d27e300565ba82d120f4943ed1f401f5d306318cf20bef6cecba80344c71881456b +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7f87487fce6b663644013178394c8194 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/6bceaa61a716c0950b176141eb838cd809e46a1ca79ec278f43b8d44bbe1f6cd22efbf2049ffda7bdcc521a5407cbe813aa07408e929a50bed5bfc0e19907391 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e9fc6ba0213cc88f15282a946779dddb +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/ed893995aadc5cbf611bfd18e9e1a06d4a82da69414529f88ef1facffa3c1adeaf9952b9a75e1b9e40b15141b9fe1eee33f9fdf6e84145b51f5da1c0751d83fa +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/43a4ed1c5818dcf25f1b828f86ae2959 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/f834da31818afff29ff02a305569a3e9928789fd749e5c18a5df1d8173768ea9803c94ed73e0b362cb9111d0822794d084a4285a4ac4d8ce8bceea6a4c276e51 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/7aa67860d888970dcd8c441a8995e6a8 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/265f9ef484b09fb38929a9d713cf17ce295796da64eb9bb3c56fc5eb17159d730fcfcfca29b4555dbe39345216ecc12d22f0808b0d855e96fa42f46e4020978a +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/d59c841e3d0129e8e036282fcdb25a48 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/88753f91f8c488ef851f2bf4548f3e4b25053ecbeaa950b24b334bf9f9ab78152d99cb4b1e88831e257990d1e8f750405f5b185fec61c81a87f6dab6e5b451da +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/7f1dab964b8d008377f42db85e200ba1 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/b407984e1b4d11d1bc3db3d094697972f34790466caa9fdc70b85ffd5b823e08b2335eb4cac1d7e67d1fe74cdc7ef29ead9a9e39f69d8cf55312d560ebd7c9e7 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/29b4d1de6645ead45db6b9f17f4fc09b +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/509e2dcd9c5d2262fff70ceeae54f6d67995bb7cec316e199fc8c7032c74ea8896ffa3431245a33c28f3f846ef267a21c5f798b2ec8f761e6b3b4af4d4412899 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9f079ae058a98362d0ee50d938a3125c +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7452b65b750ffdf596fd0cb0740b459e18b23586b833f296f4db23cc5b69a72f6744ae85a7c5e1bf77a07563addcb17480b6feab5d2314dc74feb6c80b398978 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/2bebe715afc61928505ee08cd8ed3e9b +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/c5ae5c79ef4d73d8a9d8bc99d0d418d1f0e7503c7a9e90f6e40b51bece4a33a077acd1e91d1ea355361d9911089179f93e1a5068532646cede01fa6941100d65 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/b6db0d56617db57ee07a90ea4f778b32 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/3aa62ee8df72ff360a660eb9b282f00c1690c181f739757423df618e224bfa8601e93eaf4690ab7527fa6097f1c0663cbfec7f0616ba897943a4a95bbc2d4abd +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/8d945748dd65b28134943e6f8b101dd4 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/85371381137581a804b96c4864b6bda27aab0bad39626f2e33bc69bdf1ace5ccfac9a2e2255c1ddc09fcc67aa90b25be27fe8f88bca9e0e4a0bca284cb653473 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2628e4c8be69d729c063a91a7db8493b +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/8dba6382ef1faee5fd80729e0754f288397a8dd20eb540536edbd952be6d43287aaebdb22576ffa4879a76577f275ebde9c87fc67af1064fcef815cc7b59f8f1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/d8e406a6745deac26a671c9d4c567ee1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6e615de0bd6e3020d232af587aa09d380e7d0538daaade75ee06d99851e2d899c926ce2f7316bdad7b1e4de985a99cb0b7bc37cdea4c365156346896aa256197 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/f09676e4ee1c2c18b8906ccdc7fe1833 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5aabe655e54b1045f94327f6358003ea13c435c313dd75c9c0a870a76fe47416d86d6f768fda3e4b69ff7c3040eebdfd8b7ed43edeaf45ec2e4adf438b10005f +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a7d246bcd0fa0d1425e1a405df00ef3c +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/b27be72f5269096bfde5c26f15eed7285137dd83b802d8129b35557caf4e575619e66cef31a758323b20336d3003e11320b60e3188e4826c6802dc062b236fef +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/7a4a7852ce434bdb89a202c8ea8079a1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b3e6c6337fa388a095e51d03ec235964cd92de809db8269f14f346c7cdeed8e27423c36f72aaa09e6c6d97c552830791b842c317779e986c13d489600eaa71f4 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/15ca3900fc28ac7f59142b003411398c +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/5a25f51486a42ffcb62cd5904b09017bcbff9e2d2b1ae4460989dedeed736ac62c32e5d0c3168aa1340f541bb9558eb4c4ef70435cbcf08a92635efe4de37a43 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/bcd5a589ff854f7a0f51d4809a5c90fc +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/2bc56d8899993b7dd0243eebdba4d45d869d237cc8c8c89e2505de34d5752636c9e55bbfe60d7a91fbec8b37e14f8fe736c7f97859122285b90d6d399e38f582 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9915cafbe3063f5d579d408385d8fc41 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/7a8aef9f82e6a4e7e0ab75a912841b4c02219f86cd5afba0aefb32f1060b343425b3e4ab54a89557686ffa0557bc4cbeede0219bda980e2f0d84a401892ed433 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/45a3430d5e980eaedf0e3d149a83f985 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/ff849bce1450e67f9a94676999914f0a7c8aae60f0cf1037b70a727e6587cab77dce8f549dd6531d0a20c87682780c5bd6abdf301d3c70b7a12ca1db23a5a420 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/efa6087f7827c57d282c35125d5697e5 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ca154f18d4d2861026f2d59c92a09da25794f30913efc764108427a8e33da6170c11bf4c229cba4c3ae87e05efa6f18af3a1a5b5bcbe94e96b1d52e036c65d75 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/cf88d74b55eab615c04412c1ea582fdf +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/58bce04a429a7fc893567c2c3ca5954d92183043fe5f71b7180a8bbc480dfc355bfc1fb1d5e71cb9de7806c37ad472c6fe4c7eb341fb48c856b93ab4d3d5ef97 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/25c8b00a4452cd9d66ef53edbb747276 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/25716e63d7c28600324ee68664f0457167638fa46d91a5e4a3fa5001d08fb816e0b3e0402ffa5f45b7e7f1efc014976094fa090fc4f10d71ca753da3f2a73a3d +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/de3025ca30ed8df01e8f358fd868c3ec +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/517dc881378677bdd6011b349d88fbc51cf6e64cc67542f8cc5e4382deb5d0c16177990ed67890a797fd078c298a615b0ecac94788a2bf27ba6ea353361d4e95 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/85defcd1027bf22359db1c8dbbfa5f66 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/0d3995021935454a11d27da18ad41c2c7b3e86745d9f978ef321a16a312e73d0545482e34e6cb10ed9dd255bf0d5686c240aa4ea795d56c8d55b8207bd5b66c9 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/7a30a754501c41f5efe28c0a1d5dfff5 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/26cb6342efdba3e53305330355bf266e40b1300879011215ecf2311e4fe516d65a4793aa209e7eff96354d59ea962a0b58fb1b9544559786e569fba0fb393cc3 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/92d7c4e18a1ee1c32d7f25361b63d828 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/dd64730725a3ed2bc26262b661c565fee9c2a4a8660109707c773437615cb92a148c1d196700b254d6d7950de0de7177af7802963674021da9ee2d0650de95c6 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/fbf6174730f401f14bfccdf34c97518c +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/f3d78b63241bb142787d1dead601bc2b5f06d5b9dc2b2dd78da015df6d50d9f9a7cb478db2084157eba65a57f3328e9f728ddbfa3b7d760eda1aab0f97cb916b +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/fb1c596f521fc005e3626cee940fc4bd +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/7b4ec22c2e882644aa8377e10c8ee18fa2f880e2afd1f49548c5251a74636a9933af104ab0d1cf114b1ec474342c2c296b9e5eacb681aec202868929283aea92 +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/ace5ea53262082e1962cd1e6068fdfd1 +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/2391e569a172e200ec9b1384c5e6f2f04807c8b47dcf4938243872e23867ad2b53057f511ccf9e3fd2817696af322e6285764b75e2e72cef9da06cee2650be25 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/9e207cccaf077025faf4ffbfbe911b26 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b1fd95de38aa698e519316ed14e3ad4aff6e0e11a8415edfd85b2bf4e2504915d76cffcd631b69b3c949f10ae087c4fe43ecc7b44a390646e764f44100e3039b +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/5d221a750e203aa06b6e4a6d22730acf +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/71b18114084096a3199d866f05df8741049d1f536b0ef156d90f18194a0f78262f04e8ac68e7e5e7f95bc2aef34dd524d06ab5c1f41524ff2ddad17e7002c3b3 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/c3ea41e66f7d0749e1f45b7e9d2e32ae +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/a85cfc811ff0081bf565e922d337c56544ee3f7bd10d8db51f5bc26c45928b725bd1f3a57760c868b90b0ed1a7b949afd1087331dce995cbec73f627b1fe8d87 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/a739244415d4ea0f2c0ae3a96ce9b3c3 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/8cd8d5985172dfb3558d1b5d1658b3a01dfd75327badb0dea8582d51bdeeb77db0af6890a5dfb8ae25f3611ddfd734666ac08347c2d2d63f81bed6e9796b364f +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/2d69f3b14ce0fe480dd6b3a8c030a27d +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/99f4c9e6fb8114b61aaa642fd85b945dbf47328b2eaa60cf5845b3883c2917e5b5e239a8fac13005271fd018d0f4b23d51934f5894ce3c994bc185d8eebfca17 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/5096e6347bb08fd53b0416b4d4338405 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/bcc0848e319846f930c8b89008030f1ad124cd2caa7cae11a63d33d4ccde78f044e65857a4e11f7e25ffc0e785af8c9e7083d289273d79181aee366c1f3754fb +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/022d407dbd191ba036f73bd986b067be +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/0c793bb350eb1540c09a70743801ba1b76053f07f64a57d133a1158fcbb057c52a62bc2d08403c605aef30a4dfbd7268fbcc83e18af8406523b114f939f7e830 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/644e82b0e190856d95d6af1fd1bc84ea +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/bf736aa4b882ead66b02afb504cb23ad7f77e8674d9ab9fdd769e62c7328b0f9112a360967d495a208e597f9a2f20f9cf436acf2706e6d09847d65638a3cb482 +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e90086cb8edaa9a4066b58b0bb8e2636 +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/38908f3054bee0fc7b9f6c257cdae59dc641cef19d48793faf76a027aeba774fcf9ed9d820766bdc31d3301f4ddd5135c3c328f9db9a39c613ce925db0a166be +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/98be4b9ddd1d044cd7ad6250b38fccad +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/18e881e0d752ce0334d3c37e4cd0a41db4172e376410c860fb29e31d37ad5d95f9598f1bf0280c81fabf88129a17535c4f22e7f80f519c58e3da521923667fb6 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/281234060bf9f417fbae340cafd20309 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/e9e1c01dc849473f8885c2534ecb1391186cf283076402be9ba05bb5eb8633734e9b1f03af39218eb7cc40f59b727d8feae3a82fc60d593c86b5909f316d7c8b +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/aab22266d8f08325697cfe2b8083895b +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/295a4d82a4671023df12f8faef8593863a2ca970f59a1946605add0f80650a220fa75bb250b7d3df05070b23bd10f11daba4594fc0d5a16db09a2a7b737f4bb8 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/da928189b76c03126b556d7141bb4adb +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/fe0cd8f8791bfba761ba43374c9618c06813828421e9b2a0093bb132b6dabf41b732413e0d59a6e2a97e42733d9cef8293086ec1d33bdac031bb0d528f424aa9 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/4f4c5a6edaf01f8d6b98fde4e27e73ac +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/0f83c130e14afcfd3e25b3f6200b0dc909d3f8ad498e8580c480121e7ee313c9c33972cabda9d6b25f548a1ec42e19f7df9300b1717dcae352ebb5b8004b05ea llvm-julia-20.1.2-0.tar.gz/md5/aebf84a1b2ea2384d8f7d89ae6e06a29 llvm-julia-20.1.2-0.tar.gz/sha512/0023c0ba21df6f130b543e58c4934a4061c4fdba9525867da3b7f1dd7a9aa2189eca90c8600d93958da5694780cb4aa2c05683d19817d8298305ec81d88cdc4a llvm-project-19.1.4.tar.xz/md5/1e13043b18558e4346ea3769094c9737 diff --git a/deps/checksums/zstd b/deps/checksums/zstd new file mode 100644 index 0000000000000..aea151b266966 --- /dev/null +++ b/deps/checksums/zstd @@ -0,0 +1,38 @@ +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111 +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3 +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298 +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86 +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615 +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4 +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77 +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19 +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e diff --git a/deps/csl.mk b/deps/csl.mk index 86eb50966fa4f..8319a0aeb485f 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -128,6 +128,7 @@ install-csl: cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libssp.dll.a $(build_libdir)/ endif endif + ifeq ($(OS),WINNT) uninstall-csl: uninstall-gcc-libraries uninstall-gcc-libraries: @@ -137,4 +138,5 @@ uninstall-gcc-libraries: -rm -f $(build_private_libdir)/libmsvcrt.a -rm -f $(build_private_libdir)/libssp.dll.a -rm -f $(build_libdir)/libssp.dll.a +.PHONY: uninstall-gcc-libraries endif diff --git a/deps/llvm.mk b/deps/llvm.mk index 8afa0580b4bf6..c87f8036ccc17 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -99,7 +99,7 @@ LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING="$(LLVM_TARGETS)" -DCMAKE_BUILD_TYP LLVM_CMAKE += -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="$(LLVM_EXPERIMENTAL_TARGETS)" LLVM_CMAKE += -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))" LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=FORCE_ON -DZLIB_ROOT="$(build_prefix)" -LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=OFF +LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=FORCE_ON -DZSTD_ROOT="$(build_prefix)" ifeq ($(USE_POLLY_ACC),1) LLVM_CMAKE += -DPOLLY_ENABLE_GPGPU_CODEGEN=ON endif @@ -251,6 +251,11 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) $(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +$(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zstd +endif + + # NOTE: LLVM 12 and 13 have their patches applied to JuliaLang/llvm-project # declare that all patches must be applied before running ./configure @@ -352,6 +357,10 @@ $(eval $(call bb-install,lld,LLD,false,true)) $(eval $(call bb-install,clang,CLANG,false,true)) $(eval $(call bb-install,llvm-tools,LLVM_TOOLS,false,true)) +# work-around for Yggdrasil packaging bug (https://github.com/JuliaPackaging/Yggdrasil/pull/11231) +$(build_prefix)/manifest/llvm-tools uninstall-llvm-tools: \ + TAR:=$(TAR) --exclude=llvm-config.exe + endif # USE_BINARYBUILDER_LLVM get-lld: get-llvm diff --git a/deps/llvm.version b/deps/llvm.version index f283c55d1f4e6..c68198b6d1729 100644 --- a/deps/llvm.version +++ b/deps/llvm.version @@ -2,7 +2,7 @@ ## jll artifact LLVM_JLL_NAME := libLLVM -LLVM_ASSERT_JLL_VER := 20.1.2+0 +LLVM_ASSERT_JLL_VER := 20.1.2+1 ## source build # Version number of LLVM LLVM_VER := 20.1.2 diff --git a/deps/p7zip.mk b/deps/p7zip.mk index c7c2874d49a5e..b817db31c7cba 100644 --- a/deps/p7zip.mk +++ b/deps/p7zip.mk @@ -3,6 +3,8 @@ include $(SRCDIR)/p7zip.version ifneq ($(USE_BINARYBUILDER_P7ZIP),1) +P7ZIP_BUILD_OPTS := bindir=$(build_private_libexecdir) CC="$(CC)" CXX="$(CXX)" + $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz: | $(SRCCACHE) $(JLDOWNLOAD) $@ https://github.com/p7zip-project/p7zip/archive/refs/tags/v$(P7ZIP_VER).tar.gz @@ -17,12 +19,12 @@ checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured - $(MAKE) -C $(dir $<) $(MAKE_COMMON) CC="$(CC)" CXX="$(CXX)" 7za + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za$(EXE) echo 1 > $@ define P7ZIP_INSTALL - mkdir -p $2/$$(build_bindir) - cp -a $1/bin/7za $2/$$(build_bindir)/7z + mkdir -p $2/$$(build_private_libexecdir)/ + cp -a $1/bin/7za$(EXE) $2/$$(build_private_libexecdir)/7z$(EXE) endef $(eval $(call staged-install, \ p7zip,p7zip-$(P7ZIP_VER), \ @@ -30,8 +32,8 @@ $(eval $(call staged-install, \ clean-p7zip: -rm -f $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled - -rm -f $(build_bindir)/7za - -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) clean + -rm -f $(build_bindir)/7z$(EXE) $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) + -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) clean distclean-p7zip: rm -rf $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) @@ -48,5 +50,23 @@ check-p7zip: compile-p7zip else # USE_BINARYBUILDER_P7ZIP $(eval $(call bb-install,p7zip,P7ZIP,false)) +# move from bindir to shlibdir, where we expect to install it +install-p7zip: post-install-p7zip +uninstall-p7zip: pre-uninstall-p7zip +post-install-p7zip: $(build_prefix)/manifest/p7zip + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/7z$(EXE) ] || mv $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) + [ -e $(build_private_libexecdir)/7z$(EXE) ] +ifeq ($(OS),WINNT) + [ ! -e $(build_bindir)/7z.dll ] || mv $(build_bindir)/7z.dll $(build_private_libexecdir)/7z.dll + [ -e $(build_private_libexecdir)/7z.dll ] +endif +pre-uninstall-p7zip: + -rm -f $(build_private_libexecdir)/7z$(EXE) +ifeq ($(OS),WINNT) + -rm -f $(build_private_libexecdir)/7z.dll +endif + +.PHONY: post-install-p7zip pre-uninstall-p7zip endif diff --git a/deps/sanitizers.mk b/deps/sanitizers.mk index ace86c16bcefe..2b685b8d80aef 100644 --- a/deps/sanitizers.mk +++ b/deps/sanitizers.mk @@ -17,10 +17,11 @@ install-sanitizers: $$(addprefix $$(build_libdir)/, $$(notdir $$(call pathsearch echo "Sanitizer library $(1) not found in $$(SANITIZER_LIB_PATH)"; \ exit 1; \ fi -$$(addprefix $$(build_shlibdir)/,$(2)): $$(addprefix $$(dir $$(call pathsearch_all,$(1),$$(SANITIZER_LIB_PATH))),$(2)) | $$(build_shlibdir) +$$(addprefix $$(build_shlibdir)/,$(2)): $$(addprefix $$(dir $$(call pathsearch_all,$(1),$$(SANITIZER_LIB_PATH))),$(2)) $$(PATCHELF_MANIFEST) | $$(build_shlibdir) -cp $$< $$@ - $(if $(filter $(OS), Linux), \ - -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$$$ORIGIN' $$@ , 0) +ifneq (,$(findstring $(OS),Linux)) + -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$$$ORIGIN' $$@ +endif endef ifeq ($(USECLANG),1) diff --git a/deps/tools/bb-install.mk b/deps/tools/bb-install.mk index ee7f833a8ac2b..66e3d716c90ff 100644 --- a/deps/tools/bb-install.mk +++ b/deps/tools/bb-install.mk @@ -55,9 +55,9 @@ ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST))) @# work-around a gtar bug: they do some complicated work to avoid the mkdir @# syscall, which is buggy when working with Tar.jl files so we manually do @# the mkdir calls first in a pre-pass - $(TAR) -tzf $$< | xargs -n 1 dirname | sort -u | (cd $$(build_prefix) && xargs -t mkdir -p) + $$(TAR) -tzf $$< | xargs -n 1 dirname | sort -u | (cd $$(build_prefix) && xargs -t mkdir -p) endif - $(UNTAR) $$< -C $$(build_prefix) + $$(UNTAR) $$< -C $$(build_prefix) echo '$$(UNINSTALL_$(strip $1))' > $$@ # Special "checksum-foo" target to speed up `contrib/refresh_checksums.sh` diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 01b57316f9d1a..890eca8d718fa 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -187,7 +187,7 @@ UNINSTALL_$(strip $1) := $2 staged-uninstaller $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tar | $(build_prefix)/manifest -+[ ! -e $$@ ] || $$(MAKE) uninstall-$(strip $1) - $(UNTAR) $$< -C $$(build_prefix) + $$(UNTAR) $$< -C $$(build_prefix) $6 echo '$$(UNINSTALL_$(strip $1))' > $$@ .PHONY: $(addsuffix -$(strip $1),stage install distclean uninstall reinstall) diff --git a/deps/zstd.mk b/deps/zstd.mk new file mode 100644 index 0000000000000..5ead77641858a --- /dev/null +++ b/deps/zstd.mk @@ -0,0 +1,60 @@ +## Zstd ## +ifneq ($(USE_BINARYBUILDER_ZSTD), 1) +ZSTD_GIT_URL := https://github.com/facebook/zstd.git +ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 +$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) + +ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted + echo 1 > $@ + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) + echo 1 > $@ + +$(eval $(call staged-install, \ + zstd,$(ZSTD_SRC_DIR), \ + MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \ + $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_private_libexecdir)/libzstd.$(SHLIB_EXT))) + +clean-zstd: + -rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled + -$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean + +get-zstd: $(ZSTD_SRC_FILE) +extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted +configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured +compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled +fastcheck-zstd: check-zstd +check-zstd: compile-zstd + +else # USE_BINARYBUILDER_ZSTD + +$(eval $(call bb-install,zstd,ZSTD,false)) +# move from bindir to shlibdir, where we expect to install it +install-zstd: post-install-zstd +uninstall-zstd: pre-uninstall-zstd +post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST) + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/zstdmt$(EXE) ] || mv $(build_bindir)/zstdmt$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + [ ! -e $(build_bindir)/zstd$(EXE) ] || mv $(build_bindir)/zstd$(EXE) $(build_private_libexecdir)/zstd$(EXE) + [ -e $(build_private_libexecdir)/zstd$(EXE) ] + [ -e $(build_private_libexecdir)/zstdmt$(EXE) ] +ifeq ($(OS), Darwin) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \ + done +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/$$j || exit 1; \ + done +endif + +pre-uninstall-zstd: + -rm -f $(build_private_libexecdir)/zstd$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + +endif # USE_BINARYBUILDER_ZSTD diff --git a/deps/zstd.version b/deps/zstd.version new file mode 100644 index 0000000000000..d4d960aa6f04b --- /dev/null +++ b/deps/zstd.version @@ -0,0 +1,8 @@ +# -*- makefile -*- +## jll artifact +ZSTD_JLL_NAME := Zstd + +## source build +ZSTD_VER := 1.5.7 +ZSTD_BRANCH=v1.5.7 +ZSTD_SHA1=f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/julia.spdx.json b/julia.spdx.json index 0d7ab1df94688..8664b2c653386 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -240,7 +240,7 @@ "licenseConcluded": "Apache-2.0", "licenseDeclared": "Apache-2.0", "copyrightText": "Copyright (c) 1998-2024 The OpenSSL Project Authors. Copyright (c) 1995-1998 Eric A. Young, Tim J. Hudson.", - "summary": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the TLS (formerly SSL), DTLS and QUIC (currently client side only) protocols.", + "summary": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the TLS (formerly SSL), DTLS and QUIC (currently client side only) protocols." }, { "name": "mpfr", @@ -432,6 +432,18 @@ "copyrightText": "Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler", "summary": "A massively spiffy yet delicately unobtrusive compression library." }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "downloadLocation": "git+https://github.com/facebook/zstd.git", + "filesAnalyzed": false, + "homepage": "https://www.zstd.net", + "sourceInfo": "The git hash of the version in use can be found in the file deps/zstd.version", + "licenseConcluded": "BSD-3-Clause", + "licenseDeclared": "GPL-2.0+ OR BSD-3-Clause", + "copyrightText": "Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.", + "summary": "Zstandard, or zstd as short version, is a fast lossless compression algorithm." + }, { "name": "patchelf", "SPDXID": "SPDXRef-patchelf", @@ -639,6 +651,11 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, + { + "spdxElementId": "SPDXRef-zstd", + "relationshipType": "BUILD_DEPENDENCY_OF", + "relatedSpdxElement": "SPDXRef-JuliaMain" + }, { "spdxElementId": "SPDXRef-patchelf", "relationshipType": "BUILD_TOOL_OF", diff --git a/src/Makefile b/src/Makefile index 89e8051afcb03..717afa55c6207 100644 --- a/src/Makefile +++ b/src/Makefile @@ -170,6 +170,7 @@ CG_LLVMLINK += $(LLVM_LDFLAGS) -lLLVM else CG_LLVMLINK += $(LLVM_LDFLAGS) $(LLVM_SHARED_LINK_FLAG) endif # OS +CG_LLVMLINK += -lz -lzstd endif # USE_LLVM_SHLIB endif # USE_SYSTEM_LLVM @@ -178,7 +179,9 @@ FLAGS += -DLLVM_SHLIB endif # USE_LLVM_SHLIB == 1 endif # JULIACODEGEN == LLVM -RT_LLVM_LINK_ARGS := $(shell $(LLVM_CONFIG_HOST) --libs $(RT_LLVM_LIBS) --system-libs --link-static) +# Use subst to work around llvm-configure bug in Yggdrasil build: https://github.com/llvm/llvm-project/pull/139945 +CG_LLVMLINK := $(subst /workspace/destdir/lib/libzstd.dll.a,-lzstd,$(CG_LLVMLINK)) +RT_LLVM_LINK_ARGS := $(subst /workspace/destdir/lib/libzstd.dll.a,-lzstd,$(shell $(LLVM_CONFIG_HOST) --libs $(RT_LLVM_LIBS) --system-libs --link-static)) RT_LLVMLINK += $(LLVM_LDFLAGS) $(RT_LLVM_LINK_ARGS) ifeq ($(OS), WINNT) RT_LLVMLINK += -luuid -lole32 @@ -213,6 +216,13 @@ DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) CODEGEN_OBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.o) CODEGEN_DOBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.dbg.obj) +ifeq ($(OS)_$(BINARY),WINNT_32) +OBJS += $(BUILDDIR)/llvm-Compression.o +DOBJS += $(BUILDDIR)/llvm-Compression.dbg.obj +CODEGEN_OBJS += $(BUILDDIR)/llvm-Compression.o +CODEGEN_DOBJS += $(BUILDDIR)/llvm-Compression.dbg.obj +endif + # Add SONAME defines so we can embed proper `dlopen()` calls. ADDL_SHIPFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys.$(SHLIB_EXT)))) \ -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT))) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index e114dccfe5772..edfd2d05c716d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #ifdef _OS_DARWIN_ #include @@ -336,9 +338,18 @@ void JITDebugInfoRegistry::registerJITObject(const object::ObjectFile &Object, #endif // defined(_OS_WINDOWS_) SmallVector packed; - compression::zlib::compress(ArrayRef((uint8_t*)Object.getData().data(), Object.getData().size()), packed, compression::zlib::DefaultCompression); - jl_jit_add_bytes(packed.size()); - auto ObjectCopy = new LazyObjectInfo{packed, Object.getData().size()}; // intentionally leaked so that we don't need to ref-count it, intentionally copied so that we exact-size the allocation (since no shrink_to_fit function) + ArrayRef unpacked = arrayRefFromStringRef(Object.getData()); + std::optional F; + if (compression::zstd::isAvailable()) + F = compression::Format::Zstd; + else if (compression::zlib::isAvailable()) + F = compression::Format::Zlib; + if (F) + compression::compress(*F, unpacked, packed); + // intentionally leak this so that we don't need to ref-count it + // intentionally copy the input so that we exact-size the allocation (since no shrink_to_fit function) + auto ObjectCopy = new LazyObjectInfo{SmallVector(F ? ArrayRef(packed) : unpacked), F ? Object.getData().size() : 0}; + jl_jit_add_bytes(ObjectCopy->data.size()); auto symbols = object::computeSymbolSizes(Object); bool hassection = false; for (const auto &sym_size : symbols) { @@ -1214,7 +1225,8 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, if (!lazyobject->object && !lazyobject->data.empty()) { if (lazyobject->uncompressedsize) { SmallVector unpacked; - Error E = compression::zlib::decompress(lazyobject->data, unpacked, lazyobject->uncompressedsize); + compression::Format F = compression::zstd::isAvailable() ? compression::Format::Zstd : compression::Format::Zlib; + Error E = compression::decompress(F, lazyobject->data, unpacked, lazyobject->uncompressedsize); if (E) lazyobject->data.clear(); else diff --git a/src/llvm-Compression.cpp b/src/llvm-Compression.cpp new file mode 100644 index 0000000000000..c83f626747c27 --- /dev/null +++ b/src/llvm-Compression.cpp @@ -0,0 +1,245 @@ +//===--- Compression.cpp - Compression implementation ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements compression functions. +// +//===----------------------------------------------------------------------===// + +#include "llvm-Compression.h" +#include "llvm/Support/Compression.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#if LLVM_ENABLE_ZLIB +#include +#endif +#if LLVM_ENABLE_ZSTD +#include +#endif + +using namespace llvm; +using namespace llvm::compression; + +const char *compression::getReasonIfUnsupported(compression::Format F) { + switch (F) { + case compression::Format::Zlib: + if (zlib::isAvailable()) + return nullptr; + return "LLVM was not built with LLVM_ENABLE_ZLIB or did not find zlib at " + "build time"; + case compression::Format::Zstd: + if (zstd::isAvailable()) + return nullptr; + return "LLVM was not built with LLVM_ENABLE_ZSTD or did not find zstd at " + "build time"; + } + llvm_unreachable(""); +} + +void compression::compress(Params P, ArrayRef Input, + SmallVectorImpl &Output) { + switch (P.format) { + case compression::Format::Zlib: + zlib::compress(Input, Output, P.level); + break; + case compression::Format::Zstd: + zstd::compress(Input, Output, P.level, P.zstdEnableLdm); + break; + } +} + +Error compression::decompress(DebugCompressionType T, ArrayRef Input, + uint8_t *Output, size_t UncompressedSize) { + switch (formatFor(T)) { + case compression::Format::Zlib: + return zlib::decompress(Input, Output, UncompressedSize); + case compression::Format::Zstd: + return zstd::decompress(Input, Output, UncompressedSize); + } + llvm_unreachable(""); +} + +Error compression::decompress(compression::Format F, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + switch (F) { + case compression::Format::Zlib: + return zlib::decompress(Input, Output, UncompressedSize); + case compression::Format::Zstd: + return zstd::decompress(Input, Output, UncompressedSize); + } + llvm_unreachable(""); +} + +Error compression::decompress(DebugCompressionType T, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + return decompress(formatFor(T), Input, Output, UncompressedSize); +} + +#if LLVM_ENABLE_ZLIB + +static StringRef convertZlibCodeToString(int Code) { + switch (Code) { + case Z_MEM_ERROR: + return "zlib error: Z_MEM_ERROR"; + case Z_BUF_ERROR: + return "zlib error: Z_BUF_ERROR"; + case Z_STREAM_ERROR: + return "zlib error: Z_STREAM_ERROR"; + case Z_DATA_ERROR: + return "zlib error: Z_DATA_ERROR"; + case Z_OK: + default: + llvm_unreachable("unknown or unexpected zlib status code"); + } +} + +bool zlib::isAvailable() { return true; } + +void zlib::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + unsigned long CompressedSize = ::compressBound(Input.size()); + CompressedBuffer.resize_for_overwrite(CompressedSize); + int Res = ::compress2((Bytef *)CompressedBuffer.data(), &CompressedSize, + (const Bytef *)Input.data(), Input.size(), Level); + if (Res == Z_MEM_ERROR) + report_bad_alloc_error("Allocation failed"); + assert(Res == Z_OK); + // Tell MemorySanitizer that zlib output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + if (CompressedSize < CompressedBuffer.size()) + CompressedBuffer.truncate(CompressedSize); +} + +Error zlib::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + int Res = ::uncompress((Bytef *)Output, (uLongf *)&UncompressedSize, + (const Bytef *)Input.data(), Input.size()); + // Tell MemorySanitizer that zlib output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(Output, UncompressedSize); + return Res ? make_error(convertZlibCodeToString(Res), + inconvertibleErrorCode()) + : Error::success(); +} + +Error zlib::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + Output.resize_for_overwrite(UncompressedSize); + Error E = zlib::decompress(Input, Output.data(), UncompressedSize); + if (UncompressedSize < Output.size()) + Output.truncate(UncompressedSize); + return E; +} + +#else +bool zlib::isAvailable() { return false; } +void zlib::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + llvm_unreachable("zlib::compress is unavailable"); +} +Error zlib::decompress(ArrayRef Input, uint8_t *UncompressedBuffer, + size_t &UncompressedSize) { + llvm_unreachable("zlib::decompress is unavailable"); +} +Error zlib::decompress(ArrayRef Input, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + llvm_unreachable("zlib::decompress is unavailable"); +} +#endif + +#if LLVM_ENABLE_ZSTD + +bool zstd::isAvailable() { return true; } + +#include // Ensure ZSTD library is included + +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level, + bool EnableLdm) { + ZSTD_CCtx *Cctx = ZSTD_createCCtx(); + if (!Cctx) + report_bad_alloc_error("Failed to create ZSTD_CCtx"); + + if (ZSTD_isError(ZSTD_CCtx_setParameter( + Cctx, ZSTD_c_enableLongDistanceMatching, EnableLdm ? 1 : 0))) { + ZSTD_freeCCtx(Cctx); + report_bad_alloc_error("Failed to set ZSTD_c_enableLongDistanceMatching"); + } + + if (ZSTD_isError( + ZSTD_CCtx_setParameter(Cctx, ZSTD_c_compressionLevel, Level))) { + ZSTD_freeCCtx(Cctx); + report_bad_alloc_error("Failed to set ZSTD_c_compressionLevel"); + } + + unsigned long CompressedBufferSize = ZSTD_compressBound(Input.size()); + CompressedBuffer.resize_for_overwrite(CompressedBufferSize); + + size_t const CompressedSize = + ZSTD_compress2(Cctx, CompressedBuffer.data(), CompressedBufferSize, + Input.data(), Input.size()); + + ZSTD_freeCCtx(Cctx); + + if (ZSTD_isError(CompressedSize)) + report_bad_alloc_error("Compression failed"); + + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + if (CompressedSize < CompressedBuffer.size()) + CompressedBuffer.truncate(CompressedSize); +} + +Error zstd::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + const size_t Res = ::ZSTD_decompress( + Output, UncompressedSize, (const uint8_t *)Input.data(), Input.size()); + UncompressedSize = Res; + if (ZSTD_isError(Res)) + return make_error(ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(Output, UncompressedSize); + return Error::success(); +} + +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + Output.resize_for_overwrite(UncompressedSize); + Error E = zstd::decompress(Input, Output.data(), UncompressedSize); + if (UncompressedSize < Output.size()) + Output.truncate(UncompressedSize); + return E; +} + +#else +bool zstd::isAvailable() { return false; } +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level, + bool EnableLdm) { + llvm_unreachable("zstd::compress is unavailable"); +} +Error zstd::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + llvm_unreachable("zstd::decompress is unavailable"); +} +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + llvm_unreachable("zstd::decompress is unavailable"); +} +#endif diff --git a/src/llvm-Compression.h b/src/llvm-Compression.h new file mode 100644 index 0000000000000..246ccbd6f6dcf --- /dev/null +++ b/src/llvm-Compression.h @@ -0,0 +1,136 @@ +//===-- llvm/Support/Compression.h ---Compression----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains basic functions for compression/decompression. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_COMPRESSION_H +#define LLVM_SUPPORT_COMPRESSION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/DataTypes.h" + +namespace llvm { +template class SmallVectorImpl; +class Error; + +// None indicates no compression. The other members are a subset of +// compression::Format, which is used for compressed debug sections in some +// object file formats (e.g. ELF). This is a separate class as we may add new +// compression::Format members for non-debugging purposes. +enum class DebugCompressionType { + None, ///< No compression + Zlib, ///< zlib + Zstd, ///< Zstandard +}; + +namespace compression { +namespace zlib { + +constexpr int NoCompression = 0; +constexpr int BestSpeedCompression = 1; +constexpr int DefaultCompression = 6; +constexpr int BestSizeCompression = 9; + +LLVM_ABI bool isAvailable(); + +LLVM_ABI void compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression); + +LLVM_ABI Error decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize); + +LLVM_ABI Error decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace zlib + +namespace zstd { + +constexpr int NoCompression = -5; +constexpr int BestSpeedCompression = 1; +constexpr int DefaultCompression = 5; +constexpr int BestSizeCompression = 12; + +LLVM_ABI bool isAvailable(); + +LLVM_ABI void compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression, bool EnableLdm = false); + +LLVM_ABI Error decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize); + +LLVM_ABI Error decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace zstd + +enum class Format { + Zlib, + Zstd, +}; + +inline Format formatFor(DebugCompressionType Type) { + switch (Type) { + case DebugCompressionType::None: + llvm_unreachable("not a compression type"); + case DebugCompressionType::Zlib: + return Format::Zlib; + case DebugCompressionType::Zstd: + return Format::Zstd; + } + llvm_unreachable(""); +} + +struct Params { + constexpr Params(Format F) + : format(F), level(F == Format::Zlib ? zlib::DefaultCompression + : zstd::DefaultCompression) {} + constexpr Params(Format F, int L, bool Ldm = false) + : format(F), level(L), zstdEnableLdm(Ldm) {} + Params(DebugCompressionType Type) : Params(formatFor(Type)) {} + + Format format; + int level; + bool zstdEnableLdm = false; // Enable zstd long distance matching + // This may support multi-threading for zstd in the future. Note that + // different threads may produce different output, so be careful if certain + // output determinism is desired. +}; + +// Return nullptr if LLVM was built with support (LLVM_ENABLE_ZLIB, +// LLVM_ENABLE_ZSTD) for the specified compression format; otherwise +// return a string literal describing the reason. +LLVM_ABI const char *getReasonIfUnsupported(Format F); + +// Compress Input with the specified format P.Format. If Level is -1, use +// *::DefaultCompression for the format. +LLVM_ABI void compress(Params P, ArrayRef Input, + SmallVectorImpl &Output); + +// Decompress Input. The uncompressed size must be available. +LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef Input, + uint8_t *Output, size_t UncompressedSize); +LLVM_ABI Error decompress(Format F, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); +LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace compression + +} // End of namespace llvm + +#endif diff --git a/stdlib/Makefile b/stdlib/Makefile index 3975f24b7ae3b..e788f85155887 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -19,7 +19,7 @@ $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV OPENSSL MPFR NGHTTP2 \ BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \ - LLVMUNWIND CSL UNWIND LLD + ZSTD LLVMUNWIND CSL UNWIND LLD # Initialize this with JLLs that aren't in "deps/$(LibName).version" JLL_NAMES := MozillaCACerts_jll diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index c68ebcdbe533a..71c942549e804 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -274,15 +274,20 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.3.1+2" +[[deps.Zstd_jll]] +deps = ["Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + [[deps.dSFMT_jll]] deps = ["Artifacts", "Libdl"] uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" version = "2.2.5+2" [[deps.libLLVM_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "Libdl", "Zlib_jll", "Zstd_jll"] uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "18.1.7+3" +version = "20.1.2+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] diff --git a/stdlib/Zstd_jll/Project.toml b/stdlib/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..467516843390a --- /dev/null +++ b/stdlib/Zstd_jll/Project.toml @@ -0,0 +1,15 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..c16413f963d0b --- /dev/null +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +libzstd_handle::Ptr{Cvoid} = C_NULL + +if Sys.iswindows() + const libzstd = "libzstd-1.dll" +elseif Sys.isapple() + const libzstd = "@rpath/libzstd.1.dylib" +else + const libzstd = "libzstd.so.1" +end + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) +zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) + +function __init__() + global libzstd_handle = dlopen(libzstd) + nothing +end + +end # module Zstd_jll diff --git a/stdlib/Zstd_jll/test/runtests.jl b/stdlib/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/stdlib/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/stdlib/libLLVM_jll/Project.toml b/stdlib/libLLVM_jll/Project.toml index 512f8cc4e4dd9..04280f23f58da 100644 --- a/stdlib/libLLVM_jll/Project.toml +++ b/stdlib/libLLVM_jll/Project.toml @@ -1,12 +1,16 @@ name = "libLLVM_jll" uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "20.1.2+0" +version = "20.1.2+1" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [compat] +Zlib_jll = "1" +Zstd_jll = "1.5" julia = "1.13" [extras] diff --git a/stdlib/libLLVM_jll/src/libLLVM_jll.jl b/stdlib/libLLVM_jll/src/libLLVM_jll.jl index be2acb34faa65..c7d4a9128f312 100644 --- a/stdlib/libLLVM_jll/src/libLLVM_jll.jl +++ b/stdlib/libLLVM_jll/src/libLLVM_jll.jl @@ -3,7 +3,7 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/libLLVM_jll.jl baremodule libLLVM_jll -using Base, Libdl +using Base, Libdl, Zlib_jll, Zstd_jll const PATH_list = String[] const LIBPATH_list = String[] diff --git a/stdlib/p7zip_jll/src/p7zip_jll.jl b/stdlib/p7zip_jll/src/p7zip_jll.jl index a2a90a2450ea6..af461c6719632 100644 --- a/stdlib/p7zip_jll/src/p7zip_jll.jl +++ b/stdlib/p7zip_jll/src/p7zip_jll.jl @@ -4,14 +4,10 @@ baremodule p7zip_jll using Base -const PATH_list = String[] -const LIBPATH_list = String[] - export p7zip # These get calculated in __init__() const PATH = Ref("") -const LIBPATH = Ref("") artifact_dir::String = "" p7zip_path::String = "" if Sys.iswindows() @@ -21,71 +17,44 @@ else end if Sys.iswindows() - const LIBPATH_env = "PATH" - const LIBPATH_default = "" const pathsep = ';' elseif Sys.isapple() - const LIBPATH_env = "DYLD_FALLBACK_LIBRARY_PATH" - const LIBPATH_default = "~/lib:/usr/local/lib:/lib:/usr/lib" const pathsep = ':' else - const LIBPATH_env = "LD_LIBRARY_PATH" - const LIBPATH_default = "" const pathsep = ':' end -function adjust_ENV!(env::Dict{keytype(Base.EnvDict),valtype(Base.EnvDict)}, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) - if adjust_LIBPATH - LIBPATH_base = get(env, LIBPATH_env, expanduser(LIBPATH_default)) - if !isempty(LIBPATH_base) - env[LIBPATH_env] = string(LIBPATH, pathsep, LIBPATH_base) - else - env[LIBPATH_env] = LIBPATH - end - end - if adjust_PATH && (LIBPATH_env != "PATH" || !adjust_LIBPATH) - if adjust_PATH - if !isempty(get(env, "PATH", "")) - env["PATH"] = string(PATH, pathsep, env["PATH"]) - else - env["PATH"] = PATH - end - end - end - return env +function adjust_ENV() + addPATH = PATH[] + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) end -function p7zip(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - withenv(env...) do - return f(p7zip_path) +function p7zip(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + return f(p7zip()) end end -function p7zip(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - return Cmd(Cmd([p7zip_path]); env) -end +# the 7z.exe we ship has no dependencies, so it needs no PATH adjustment +p7zip(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) = `$p7zip_path` function init_p7zip_path() # Prefer our own bundled p7zip, but if we don't have one, pick it up off of the PATH - # If this is an in-tree build, `7z` will live in `bindir`. Otherwise, it'll be in `private_libexecdir` - for bundled_p7zip_path in (joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, p7zip_exe), - joinpath(Sys.BINDIR, p7zip_exe)) - if isfile(bundled_p7zip_path) - global p7zip_path = abspath(bundled_p7zip_path) - return - end + # Our `7z` lives in `private_libexecdir` + bundled_p7zip_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, p7zip_exe) + if isfile(bundled_p7zip_path) + global p7zip_path = abspath(bundled_p7zip_path) + else + global p7zip_path = something(Sys.which(p7zip_exe), p7zip_exe) end - global p7zip_path = something(Sys.which(p7zip_exe), p7zip_exe) end function __init__() global artifact_dir = dirname(Sys.BINDIR) init_p7zip_path() - PATH[] = dirname(p7zip_path) - push!(PATH_list, PATH[]) - append!(LIBPATH_list, [joinpath(Sys.BINDIR, Base.LIBDIR, "julia"), joinpath(Sys.BINDIR, Base.LIBDIR)]) - LIBPATH[] = join(LIBPATH_list, pathsep) + PATH[] = path = dirname(p7zip_path) + nothing end # JLLWrappers API compatibility shims. Note that not all of these will really make sense. diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 006b7a276a3b3..3184ac9c3305f 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -9,7 +9,7 @@ INDEPENDENT_STDLIBS := \ SparseArrays Statistics StyledStrings SuiteSparse_jll Tar Test TOML Unicode UUIDs \ dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll \ - MPFR_jll OpenLibm_jll OpenSSL_jll PCRE2_jll p7zip_jll Zlib_jll + MPFR_jll OpenLibm_jll OpenSSL_jll PCRE2_jll p7zip_jll Zlib_jll Zstd_jll STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) From 720d0f5fa08d46017075e6159aea9a5d56935cf5 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 20 May 2025 20:01:09 -0500 Subject: [PATCH 293/662] Fail when precompiles fail during build on CI (and fix bad precompile) (#58474) --- contrib/generate_precompile.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 20aa0079d749c..578e27110f436 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -49,7 +49,6 @@ precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf precompile(Tuple{typeof(Base.getindex), Type{Pair{Base.PkgId, UInt128}}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Vararg{Pair{Base.PkgId, UInt128}}}) precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}, Core.SimpleVector}) precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) -precompile(Tuple{Base.IncludeInto, RelocatableFolders.Path}) # LazyArtifacts (but more generally helpful) precompile(Tuple{Type{Base.Val{x} where x}, Module}) @@ -398,6 +397,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe if precompile(ps...) n_succeeded += 1 else + Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 end failed = length(statements) - n_succeeded @@ -405,6 +405,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : "")) catch ex # See #28808 + Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") @warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0 end end From 4c0017684829a410b5d7a2df16ce6e819a77fb73 Mon Sep 17 00:00:00 2001 From: Shahab Lavasani <122942441+ShahabSL@users.noreply.github.com> Date: Wed, 21 May 2025 06:38:48 -0500 Subject: [PATCH 294/662] REPL: Show input alias (e.g., \\:cat:) for Char display (#58181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR enhances the REPL display for Char types. It adds the LaTeX/emoji input alias after the standard Unicode information, making it consistent with the help?> mode behavior. For example, `'😼'` will now display as: `'😼': Unicode U+1F63C (category So: Symbol, other), input as \:smirk_cat:` **(Self-contained paragraph explaining the choice):** Considered modifying `Base.show` directly, but opted for adding a `REPL.show_repl` method instead to avoid introducing a dependency from `Base` on the `REPL` module's symbol mapping (`symbol_latex`). This approach keeps the REPL-specific display logic contained within the REPL module. Closes #58158 --- NEWS.md | 2 ++ stdlib/REPL/src/REPL.jl | 10 ++++++++++ stdlib/REPL/test/repl.jl | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/NEWS.md b/NEWS.md index fcf94b340ad66..2e4da039d3cd0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -62,6 +62,8 @@ Standard library changes #### REPL +* The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). + #### Test * Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 169b6a71858ef..5772b481eb151 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -555,6 +555,16 @@ display(d::REPLDisplay, x) = display(d, MIME("text/plain"), x) show_repl(io::IO, mime::MIME"text/plain", x) = show(io, mime, x) +function show_repl(io::IO, mime::MIME"text/plain", c::AbstractChar) + show(io, mime, c) # Call the original Base.show + # Check for LaTeX/emoji alias and print if found and using symbol_latex which is used in help?> mode + latex = symbol_latex(string(c)) + if !isempty(latex) + print(io, ", input as ") + printstyled(io, latex, ""; color=:cyan) + end +end + show_repl(io::IO, ::MIME"text/plain", ex::Expr) = print(io, JuliaSyntaxHighlighting.highlight( sprint(show, ex, context=IOContext(io, :color => false)))) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index b2ddbefd0c25a..a04ade2a5e78f 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1999,3 +1999,32 @@ end write(proj_file, "name = \"Bar\"\n") @test get_prompt("--project=$proj_file") == "(Bar) pkg> " end + +# Issue #58158 add alias for Char display in REPL +@testset "REPL show_repl Char alias" begin + # Test character with a known emoji alias + output = sprint(REPL.show_repl, MIME("text/plain"), '😼'; context=(:color => true)) + # Check for base info and the specific alias + @test occursin("'😼': Unicode U+1F63C (category So: Symbol, other)", output) + @test occursin(", input as ", output) # Check for the prefix text + @test occursin("\\:smirk_cat:", output) # Check for the alias text (may be colored) + + # Test character with a known LaTeX alias + output = sprint(REPL.show_repl, MIME("text/plain"), 'α'; context=(:color => true)) + # Check for base info and the specific alias + @test occursin("'α': Unicode U+03B1 (category Ll: Letter, lowercase)", output) + @test occursin(", input as ", output) # Check for the prefix text + @test occursin("\\alpha", output) # Check for the alias text (may be colored) + + # Test character without an alias + output = sprint(REPL.show_repl, MIME("text/plain"), 'X'; context=(:color => true)) + # Check for base info only + @test occursin("'X': ASCII/Unicode U+0058 (category Lu: Letter, uppercase)", output) + # Ensure alias part is *not* printed + @test !occursin(", input as ", output) + + # Test another character without an alias (symbol) + output = sprint(REPL.show_repl, MIME("text/plain"), '+'; context=(:color => true)) + @test occursin("'+': ASCII/Unicode U+002B (category Sm: Symbol, math)", output) + @test !occursin(", input as ", output) +end From 3466a161d5ba78851795290071c1d3b384d3446f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 21 May 2025 09:32:13 -0400 Subject: [PATCH 295/662] ir: Don't fail if :invoke has zero arguments (#58477) All julia functions always have at least one argument (ignoring toplevel thunks, but those have special representation). However, with ABIOverride it is possible to create zero argument CodeInstances and :invoke them. I'm not entirely sure this is useful, but things do generally seem to go through, so let's not unnecessarily error in the optimizer. --- Compiler/src/ssair/passes.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Compiler/src/ssair/passes.jl b/Compiler/src/ssair/passes.jl index f2e41d1142e65..f16163554b75b 100644 --- a/Compiler/src/ssair/passes.jl +++ b/Compiler/src/ssair/passes.jl @@ -9,7 +9,9 @@ end function is_known_invoke_or_call(@nospecialize(x), @nospecialize(func), ir::Union{IRCode,IncrementalCompact}) isinvoke = isexpr(x, :invoke) (isinvoke || isexpr(x, :call)) || return false - ft = argextype(x.args[isinvoke ? 2 : 1], ir) + narg = isinvoke ? 2 : 1 + length(x.args) < narg && return false + ft = argextype(x.args[narg], ir) return singleton_type(ft) === func end From c940a31e07eccdc0e864df9a78d1eb5f5c661006 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Wed, 21 May 2025 15:51:34 +0200 Subject: [PATCH 296/662] Remove unreachable error branch by casting to UInt in parsing (#58480) The `ncodeunits` field of substrings - and therefore also their size, cannot be negative. Ideally we would also remove the unnecessary checks in the other `tryparse` methods, that take start and end indices, but that should probably be rewritten to take a view, but that's a PR for another day. --- base/parse.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/parse.jl b/base/parse.jl index 2530e5a46146a..7e6f35cf6b35a 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -261,12 +261,12 @@ tryparse(::Type{Union{}}, slurp...; kwargs...) = error("cannot parse a value as function tryparse(::Type{Float64}, s::String) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s)) + (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end function tryparse(::Type{Float64}, s::SubString{String}) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits) + (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) hasvalue ? val : nothing end function tryparse_internal(::Type{Float64}, s::String, startpos::Int, endpos::Int) @@ -281,12 +281,12 @@ function tryparse_internal(::Type{Float64}, s::SubString{String}, startpos::Int, end function tryparse(::Type{Float32}, s::String) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s)) + (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end function tryparse(::Type{Float32}, s::SubString{String}) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits) + (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) hasvalue ? val : nothing end function tryparse_internal(::Type{Float32}, s::String, startpos::Int, endpos::Int) From 4b90899c7dfc98db75ff47e9005d208d69aa29d9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 21 May 2025 20:25:01 +0200 Subject: [PATCH 297/662] Expose native_code's jl_sysimg_gvars. (#58423) https://github.com/JuliaLang/julia/pull/57010 overhauled how global variables are initialized in code, breaking GPUCompiler.jl. After talking to @vtjnash, we are apparently supposed to use `jl_get_llvm_gvs` now to get the Julia memory location of a global variable (for a binding or constant value), however, the elements in there don't exactly correspond to the global variables in the module (not all module globals are "managed" by Julia, and the order is different). To make GPUCompiler.jl work again, here I propose renaming `jl_get_llvm_gvs` to `jl_get_llvm_gv_inits`, and making `jl_get_llvm_gvs` instead return the list of the managed global variables, making it possible to perform the global variable initialization that was done by Julia before using something like: ```julia if VERSION >= v"1.13.0-DEV.533" num_gvars = Ref{Csize_t}(0) @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing gvs = Vector{Ptr{LLVM.API.LLVMOpaqueValue}}(undef, num_gvars[]) @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, gvs::Ptr{LLVM.API.LLVMOpaqueValue})::Nothing inits = Vector{Ptr{Cvoid}}(undef, num_gvars[]) @ccall jl_get_llvm_gv_inits(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, inits::Ptr{Cvoid})::Nothing for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) ptr = const_inttoptr(ConstantInt(Int64(init)), value_type(gv)) initializer!(gv, ptr) obj = Base.unsafe_pointer_to_objref(init) @safe_info "Resolved $(name(gv)) to $obj" end end ``` --- src/aotcompile.cpp | 27 ++++++++++++++++++++++----- src/codegen-stubs.c | 1 + src/jl_exported_funcs.inc | 1 + src/julia_internal.h | 3 ++- src/staticdata.c | 4 ++-- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 144ad6fc804d7..33c0ad011e6f0 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -108,20 +108,37 @@ jl_get_llvm_mis_impl(void *native_code, size_t *num_elements, jl_method_instance } } +// get the list of global variables managed by the compiler extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_gvs_impl(void *native_code, size_t *num_elements, void **data) { - // map a memory location (jl_value_t or jl_binding_t) to a GlobalVariable jl_native_code_desc_t *desc = (jl_native_code_desc_t *)native_code; - auto &value_map = desc->jl_value_to_llvm; + auto &gvars = desc->jl_sysimg_gvars; if (data == NULL) { - *num_elements = value_map.size(); + *num_elements = gvars.size(); return; } - assert(*num_elements == value_map.size()); - memcpy(data, value_map.data(), *num_elements * sizeof(void *)); + assert(*num_elements == gvars.size()); + memcpy(data, gvars.data(), *num_elements * sizeof(void *)); +} + +// get the initializer values (jl_value_t or jl_binding_t ptr) of managed global variables +extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_gv_inits_impl(void *native_code, + size_t *num_elements, + void **data) +{ + jl_native_code_desc_t *desc = (jl_native_code_desc_t *)native_code; + auto &inits = desc->jl_value_to_llvm; + + if (data == NULL) { + *num_elements = inits.size(); + return; + } + + assert(*num_elements == inits.size()); + memcpy(data, inits.data(), *num_elements * sizeof(void *)); } extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_external_fns_impl(void *native_code, diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 04f38fb9091be..e083d1d90eeba 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -14,6 +14,7 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, ios_t *z, ios_t *s) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE +JL_DLLEXPORT void jl_get_llvm_gv_inits_fallback(void *native_code, arraylist_t *inits) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_mis_fallback(void *native_code, arraylist_t* MIs) UNAVAILABLE diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 7b204066b8c28..ea6f4c38f7f70 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -514,6 +514,7 @@ YY(jl_get_LLVM_VERSION) \ YY(jl_dump_native) \ YY(jl_get_llvm_gvs) \ + YY(jl_get_llvm_gv_inits) \ YY(jl_get_llvm_external_fns) \ YY(jl_get_llvm_mis) \ YY(jl_dump_function_asm) \ diff --git a/src/julia_internal.h b/src/julia_internal.h index d4167d3e8a5b9..133a347bf82dc 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -2043,8 +2043,9 @@ JL_DLLIMPORT void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, ios_t *z, ios_t *s, jl_emission_params_t *params); JL_DLLIMPORT void jl_get_llvm_gvs(void *native_code, size_t *num_els, void **gvs); +JL_DLLIMPORT void jl_get_llvm_gv_inits(void *native_code, size_t *num_els, void **inits); JL_DLLIMPORT void jl_get_llvm_external_fns(void *native_code, size_t *num_els, - jl_code_instance_t *gvs); + jl_code_instance_t *fns); JL_DLLIMPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx); JL_DLLIMPORT void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, diff --git a/src/staticdata.c b/src/staticdata.c index 211ba09ac72d7..301968e6e2e26 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3055,9 +3055,9 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, int en = jl_gc_enable(0); if (native_functions) { size_t num_gvars, num_external_fns; - jl_get_llvm_gvs(native_functions, &num_gvars, NULL); + jl_get_llvm_gv_inits(native_functions, &num_gvars, NULL); arraylist_grow(&gvars, num_gvars); - jl_get_llvm_gvs(native_functions, &num_gvars, gvars.items); + jl_get_llvm_gv_inits(native_functions, &num_gvars, gvars.items); jl_get_llvm_external_fns(native_functions, &num_external_fns, NULL); arraylist_grow(&external_fns, num_external_fns); jl_get_llvm_external_fns(native_functions, &num_external_fns, From e3982cd84030e4baf7b8e97119bbeec0fa982935 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 21 May 2025 23:46:59 -0300 Subject: [PATCH 298/662] Fix tbaa usage when storing into heap allocated immutable structs (#58483) --- Compiler/test/codegen.jl | 22 ++++++++++++++++++++++ src/cgutils.cpp | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 373d1d91cd912..c2e7abe34c34a 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1047,3 +1047,25 @@ f57872() = (Core.isdefinedglobal(@__MODULE__, Base.compilerbarrier(:const, :x578 @noinline f_mutateany(@nospecialize x) = x[] = 1 g_mutateany() = (y = Ref(0); f_mutateany(y); y[]) @test g_mutateany() === 1 + +# 58470 tbaa for unionselbyte of heap allocated mutables +mutable struct Wrapper58470 + x::Union{Nothing,Int} +end + +function findsomething58470(dict, inds) + default = Wrapper58470(nothing) + for i in inds + x = get(dict, i, default).x + if !isnothing(x) + return x + end + end + return nothing +end + +let io = IOBuffer() + code_llvm(io, findsomething58470, Tuple{Dict{Int64, Wrapper58470}, Vector{Int}}, dump_module=true, raw=true, optimize=false) + str = String(take!(io)) + @test !occursin("jtbaa_unionselbyte", str) +end diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 359aaf0772d49..b720945647756 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4141,7 +4141,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, size_t fsz1 = jl_field_size(sty, idx0) - 1; Value *ptindex = emit_ptrgep(ctx, addr, fsz1); setNameWithField(ctx.emission_context, ptindex, get_objname, sty, idx0, Twine(".tindex_ptr")); - return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, ctx.tbaa().tbaa_unionselbyte, + return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, strct.tbaa, Order, FailOrder, needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, issetfieldonce, modifyop, fname); @@ -4409,7 +4409,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg undef_derived_strct(ctx, strct, sty, strctinfo.tbaa); for (size_t i = nargs; i < nf; i++) { if (!jl_field_isptr(sty, i) && jl_is_uniontype(jl_field_type(sty, i))) { - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, strctinfo.tbaa); ai.decorateInst(ctx.builder.CreateAlignedStore( ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), emit_ptrgep(ctx, strct, jl_field_offset(sty, i) + jl_field_size(sty, i) - 1), From 56699ca713c577fd48d536e6ca76d2b207bd91d7 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Wed, 21 May 2025 23:06:15 -0400 Subject: [PATCH 299/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=2080d2e7505=20to=2088629b552=20(#58489)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 | 1 - .../Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 | 1 - .../Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 | 1 + .../Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 create mode 100644 deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 diff --git a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 deleted file mode 100644 index 8f0c112e59907..0000000000000 --- a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -afd0c5ac79adc7842dd414223782db56 diff --git a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 b/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 deleted file mode 100644 index c896e2abc199d..0000000000000 --- a/deps/checksums/Pkg-80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -8db5ccdfadbc3acfb2b22aecc19f9466d44d00eb60bd54a78b690a4ba8ec6ad1a37917fd8af3f25f3e3c15ba82bda2c46182d28eac223464ac3ad6a38201edd3 diff --git a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 new file mode 100644 index 0000000000000..67dc729b08313 --- /dev/null +++ b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 @@ -0,0 +1 @@ +2b20498f7b8114ec06e809af6051679c diff --git a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 new file mode 100644 index 0000000000000..009cd139a3f2c --- /dev/null +++ b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 @@ -0,0 +1 @@ +5fa605f7af3378a6b7037cb01c08cbeac050d0dfa0d3c1db70b69a464801034c896334cac4234c43b91e78a3da2b1bbf0e598c7100a12ef57d4d662e1fb2324b diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 216f7ec8d13c0..a5b7ea80e9d56 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0 +PKG_SHA1 = 88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 157b8bc303fd4e5d4e03f4b7ecf8388e7b68ac6c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 21 May 2025 23:22:17 -0400 Subject: [PATCH 300/662] Correct `LogRange` spelling in documentation (#58486) Also add necessary `Base` prefix. --- base/range.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/range.jl b/base/range.jl index d86e0092d9897..9d2b9fd736b22 100644 --- a/base/range.jl +++ b/base/range.jl @@ -569,7 +569,7 @@ julia> collect(LinRange(-0.1, 0.3, 5)) 0.3 ``` -See also [`Logrange`](@ref Base.LogRange) for logarithmically spaced points. +See also [`Base.LogRange`](@ref Base.LogRange) for logarithmically spaced points. """ struct LinRange{T,L<:Integer} <: AbstractRange{T} start::T From 6f52a98ac3da243090ae06ac1cdae185f04aa184 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 22 May 2025 09:30:49 -0300 Subject: [PATCH 301/662] Allow constant propagation of mutables with const fields (#58371) This make the ScopedValue code slightly better by folding away the has_default and default lookups Still needs a test --- Compiler/src/abstractlattice.jl | 5 +++-- Compiler/test/irpasses.jl | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Compiler/src/abstractlattice.jl b/Compiler/src/abstractlattice.jl index f1514c6c9eed6..4d0accedfc765 100644 --- a/Compiler/src/abstractlattice.jl +++ b/Compiler/src/abstractlattice.jl @@ -227,9 +227,10 @@ that should be forwarded along with constant propagation. end @nospecializeinfer function is_const_prop_profitable_arg(𝕃::ConstsLattice, @nospecialize t) if isa(t, Const) - # don't consider mutable values useful constants + # don't consider mutable values useful constants unless they have const fields val = t.val - return isa(val, Symbol) || isa(val, Type) || isa(val, Method) || isa(val, CodeInstance) || !ismutable(val) + return isa(val, Symbol) || isa(val, Type) || isa(val, Method) || isa(val, CodeInstance) || + !ismutable(val) || (typeof(val).name.constfields != C_NULL) end isa(t, PartialTypeVar) && return false # this isn't forwardable return is_const_prop_profitable_arg(widenlattice(𝕃), t) diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index d777a2c387b3d..758efaab9ab6b 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -2111,3 +2111,13 @@ let src = code_typed1(()) do end @test count(iscall((src, isdefined)), src.code) == 0 end +# We should successfully fold the default values of a ScopedValue +const svalconstprop = ScopedValue(1) +foosvalconstprop() = svalconstprop[] + +let src = code_typed1(foosvalconstprop, ()) + function is_constfield_load(expr) + iscall((src, getfield))(expr) && expr.args[3] in (:(:has_default), :(:default)) + end + @test count(is_constfield_load, src.code) == 0 +end From e506deebce6c9158fc2c46dc00ee3af28b16afc7 Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Thu, 22 May 2025 22:46:24 +0530 Subject: [PATCH 302/662] Update init.c for error on --handle-signals=no with multiple threads (#58464) --- src/init.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/init.c b/src/init.c index 5a8a3eaf5ccf2..c3a20443b61f3 100644 --- a/src/init.c +++ b/src/init.c @@ -557,6 +557,12 @@ static void restore_fp_env(void) if (jl_set_zero_subnormals(0) || jl_set_default_nans(0)) { jl_error("Failed to configure floating point environment"); } + if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_OFF && jl_atomic_load_relaxed(&jl_n_threads) > 1) { + jl_error("Cannot use `--handle-signals=no` with multiple threads (JULIA_NUM_THREADS > 1).\n" + "This will cause segmentation faults due to GC safepoint failures.\n" + "Remove `--handle-signals=no` or set JULIA_NUM_THREADS=1.\n" + "See: https://github.com/JuliaLang/julia/issues/50278"); + } } static NOINLINE void _finish_jl_init_(jl_image_buf_t sysimage, jl_ptls_t ptls, jl_task_t *ct) { From 053c739862919545f6e04e2a488c3692e881e7f4 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 22 May 2025 14:30:39 -0400 Subject: [PATCH 303/662] MozillaCACerts: Update to 2025-05-20 (#58496) --- deps/checksums/cacert-2025-02-25.pem/md5 | 1 - deps/checksums/cacert-2025-02-25.pem/sha512 | 1 - deps/checksums/cacert-2025-05-20.pem/md5 | 1 + deps/checksums/cacert-2025-05-20.pem/sha512 | 1 + deps/libgit2.version | 2 +- stdlib/MozillaCACerts_jll/Project.toml | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/cacert-2025-02-25.pem/md5 delete mode 100644 deps/checksums/cacert-2025-02-25.pem/sha512 create mode 100644 deps/checksums/cacert-2025-05-20.pem/md5 create mode 100644 deps/checksums/cacert-2025-05-20.pem/sha512 diff --git a/deps/checksums/cacert-2025-02-25.pem/md5 b/deps/checksums/cacert-2025-02-25.pem/md5 deleted file mode 100644 index 3dced8d2bee6b..0000000000000 --- a/deps/checksums/cacert-2025-02-25.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -1a7de82bb9f0fcc779ca18a7a9310898 diff --git a/deps/checksums/cacert-2025-02-25.pem/sha512 b/deps/checksums/cacert-2025-02-25.pem/sha512 deleted file mode 100644 index bb59a65af401e..0000000000000 --- a/deps/checksums/cacert-2025-02-25.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -e5fe41820460e6b65e8cd463d1a5f01b7103e1ef66cb75fedc15ebcba3ba6600d77e5e7c2ab94cbb1f11c63b688026a04422bbe2d7a861f7a988f67522ffae3c diff --git a/deps/checksums/cacert-2025-05-20.pem/md5 b/deps/checksums/cacert-2025-05-20.pem/md5 new file mode 100644 index 0000000000000..06a21e784fc78 --- /dev/null +++ b/deps/checksums/cacert-2025-05-20.pem/md5 @@ -0,0 +1 @@ +a4e2b0c77e807b80a4b8a58c411e4a15 diff --git a/deps/checksums/cacert-2025-05-20.pem/sha512 b/deps/checksums/cacert-2025-05-20.pem/sha512 new file mode 100644 index 0000000000000..4e5117cbf9f49 --- /dev/null +++ b/deps/checksums/cacert-2025-05-20.pem/sha512 @@ -0,0 +1 @@ +97bc802a7c055e6e58384920feb593596bc30bc9493a7550a168f4d7337d34166dc8f350713c468c605f81ed1c3b6380050f04e31b86b4877c9de90ce3512867 diff --git a/deps/libgit2.version b/deps/libgit2.version index 3f1f7a66fe972..ffba640e3b24e 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -11,4 +11,4 @@ LIBGIT2_SHA1=338e6fb681369ff0537719095e22ce9dc602dbf0 # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. # Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. -MOZILLA_CACERT_VERSION := 2025-02-25 +MOZILLA_CACERT_VERSION := 2025-05-20 diff --git a/stdlib/MozillaCACerts_jll/Project.toml b/stdlib/MozillaCACerts_jll/Project.toml index a951435168922..57c5526a6f1f2 100644 --- a/stdlib/MozillaCACerts_jll/Project.toml +++ b/stdlib/MozillaCACerts_jll/Project.toml @@ -1,7 +1,7 @@ name = "MozillaCACerts_jll" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" # Keep in sync with `deps/libgit2.version`. -version = "2025.02.25" +version = "2025.05.20" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 3f1fc0d0aee952856a9385d35eb5d362fe82ec3f Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Thu, 22 May 2025 14:42:26 -0400 Subject: [PATCH 304/662] make `add_sum(::Bool, ::BitIntegerSmall)` promote to `Int` (#58374) --- base/reduce.jl | 12 +++++++----- test/reduce.jl | 10 +++++----- test/reducedim.jl | 10 +++++----- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/base/reduce.jl b/base/reduce.jl index 9041fd088ccf3..cb4e9f07a21d2 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -14,7 +14,7 @@ The reduction operator used in `sum`. The main difference from [`+`](@ref) is th integers are promoted to `Int`/`UInt`. """ add_sum(x, y) = x + y -add_sum(x::BitSignedSmall, y::BitSignedSmall) = Int(x) + Int(y) +add_sum(x::Union{Bool,BitIntegerSmall}, y::Union{Bool,BitIntegerSmall}) = Int(x) + Int(y) add_sum(x::BitUnsignedSmall, y::BitUnsignedSmall) = UInt(x) + UInt(y) add_sum(x::Real, y::Real)::Real = x + y @@ -1073,8 +1073,8 @@ julia> count(i->(4<=i<=6), [2,3,4,5,6]) julia> count([true, false, true, true]) 3 -julia> count(>(3), 1:7, init=0x03) -0x07 +julia> count(>(3), 1:7, init=UInt(0)) +0x0000000000000004 ``` """ count(itr; init=0) = count(identity, itr; init) @@ -1083,8 +1083,10 @@ count(f, itr; init=0) = _simple_count(f, itr, init) _simple_count(pred, itr, init) = sum(_bool(pred), itr; init) -function _simple_count(::typeof(identity), x::Array{Bool}, init::T=0) where {T} - n::T = init +function _simple_count(::typeof(identity), x::Array{Bool}, init=0) + v0 = Base.add_sum(init, false) + T = typeof(v0) + n::T = v0 chunks = length(x) ÷ sizeof(UInt) mask = 0x0101010101010101 % UInt GC.@preserve x begin diff --git a/test/reduce.jl b/test/reduce.jl index c6274711cdef4..7500b00197847 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -586,11 +586,11 @@ struct NonFunctionIsZero end @test count(NonFunctionIsZero(), [0]) == 1 @test count(NonFunctionIsZero(), [1]) == 0 -@test count(Iterators.repeated(true, 3), init=0x04) === 0x07 -@test count(!=(2), Iterators.take(1:7, 3), init=Int32(0)) === Int32(2) -@test count(identity, [true, false], init=Int8(5)) === Int8(6) -@test count(!, [true false; false true], dims=:, init=Int16(0)) === Int16(2) -@test isequal(count(identity, [true false; false true], dims=2, init=UInt(4)), reshape(UInt[5, 5], 2, 1)) +@test count(Iterators.repeated(true, 3), init=UInt(0)) === UInt(3) +@test count(!=(2), Iterators.take(1:7, 3), init=Int32(0)) === 2 +@test count(identity, [true, false], init=Int8(0)) === 1 +@test count(!, [true false; false true], dims=:, init=Int16(0)) === 2 +@test isequal(count(identity, [true false; false true], dims=2, init=UInt(0)), reshape(UInt[1, 1], 2, 1)) ## cumsum, cummin, cummax diff --git a/test/reducedim.jl b/test/reducedim.jl index 6a6f20214058c..1664b2708d7e3 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -88,12 +88,12 @@ safe_minabs(A::Array{T}, region) where {T} = safe_mapslices(minimum, abs.(A), re @test @inferred(count(!, Breduc, dims=region)) ≈ safe_count(.!Breduc, region) @test isequal( - @inferred(count(Breduc, dims=region, init=0x02)), - safe_count(Breduc, region) .% UInt8 .+ 0x02, + @inferred(Array{UInt8,ndims(Breduc)}, count(Breduc, dims=region, init=0x00)), + safe_count(Breduc, region), ) @test isequal( - @inferred(count(!, Breduc, dims=region, init=Int16(0))), - safe_count(.!Breduc, region) .% Int16, + @inferred(Array{Int16,ndims(Breduc)}, count(!, Breduc, dims=region, init=Int16(0))), + safe_count(.!Breduc, region), ) end @@ -693,7 +693,7 @@ end @test_throws TypeError count!([1], [1]) end -@test @inferred(count(false:true, dims=:, init=0x0004)) === 0x0005 +@test @inferred(UInt16, count(false:true, dims=:, init=0x0000)) === 1 @test @inferred(count(isodd, reshape(1:9, 3, 3), dims=:, init=Int128(0))) === Int128(5) @testset "reduced_index for BigInt (issue #39995)" begin From d827fd7cdd939bb4a3a23ffd03c2596ae8d7899a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 22 May 2025 14:59:13 -0400 Subject: [PATCH 305/662] reduce: unalias arrays for hcat/vcat (map)reduce (#58490) Implements mapreduce_first for hcat and vcat --- base/reduce.jl | 2 ++ test/reduce.jl | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/base/reduce.jl b/base/reduce.jl index cb4e9f07a21d2..968ccda4db28e 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -407,6 +407,8 @@ reduce_first(::typeof(add_sum), x::BitUnsignedSmall) = UInt(x) reduce_first(::typeof(mul_prod), x) = reduce_first(*, x) reduce_first(::typeof(mul_prod), x::BitSignedSmall) = Int(x) reduce_first(::typeof(mul_prod), x::BitUnsignedSmall) = UInt(x) +reduce_first(::typeof(vcat), x) = vcat(x) +reduce_first(::typeof(hcat), x) = hcat(x) """ Base.mapreduce_first(f, op, x) diff --git a/test/reduce.jl b/test/reduce.jl index 7500b00197847..a1d66a03e2c42 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -771,3 +771,9 @@ end Val(any(in((:one,:two,:three)),(:four,:three))) end |> only == Val{true} end + +# `reduce(vcat, A)` should not alias the input for length-1 collections +let A=[1;;] + @test reduce(vcat, Any[A]) !== A + @test reduce(hcat, Any[A]) !== A +end From f277bcd11b6af3a226c7efd664548eacfd5b1cf8 Mon Sep 17 00:00:00 2001 From: Martin Kunz Date: Thu, 22 May 2025 21:10:56 +0200 Subject: [PATCH 306/662] Make `showarg` a public function (#58494) --- base/public.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/public.jl b/base/public.jl index 380eea344a3a8..3308c58ff1780 100644 --- a/base/public.jl +++ b/base/public.jl @@ -111,6 +111,7 @@ public reseteof, link_pipe!, dup, + showarg, # filesystem operations rename, From d9c8c42ad3fc54fa20f56886ad133a0c02729c7a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 22 May 2025 16:18:31 -0400 Subject: [PATCH 307/662] use `@main` for juliac executable entry point (#57588) Use at-main mechanism in juliac executables. Also add support for command line arguments. I changed `Core.ARGS` from a `Vector{Any}` to a `Vector{String}`. I don't know why we didn't do that years ago. Surely this is ok? --- contrib/juliac-buildscript.jl | 42 ++++++++++++++++++++++++++--------- src/gf.c | 2 -- src/jlapi.c | 34 +++++++++++++++------------- src/julia.h | 2 +- test/trimming/basic_jll.jl | 6 +---- test/trimming/hello.jl | 7 ++---- test/trimming/trimming.jl | 2 +- 7 files changed, 55 insertions(+), 40 deletions(-) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 4bfbfd2272220..1697ed3fd1f03 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -1,9 +1,5 @@ # Script to run in the process that generates juliac's object file output -inputfile = ARGS[1] -output_type = ARGS[2] -add_ccallables = ARGS[3] == "true" - # Run the verifier in the current world (before modifications), so that error # messages and types print in their usual way. Core.Compiler._verify_trim_world_age[] = Base.get_world_counter() @@ -189,13 +185,37 @@ end import Base.Experimental.entrypoint -let mod = Base.include(Base.__toplevel__, inputfile) - if !isa(mod, Module) - mod = Main - end +# for use as C main if needed +function _main(argc::Cint, argv::Ptr{Ptr{Cchar}})::Cint + args = ccall(:jl_set_ARGS, Any, (Cint, Ptr{Ptr{Cchar}}), argc, argv)::Vector{String} + return Main.main(args) +end + +let mod = Base.include(Main, ARGS[1]) Core.@latestworld - if output_type == "--output-exe" && isdefined(mod, :main) && !add_ccallables - entrypoint(mod.main, ()) + if ARGS[2] == "--output-exe" + have_cmain = false + if isdefined(Main, :main) + for m in methods(Main.main) + if isdefined(m, :ccallable) + # TODO: possibly check signature and return type + have_cmain = true + break + end + end + end + if !have_cmain + if Base.should_use_main_entrypoint() + if hasmethod(Main.main, Tuple{Vector{String}}) + entrypoint(_main, (Cint, Ptr{Ptr{Cchar}})) + Base._ccallable("main", Cint, Tuple{typeof(_main), Cint, Ptr{Ptr{Cchar}}}) + else + error("`@main` must accept a `Vector{String}` argument.") + end + else + error("To generate an executable a `@main` function must be defined.") + end + end end #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String)) #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) @@ -204,7 +224,7 @@ let mod = Base.include(Base.__toplevel__, inputfile) entrypoint(Base.wait_forever, ()) entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) entrypoint(Base.checktaskempty, ()) - if add_ccallables + if ARGS[3] == "true" ccall(:jl_add_ccallable_entrypoints, Cvoid, ()) end end diff --git a/src/gf.c b/src/gf.c index 436b4c84347dc..5a960c33ef503 100644 --- a/src/gf.c +++ b/src/gf.c @@ -4642,8 +4642,6 @@ JL_DLLEXPORT void jl_extern_c(jl_value_t *name, jl_value_t *declrt, jl_tupletype jl_error("@ccallable: function object must be a singleton"); // compute / validate return type - if (!jl_is_concrete_type(declrt) || jl_is_kind(declrt)) - jl_error("@ccallable: return type must be concrete and correspond to a C type"); if (!jl_type_mappable_to_c(declrt)) jl_error("@ccallable: return type doesn't correspond to a C type"); diff --git a/src/jlapi.c b/src/jlapi.c index 47d5b84fa5606..b27c01d9e0f1a 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -50,24 +50,28 @@ JL_DLLEXPORT int jl_is_initialized(void) * @param argc The number of command line arguments. * @param argv Array of command line arguments. */ -JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) +JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv) { - if (jl_core_module != NULL) { - jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); - if (args == NULL) { - args = jl_alloc_vec_any(0); - JL_GC_PUSH1(&args); + jl_array_t *args = NULL; + jl_value_t *vecstr = NULL; + JL_GC_PUSH2(&args, &vecstr); + if (jl_core_module != NULL) + args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); + if (args == NULL) { + vecstr = jl_apply_array_type((jl_value_t*)jl_string_type, 1); + args = jl_alloc_array_1d(vecstr, 0); + if (jl_core_module != NULL) jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); - JL_GC_POP(); - } - assert(jl_array_nrows(args) == 0); - jl_array_grow_end(args, argc); - int i; - for (i = 0; i < argc; i++) { - jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - jl_array_ptr_set(args, i, s); - } } + assert(jl_array_nrows(args) == 0); + jl_array_grow_end(args, argc); + int i; + for (i = 0; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_array_ptr_set(args, i, s); + } + JL_GC_POP(); + return (jl_value_t*)args; } JL_DLLEXPORT void jl_init_with_image_handle(void *handle) { diff --git a/src/julia.h b/src/julia.h index 4cb4f18d30c8e..c7b19fb8b4530 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2579,7 +2579,7 @@ uint64_t parse_heap_size_hint(const char *optarg, const char *option_name); // Set julia-level ARGS array according to the arguments provided in // argc/argv -JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv); +JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv); JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index fc0137dd4eab2..334b5466343d0 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -1,14 +1,10 @@ -module MyApp - using Libdl using Zstd_jll -Base.@ccallable function main()::Cint +function @main(args::Vector{String})::Cint println(Core.stdout, "Julia! Hello, world!") fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ()))) return 0 end - -end diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index fef25f9e8558f..579ef4e18de38 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,13 +1,10 @@ -module MyApp - world::String = "world!" const str = OncePerProcess{String}() do return "Hello, " * world end -Base.@ccallable function main()::Cint +function @main(args::Vector{String})::Cint println(Core.stdout, str()) + foreach(x->println(Core.stdout, x), args) return 0 end - -end diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index 0f6d452c25c9b..5d55ed62b03a8 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -6,7 +6,7 @@ bindir = dirname(ARGS[1]) let exe_suffix = splitext(Base.julia_exename())[2] hello_exe = joinpath(bindir, "hello" * exe_suffix) - @test readchomp(`$hello_exe`) == "Hello, world!" + @test readchomp(`$hello_exe arg1 arg2`) == "Hello, world!\n$hello_exe\narg1\narg2" @test filesize(hello_exe) < 2_000_000 basic_jll_exe = joinpath(bindir, "basic_jll" * exe_suffix) From c5df01807e3b3549b2669afdea11526b80049f3a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 22 May 2025 17:00:12 -0400 Subject: [PATCH 308/662] don't strip keyword argument names with --strip-metadata (#58412) These are used by `hasmethod`, and in any case including keyword argument symbols is unavoidable so there is no cost to keeping these. --- src/staticdata.c | 14 +++++++++----- test/cmdlineargs.jl | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 301968e6e2e26..5465cb1da0218 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2638,12 +2638,12 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED jl_gc_wb(m, jl_atomic_load_relaxed(&bindingkeyset2)); } -static void strip_slotnames(jl_array_t *slotnames) +static void strip_slotnames(jl_array_t *slotnames, int n) { // replace slot names with `?`, except unused_sym since the compiler looks at it jl_sym_t *questionsym = jl_symbol("?"); - int i, l = jl_array_len(slotnames); - for (i = 0; i < l; i++) { + int i; + for (i = 0; i < n; i++) { jl_value_t *s = jl_array_ptr_ref(slotnames, i); if (s != (jl_value_t*)jl_unused_sym) jl_array_ptr_set(slotnames, i, questionsym); @@ -2662,7 +2662,7 @@ static jl_value_t *strip_codeinfo_meta(jl_method_t *m, jl_value_t *ci_, jl_code_ else { ci = (jl_code_info_t*)ci_; } - strip_slotnames(ci->slotnames); + strip_slotnames(ci->slotnames, jl_array_len(ci->slotnames)); ci->debuginfo = jl_nulldebuginfo; jl_gc_wb(ci, ci->debuginfo); jl_value_t *ret = (jl_value_t*)ci; @@ -2736,7 +2736,11 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) } jl_array_t *slotnames = jl_uncompress_argnames(m->slot_syms); JL_GC_PUSH1(&slotnames); - strip_slotnames(slotnames); + int tostrip = jl_array_len(slotnames); + // for keyword methods, strip only nargs to keep the keyword names at the end for reflection + if (jl_tparam0(jl_unwrap_unionall(m->sig)) == jl_typeof(jl_kwcall_func)) + tostrip = m->nargs; + strip_slotnames(slotnames, tostrip); m->slot_syms = jl_compress_argnames(slotnames); jl_gc_wb(m, m->slot_syms); JL_GC_POP(); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 5fd8513849435..12b13c1984a5e 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1258,3 +1258,13 @@ end timeout = 120 @test parse(Int,read(`$exename --timeout-for-safepoint-straggler=$timeout -E "Base.JLOptions().timeout_for_safepoint_straggler_s"`, String)) == timeout end + +@testset "--strip-metadata" begin + mktempdir() do dir + @test success(pipeline(`$(Base.julia_cmd()) --strip-metadata -t1,0 --output-o $(dir)/sys.o.a -e 0`, stderr=stderr, stdout=stdout)) + if isfile(joinpath(dir, "sys.o.a")) + Base.Linking.link_image(joinpath(dir, "sys.o.a"), joinpath(dir, "sys.so")) + @test readchomp(`$(Base.julia_cmd()) -t1,0 -J $(dir)/sys.so -E 'hasmethod(sort, (Vector{Int},), (:dims,))'`) == "true" + end + end +end From e24c30a22d1bd94bfa84bcb27984a1900b7b28d3 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 22 May 2025 23:14:58 -0400 Subject: [PATCH 309/662] Add a new `make -C doc alldeps` target to install Documenter (#58504) This adds a new `alldeps` target to `make -C doc` that also downloads the Documenter.jl dependency. The motivation here is to make sure to have all dependencies necessary for running the doctests in an environment that may not have network connectivity (e.g. the OpenAI Codex sandbox). --- doc/Makefile | 3 +++ doc/make.jl | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/doc/Makefile b/doc/Makefile index 207adb850a49b..0bed117989bdb 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -34,6 +34,9 @@ deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" cp "$<" UnicodeData.txt +alldeps: deps + $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps + checksum-unicodedata: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" diff --git a/doc/make.jl b/doc/make.jl index ac158afffcaa6..b8e908a42c3ea 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -8,6 +8,10 @@ push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg Pkg.instantiate() +if "deps" in ARGS + exit() +end + using Documenter import LibGit2 From 2bd37fbbabec8aa178cb32ef6f89b0fef973be2b Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 23 May 2025 00:54:02 -0400 Subject: [PATCH 310/662] Docs: Update doctest info, enhance stdlib script (#58503) - Revise `doc/README.md` to mention `JULIA_EXECUTABLE` usage with doctests, adding a warning regarding built-in stdlibs - Enhance `contrib/print_sorted_stdlibs.jl` script to add `--only-sysimg` option to accurately list standard libraries built into the system image. These modifications aim to provide clearer documentation and more effective tooling for contributors and AI agents wanting to run the doctests without building julia from scratch. Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- contrib/print_sorted_stdlibs.jl | 22 ++++++++++++++++++---- doc/README.md | 8 ++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/contrib/print_sorted_stdlibs.jl b/contrib/print_sorted_stdlibs.jl index 6bc2023c4f1cc..c4cf391efb623 100644 --- a/contrib/print_sorted_stdlibs.jl +++ b/contrib/print_sorted_stdlibs.jl @@ -12,12 +12,13 @@ function check_flag(flag) end if check_flag("--help") || check_flag("-h") - println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage]") + println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage] [--only-sysimg]") end # Allow users to ask for JLL or no JLLs exclude_jlls = check_flag("--exclude-jlls") exclude_sysimage = check_flag("--exclude-sysimage") +only_sysimage = check_flag("--only-sysimg") # Default to the `stdlib/vX.Y` directory STDLIB_DIR = get(ARGS, 1, joinpath(@__DIR__, "..", "usr", "share", "julia", "stdlib")) @@ -81,9 +82,19 @@ if exclude_jlls filter!(p -> !endswith(p, "_jll"), sorted_projects) end -if exclude_sysimage - loaded_modules = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) - filter!(p->!in(p, loaded_modules), sorted_projects) +if only_sysimage && exclude_sysimage + println(stderr, "Warning: --only-sysimg and --exclude-sysimage are mutually exclusive. Prioritizing --only-sysimg.") + exclude_sysimage = false +end + +if only_sysimage || exclude_sysimage + loaded_modules_set = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) + + if only_sysimage + filter!(p -> in(p, loaded_modules_set), sorted_projects) + else + filter!(p -> !in(p, loaded_modules_set), sorted_projects) + end end # Print out sorted projects, ready to be pasted into `sysimg.jl` @@ -92,6 +103,9 @@ println(" # Stdlibs sorted in dependency, then alphabetical, order by contrib if exclude_jlls println(" # Run with the `--exclude-jlls` option to filter out all JLL packages") end +if only_sysimage + println(" # Run with the `--only-sysimg` option to filter for only packages included in the system image") +end if exclude_sysimage println(" # Run with the `--exclude-sysimage` option to filter out all packages included in the system image") end diff --git a/doc/README.md b/doc/README.md index be5426018084d..d282485bc584a 100644 --- a/doc/README.md +++ b/doc/README.md @@ -27,3 +27,11 @@ $ make -C doc doctest=true ``` from the root directory. + +## Customizing Doctest Execution + +By default, doctests are run using the in-tree Julia executable. +This behavior can be changed by setting the `JULIA_EXECUTABLE` Makefile variable. + +> [!WARNING] +> Using a custom `JULIA_EXECUTABLE` will not pick up changes to docstrings for Base or any standard library built into the system image. To see the list of standard libraries that are part of the system image, you can run the `contrib/print_sorted_stdlibs.jl` script (e.g., `julia contrib/print_sorted_stdlibs.jl --only-sysimg`). From c9f6903da83136730f67ccb5e01af1616062f15e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 23 May 2025 03:33:25 -0400 Subject: [PATCH 311/662] Revise jldoctest best practices --- doc/make.jl | 1 + doc/src/devdocs/jldoctests.md | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 doc/src/devdocs/jldoctests.md diff --git a/doc/make.jl b/doc/make.jl index b8e908a42c3ea..9d547187ad594 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -217,6 +217,7 @@ DevDocs = [ "devdocs/functions.md", "devdocs/cartesian.md", "devdocs/meta.md", + "devdocs/jldoctests.md", "devdocs/subarrays.md", "devdocs/isbitsunionarrays.md", "devdocs/sysimg.md", diff --git a/doc/src/devdocs/jldoctests.md b/doc/src/devdocs/jldoctests.md new file mode 100644 index 0000000000000..07f2ec4c75f81 --- /dev/null +++ b/doc/src/devdocs/jldoctests.md @@ -0,0 +1,85 @@ +# Best practices for jldoctests +This page describes how to write and maintain `jldoctest` blocks in the documentation. Following these guidelines helps keep doctests reliable and easy to read. + + +## Filters + +Use `filter =` whenever output contains text that might vary across runs. +The documentation relies on several recurring patterns: + +- `r"int.jl:\\d+"` — remove line numbers from introspection macros. +- `r"Stacktrace:(\\n \\[0-9]+\\].*)*"` — hide stack traces when illustrating + errors. +- `r"Closest candidates.*\\n .*"` — skip the method suggestions printed by + `MethodError`. +- `r"@ .*"` — strip file locations from the output of `methods` or + `@which`. +- `r"\\@world\\(MyStruct, \\d+:\\d+\\)"` — filter world age numbers. +- `r"with \\d+ methods"` — ignore method counts when redefining functions. +- `r"[0-9\\.]+ seconds \\(.*?\\)"` — remove timing output with memory + information. +- `r"[0-9\\.]+ seconds"` — remove simple timing results. +- `r"[0-9\\.]+"` — filter digits from names such as anonymous functions. +- `r"([A-B] [0-5])"` and `r"[A-B] [X-Z] [0-5]"` — account for non-deterministic + process output. +- `r"(world\\nhello|hello\\nworld)"` — allow either ordering of interleaved + output. + +If none of these match your situation, craft a regular expression that +removes the varying text. Using filters keeps doctests stable across +platforms and Julia versions. + +## Setup code + +Small setup expressions may be placed inline using the `setup =` option: + +```` +```jldoctest; setup = :(using InteractiveUtils) +... +``` +```` + +For longer setup code or if multiple blocks require the same environment, use the +`DocTestSetup` meta block: + +```` +```@meta +DocTestSetup = :(import Random; Random.seed!(1234)) +``` +```` + +and disable it afterwards with + +```` +```@meta +DocTestSetup = nothing +``` +```` + +## Maintaining state between snippets + +Related doctest blocks can share state by giving them the same label after the +`jldoctest` keyword. The manual uses this pattern to demonstrate mutation: + +```` +```jldoctest mutation_vs_rebind +julia> a = [1,2,3] +... +``` +```` + +and later + +```` +```jldoctest mutation_vs_rebind +julia> a[1] = 42 +... +``` +```` + +Blocks with the same name execute sequentially during doctesting, so variables +created in the first block remain available in the following ones. + +When a snippet needs to preserve its result for later examples, give it a label +and reuse that label. This avoids repeating setup code and mirrors a REPL +session more closely. From d333d8c7926c2ccd0eeb9f9eb3846088bee7f180 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 26 May 2025 22:39:04 +0000 Subject: [PATCH 312/662] Rearrange CONTRIBUTING.md into "Contributor's Guide" devdocs chapter This moves the various sections of CONTRIBUTING.md into their own pages in a new "Contributor's Guide" chapter of the devdocs. I think this organization makes more sense as CONTRIBUTING.md is a bit of a weird mix, between widely applicable things (how to sign up to GitHub) and very niche things (e.g. how to prepare a backport). I think this also sets the stage for writing more specific documentation (e.g. my new jldoctest page) that would have been far too in-depth to put directly into CONTRIBUTING.md. --- CONTRIBUTING.md | 332 +----------------- doc/make.jl | 10 +- doc/src/devdocs/contributing/code-changes.md | 112 ++++++ doc/src/devdocs/contributing/documentation.md | 87 +++++ doc/src/devdocs/contributing/formatting.md | 22 ++ doc/src/devdocs/contributing/git-workflow.md | 19 + .../devdocs/{ => contributing}/jldoctests.md | 4 +- .../devdocs/contributing/patch-releases.md | 45 +++ doc/src/devdocs/contributing/tests.md | 19 + 9 files changed, 323 insertions(+), 327 deletions(-) create mode 100644 doc/src/devdocs/contributing/code-changes.md create mode 100644 doc/src/devdocs/contributing/documentation.md create mode 100644 doc/src/devdocs/contributing/formatting.md create mode 100644 doc/src/devdocs/contributing/git-workflow.md rename doc/src/devdocs/{ => contributing}/jldoctests.md (98%) create mode 100644 doc/src/devdocs/contributing/patch-releases.md create mode 100644 doc/src/devdocs/contributing/tests.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c01890212df70..475294baaa00d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,25 +4,6 @@ Hi! If you are new to the Julia community: welcome, and thanks for trying Julia. If you are already familiar with Julia itself, this blog post by Katharine Hyatt on [Making your first Julia pull request](https://kshyatt.github.io/post/firstjuliapr/) is a great way to get started. - -# Table of Contents - -1. [Learning Julia](#learning-julia) -2. [Filing an issue](#filing-an-issue) - - [Before filing an issue](#before-filing-an-issue) - - [How to file a bug report](#how-to-file-a-bug-report) -3. [Submitting contributions](#submitting-contributions) - - [Contributor Checklist](#contributor-checklist) - - [Writing tests](#writing-tests) - - [Improving documentation](#improving-documentation) - - [Contributing to core functionality or base libraries](#contributing-to-core-functionality-or-base-libraries) - - [Contributing to the standard library](#contributing-to-the-standard-library) - - [Contributing to patch releases](#contributing-to-patch-releases) - - [Code Formatting Guidelines](#code-formatting-guidelines) - - [Git Recommendations For Pull Requests](#git-recommendations-for-pull-requests) -4. [Resources](#resources) - - ## Learning Julia [The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike. @@ -73,315 +54,18 @@ A useful bug report filed as a GitHub issue provides information about how to re * Review discussions on the [Julia Discourse forum](https://discourse.julialang.org). -* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below. - * Relax and have fun! -### Writing tests - -There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it. - -1. Go visit https://codecov.io/github/JuliaLang/julia. - -2. Browse through the source files and find some untested functionality (highlighted in red) that you think you might be able to write a test for. - -3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/v1/stdlib/Test/ may be helpful in explaining how the testing infrastructure works. - -4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace. - -5. Submit the test as a pull request (PR). - -* Code for the buildbot configuration is maintained at: https://github.com/staticfloat/julia-buildbot -* You can see the current buildbot setup at: https://build.julialang.org/builders -* [Issue 9493](https://github.com/JuliaLang/julia/issues/9493) and [issue 11885](https://github.com/JuliaLang/julia/issues/11885) have more detailed discussion on code coverage. - -Code coverage shows functionality that still needs "proof of concept" tests. These are important, as are tests for tricky edge cases, such as converting between integer types when the number to convert is near the maximum of the range of one of the integer types. Even if a function already has some coverage on Codecov, it may still benefit from tests for edge cases. - -### Improving documentation - -*By contributing documentation to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* - -Julia's documentation source files are stored in the `doc/` directory and all docstrings are found in `base/`. Like everything else these can be modified using `git`. Documentation is built with [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl), which uses Markdown syntax. The HTML documentation can be built locally by running - -``` -make docs -``` - -from Julia's root directory. This will rebuild the Julia system image, then install or update the package dependencies required to build the documentation, and finally build the HTML documentation and place the resulting files in `doc/_build/html/`. - -> **Note** -> -> When making changes to any of Julia's documentation it is recommended that you run `make docs` to check that your changes are valid and do not produce any errors before opening a pull request. - -Below are outlined the three most common types of documentation changes and the steps required to perform them. Please note that the following instructions do not cover the full range of features provided by Documenter.jl. Refer to [Documenter's documentation](https://juliadocs.github.io/Documenter.jl/stable) if you encounter anything that is not covered by the sections below. - -#### Modifying files in `doc/src/` - -Most of the source text for the Julia Manual is located in `doc/src/`. To update or add new text to any one of the existing files the following steps should be followed: - -1. update the text in whichever `.md` files are applicable; -2. run `make docs` from the root directory; -3. check the output in `doc/_build/html/` to make sure the changes are correct; -4. commit your changes and open a pull request. - -> **Note** -> -> The contents of `doc/_build/` does **not** need to be committed when you make changes. - -To add a **new file** to `doc/src/` rather than updating a file replace step `1` above with - -1. add the file to the appropriate subdirectory in `doc/src/` and also add the file path to the `PAGES` vector in `doc/make.jl`. - -#### Modifying an existing docstring in `base/` - -All docstrings are written inline above the methods or types they are associated with and can be found by clicking on the `source` link that appears below each docstring in the HTML file. The steps needed to make a change to an existing docstring are listed below: - -1. find the docstring in `base/`; -2. update the text in the docstring; -3. run `make docs` from the root directory; -4. check the output in `doc/_build/html/` to make sure the changes are correct; -5. commit your changes and open a pull request. - -#### Adding a new docstring to `base/` - -The steps required to add a new docstring are listed below: - -1. find a suitable definition in `base/` that the docstring will be most applicable to; -2. add a docstring above the definition; -3. find a suitable `@docs` code block in one of the `doc/src/stdlib/` files where you would like the docstring to appear; -4. add the name of the definition to the `@docs` code block. For example, with a docstring added to a function `bar` - - ```julia - "..." - function bar(args...) - # ... - end - ``` - - you would add the name `bar` to a `@docs` block in `doc/src/stdlib/` - - ```@docs - foo - bar # <-- Added this one. - baz - ``` - -5. run `make docs` from the root directory; -6. check the output in `doc/_build/html` to make sure the changes are correct; -7. commit your changes and open a pull request. - -#### Doctests - -Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`. - - ```jldoctest - julia> uppercase("Docstring test") - "DOCSTRING TEST" - ``` - -A doctest needs to match an interactive REPL including the `julia>` prompt. It is recommended to add the header `# Examples` above the doctests. - -To run doctests you need to run `make -C doc doctest=true` from the root directory. You can use `make -C doc doctest=true revise=true` if you are modifying the doctests and don't want to rebuild Julia after each change (see details below about the Revise.jl workflow). - -#### News-worthy changes - -For new functionality and other substantial changes, add a brief summary to `NEWS.md`. The news item should cross reference the pull request (PR) parenthetically, in the form `([#pr])`. To add the PR reference number, first create the PR, then push an additional commit updating `NEWS.md` with the PR reference number. We periodically run `./julia doc/NEWS-update.jl` from the julia directory to update the cross-reference links, but this should not be done in a typical PR in order to avoid conflicting commits. - -#### Annotations for new features, deprecations and behavior changes - -API additions and deprecations, and minor behavior changes are allowed in minor version releases. -For documented features that are part of the public API, a compatibility note should be added into -the manual or the docstring. It should state the Julia minor version that changed the behavior -and have a brief message describing the change. - -At the moment, this should always be done with the following `compat` admonition -(so that it would be possible to programmatically find the annotations in the future): - - ``` - !!! compat "Julia 1.X" - This method was added in Julia 1.X. - ``` - -### Contributing to core functionality or base libraries - -*By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* - -The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests (PR). - -Issues and pull requests should have self explanatory titles such that they can be understood from the list of PRs and Issues. -i.e. `Add {feature}` and `Fix {bug}` are good, `Fix #12345. Corrects the bug.` is bad. - -You can make pull requests for incomplete features to get code review. The convention is to open these as draft PRs and prefix -the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready -for merging. This will prevent accidental merging of work that is in progress. - -Note: These instructions are for adding to or improving functionality in the base library. Before getting started, it can be helpful to discuss the proposed changes or additions on the [Julia Discourse forum](https://discourse.julialang.org) or in a GitHub issue---it's possible your proposed change belongs in a package rather than the core language. Also, keep in mind that changing stuff in the base can potentially break a lot of things. Finally, because of the time required to build Julia, note that it's usually faster to develop your code in stand-alone files, get it working, and then migrate it into the base libraries. - -Add new code to Julia's base libraries as follows (this is the "basic" approach; see a more efficient approach in the next section): - - 1. Edit the appropriate file in the `base/` directory, or add new files if necessary. Create tests for your functionality and add them to files in the `test/` directory. If you're editing C or Scheme code, most likely it lives in `src/` or one of its subdirectories, although some aspects of Julia's REPL initialization live in `cli/`. - - 2. Add any new files to `sysimg.jl` in order to build them into the Julia system image. - - 3. Add any necessary export symbols in `exports.jl`. - - 4. Include your tests in `test/Makefile` and `test/choosetests.jl`. - -Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git). - -#### Running specific tests - -There are `make` targets for running specific tests: - - make test-bitarray - -You can also use the `runtests.jl` script, e.g. to run `test/bitarray.jl` and `test/math.jl`: - - ./usr/bin/julia test/runtests.jl bitarray math - -#### Modifying base more efficiently with Revise.jl - -[Revise](https://github.com/timholy/Revise.jl) is a package that -tracks changes in source files and automatically updates function -definitions in your running Julia session. Using it, you can make -extensive changes to Base without needing to rebuild in order to test -your changes. - -Here is the standard procedure: - -1. If you are planning changes to any types or macros, make those - changes and build julia using `make`. (This is - necessary because `Revise` cannot handle changes to type - definitions or macros.) Unless it's - required to get Julia to build, you do not have to add any - functionality based on the new types, just the type definitions - themselves. - -2. Start a Julia REPL session. Then issue the following commands: - -```julia -using Revise # if you aren't launching it in your `.julia/config/startup.jl` -Revise.track(Base) -``` - -3. Edit files in `base/`, save your edits, and test the - functionality. - -If you need to restart your Julia session, just start at step 2 above. -`Revise.track(Base)` will note any changes from when Julia was last -built and incorporate them automatically. You only need to rebuild -Julia if you made code-changes that Revise cannot handle. - -For convenience, there are also `test-revise-*` targets for every [`test-*` -target](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#running-specific-tests) that use Revise to load any modifications to Base into the current -system image before running the corresponding test. This can be useful as a shortcut -on the command line (since tests aren't always designed to be run outside the -runtest harness). - -### Contributing to the standard library - -The standard library (stdlib) packages are baked into the Julia system image. -When running the ordinary test workflow on the stdlib packages, the system image -version overrides the version you are developing. -To test stdlib packages, you can do the following steps: - -1. Edit the UUID field of the `Project.toml` in the stdlib package -2. Change the current directory to the directory of the stdlib you are developing -3. Start julia with `julia --project=.` -4. You can now test the package by running `pkg> test` in Pkg mode. - -Because you changed the UUID, the package manager treats the stdlib package as -different from the one in the system image, and the system image version will -not override the package. - -Be sure to change the UUID value back before making the pull request. - -### Contributing to patch releases - -The process of [creating a patch release](https://docs.julialang.org/en/v1/devdocs/build/distributing/#Point-releasing-101) is roughly as follows: - -1. Create a new branch (e.g. `backports-release-1.10`) against the relevant minor release - branch (e.g. `release-1.10`). Usually a corresponding pull request is created as well. - -2. Add commits, nominally from `master` (hence "backports"), to that branch. - See below for more information on this process. - -3. Run the [BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) benchmark - suite and [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) package ecosystem - exerciser against that branch. Nominally BaseBenchmarks.jl and PkgEval.jl are - invoked via [Nanosoldier.jl](https://github.com/JuliaCI/Nanosoldier.jl) from - the pull request associated with the backports branch. Fix any issues. - -4. Once all test and benchmark reports look good, merge the backports branch into - the corresponding release branch (e.g. merge `backports-release-1.10` into - `release-1.10`). - -5. Open a pull request that bumps the version of the relevant minor release to the - next patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37718). - -6. Ping `@JuliaLang/releases` to tag the patch release and update the website. - -7. Open a pull request that bumps the version of the relevant minor release to the - next prerelease patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37724). - -Step 2 above, i.e. backporting commits to the `backports-release-X.Y` branch, has largely -been automated via [`Backporter`](https://github.com/KristofferC/Backporter): Backporter -searches for merged pull requests with the relevant `backport-X.Y` tag, and attempts to -cherry-pick the commits from those pull requests onto the `backports-release-X.Y` branch. -Some commits apply successfully without intervention, others not so much. The latter -commits require "manual" backporting, with which help is generally much appreciated. -Backporter generates a report identifying those commits it managed to backport automatically -and those that require manual backporting; this report is usually copied into the first -post of the pull request associated with `backports-release-X.Y` and maintained as -additional commits are automatically and/or manually backported. - -When contributing a manual backport, if you have the necessary permissions, please push the -backport directly to the `backports-release-X.Y` branch. If you lack the relevant -permissions, please open a pull request against the `backports-release-X.Y` branch with the -manual backport. Once the manual backport is live on the `backports-release-X.Y` branch, -please remove the `backport-X.Y` tag from the originating pull request for the commits. - -### Code Formatting Guidelines - -#### General Formatting Guidelines for Julia code contributions - - - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). - - use whitespace to make the code more readable - - no whitespace at the end of a line (trailing whitespace) - - comments are good, especially when they explain the algorithm - - try to adhere to a 92 character line length limit - - it is generally preferred to use ASCII operators and identifiers over - Unicode equivalents whenever possible - - in docstrings refer to the language as "Julia" and the executable as "`julia`" - -#### General Formatting Guidelines For C code contributions - - - 4 spaces per indentation level, no tabs - - space between `if` and `(` (`if (x) ...`) - - newline before opening `{` in function definitions - - `f(void)` for 0-argument function declarations - - newline between `}` and `else` instead of `} else {` - - if one part of an `if..else` chain uses `{ }` then all should - - no whitespace at the end of a line - -### Git Recommendations For Pull Requests - - - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes. - - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes. - - If any conflicts arise due to changes in Julia's `master`, prefer updating your pull request branch with `git rebase` versus `git merge` or `git pull`, since the latter will introduce merge commits that clutter the git history with noise that makes your changes more difficult to review. - - Descriptive commit messages are good. - - Using `git add -p` or `git add -i` can be useful to avoid accidentally committing unrelated changes. - - When linking to specific lines of code in discussion of an issue or pull request, hit the `y` key while viewing code on GitHub to reload the page with a URL that includes the specific version that you're viewing. That way any lines of code that you refer to will still make sense in the future, even if the content of the file changes. - - Whitespace can be automatically removed from existing commits with `git rebase`. - - To remove whitespace for the previous commit, run - `git rebase --whitespace=fix HEAD~1`. - - To remove whitespace relative to the `master` branch, run - `git rebase --whitespace=fix master`. - -#### Git Recommendations For Pull Request Reviewers +### Guidance for specific changes -- When merging, we generally like `squash+merge`. Unless it is the rare case of a PR with carefully staged individual commits that you want in the history separately, in which case `merge` is acceptable, but usually prefer `squash+merge`. +The julia project maintains a more in-depth `Contributor's Guide` as part of our +developer documentation. Here you can find more in-depth guidance for how to write +specific kinds of changes. In particular, you want want to read: +- [How to contribute code changes](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/code-changes.md) +- [How to contribute additional tests](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/code-changes.md) +- [How to work on documentation](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/documentation.md) +- [Workflow tips for working with git](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/git-workflow.md) ## Resources diff --git a/doc/make.jl b/doc/make.jl index 9d547187ad594..bf56e8ae90d27 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -217,7 +217,6 @@ DevDocs = [ "devdocs/functions.md", "devdocs/cartesian.md", "devdocs/meta.md", - "devdocs/jldoctests.md", "devdocs/subarrays.md", "devdocs/isbitsunionarrays.md", "devdocs/sysimg.md", @@ -257,6 +256,15 @@ DevDocs = [ "devdocs/build/arm.md", "devdocs/build/riscv.md", "devdocs/build/distributing.md", + ], + "Contributor's Guide" => [ + "devdocs/contributing/code-changes.md", + "devdocs/contributing/tests.md", + "devdocs/contributing/documentation.md", + "devdocs/contributing/jldoctest.md", + "devdocs/contributing/patch-releases.md", + "devdocs/contributing/formatting.md", + "devdocs/contributing/git-workflow.md", ] ] diff --git a/doc/src/devdocs/contributing/code-changes.md b/doc/src/devdocs/contributing/code-changes.md new file mode 100644 index 0000000000000..14dbd65b1b5ce --- /dev/null +++ b/doc/src/devdocs/contributing/code-changes.md @@ -0,0 +1,112 @@ +### Contributing to core functionality or base libraries + +*By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* + +The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests (PR). + +Issues and pull requests should have self explanatory titles such that they can be understood from the list of PRs and Issues. +i.e. `Add {feature}` and `Fix {bug}` are good, `Fix #12345. Corrects the bug.` is bad. + +You can make pull requests for incomplete features to get code review. The convention is to open these as draft PRs and prefix +the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready +for merging. This will prevent accidental merging of work that is in progress. + +Note: These instructions are for adding to or improving functionality in the base library. Before getting started, it can be helpful to discuss the proposed changes or additions on the [Julia Discourse forum](https://discourse.julialang.org) or in a GitHub issue---it's possible your proposed change belongs in a package rather than the core language. Also, keep in mind that changing stuff in the base can potentially break a lot of things. Finally, because of the time required to build Julia, note that it's usually faster to develop your code in stand-alone files, get it working, and then migrate it into the base libraries. + +Add new code to Julia's base libraries as follows (this is the "basic" approach; see a more efficient approach in the next section): + + 1. Edit the appropriate file in the `base/` directory, or add new files if necessary. Create tests for your functionality and add them to files in the `test/` directory. If you're editing C or Scheme code, most likely it lives in `src/` or one of its subdirectories, although some aspects of Julia's REPL initialization live in `cli/`. + + 2. Add any new files to `sysimg.jl` in order to build them into the Julia system image. + + 3. Add any necessary export symbols in `exports.jl`. + + 4. Include your tests in `test/Makefile` and `test/choosetests.jl`. + +Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git). + +#### Running specific tests + +There are `make` targets for running specific tests: + + make test-bitarray + +You can also use the `runtests.jl` script, e.g. to run `test/bitarray.jl` and `test/math.jl`: + + ./usr/bin/julia test/runtests.jl bitarray math + +#### Modifying base more efficiently with Revise.jl + +[Revise](https://github.com/timholy/Revise.jl) is a package that +tracks changes in source files and automatically updates function +definitions in your running Julia session. Using it, you can make +extensive changes to Base without needing to rebuild in order to test +your changes. + +Here is the standard procedure: + +1. If you are planning changes to any types or macros, make those + changes and build julia using `make`. (This is + necessary because `Revise` cannot handle changes to type + definitions or macros.) Unless it's + required to get Julia to build, you do not have to add any + functionality based on the new types, just the type definitions + themselves. + +2. Start a Julia REPL session. Then issue the following commands: + +```julia +using Revise # if you aren't launching it in your `.julia/config/startup.jl` +Revise.track(Base) +``` + +3. Edit files in `base/`, save your edits, and test the + functionality. + +If you need to restart your Julia session, just start at step 2 above. +`Revise.track(Base)` will note any changes from when Julia was last +built and incorporate them automatically. You only need to rebuild +Julia if you made code-changes that Revise cannot handle. + +For convenience, there are also `test-revise-*` targets for every [`test-*` +target](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#running-specific-tests) that use Revise to load any modifications to Base into the current +system image before running the corresponding test. This can be useful as a shortcut +on the command line (since tests aren't always designed to be run outside the +runtest harness). + +### Contributing to the standard library + +The standard library (stdlib) packages are baked into the Julia system image. +When running the ordinary test workflow on the stdlib packages, the system image +version overrides the version you are developing. +To test stdlib packages, you can do the following steps: + +1. Edit the UUID field of the `Project.toml` in the stdlib package +2. Change the current directory to the directory of the stdlib you are developing +3. Start julia with `julia --project=.` +4. You can now test the package by running `pkg> test` in Pkg mode. + +Because you changed the UUID, the package manager treats the stdlib package as +different from the one in the system image, and the system image version will +not override the package. + +Be sure to change the UUID value back before making the pull request. + +#### News-worthy changes + +For new functionality and other substantial changes, add a brief summary to `NEWS.md`. The news item should cross reference the pull request (PR) parenthetically, in the form `([#pr])`. To add the PR reference number, first create the PR, then push an additional commit updating `NEWS.md` with the PR reference number. We periodically run `./julia doc/NEWS-update.jl` from the julia directory to update the cross-reference links, but this should not be done in a typical PR in order to avoid conflicting commits. + +#### Annotations for new features, deprecations and behavior changes + +API additions and deprecations, and minor behavior changes are allowed in minor version releases. +For documented features that are part of the public API, a compatibility note should be added into +the manual or the docstring. It should state the Julia minor version that changed the behavior +and have a brief message describing the change. + +At the moment, this should always be done with the following `compat` admonition +(so that it would be possible to programmatically find the annotations in the future): + + ``` + !!! compat "Julia 1.X" + This method was added in Julia 1.X. + ``` diff --git a/doc/src/devdocs/contributing/documentation.md b/doc/src/devdocs/contributing/documentation.md new file mode 100644 index 0000000000000..3c7b7de6bd0c1 --- /dev/null +++ b/doc/src/devdocs/contributing/documentation.md @@ -0,0 +1,87 @@ +### Improving documentation + +*By contributing documentation to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* + +Julia's documentation source files are stored in the `doc/` directory and all docstrings are found in `base/`. Like everything else these can be modified using `git`. Documentation is built with [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl), which uses Markdown syntax. The HTML documentation can be built locally by running + +``` +make docs +``` + +from Julia's root directory. This will rebuild the Julia system image, then install or update the package dependencies required to build the documentation, and finally build the HTML documentation and place the resulting files in `doc/_build/html/`. + +> **Note** +> +> When making changes to any of Julia's documentation it is recommended that you run `make docs` to check that your changes are valid and do not produce any errors before opening a pull request. + +Below are outlined the three most common types of documentation changes and the steps required to perform them. Please note that the following instructions do not cover the full range of features provided by Documenter.jl. Refer to [Documenter's documentation](https://juliadocs.github.io/Documenter.jl/stable) if you encounter anything that is not covered by the sections below. + +#### Modifying files in `doc/src/` + +Most of the source text for the Julia Manual is located in `doc/src/`. To update or add new text to any one of the existing files the following steps should be followed: + +1. update the text in whichever `.md` files are applicable; +2. run `make docs` from the root directory; +3. check the output in `doc/_build/html/` to make sure the changes are correct; +4. commit your changes and open a pull request. + +> **Note** +> +> The contents of `doc/_build/` does **not** need to be committed when you make changes. + +To add a **new file** to `doc/src/` rather than updating a file replace step `1` above with + +1. add the file to the appropriate subdirectory in `doc/src/` and also add the file path to the `PAGES` vector in `doc/make.jl`. + +#### Modifying an existing docstring in `base/` + +All docstrings are written inline above the methods or types they are associated with and can be found by clicking on the `source` link that appears below each docstring in the HTML file. The steps needed to make a change to an existing docstring are listed below: + +1. find the docstring in `base/`; +2. update the text in the docstring; +3. run `make docs` from the root directory; +4. check the output in `doc/_build/html/` to make sure the changes are correct; +5. commit your changes and open a pull request. + +#### Adding a new docstring to `base/` + +The steps required to add a new docstring are listed below: + +1. find a suitable definition in `base/` that the docstring will be most applicable to; +2. add a docstring above the definition; +3. find a suitable `@docs` code block in one of the `doc/src/stdlib/` files where you would like the docstring to appear; +4. add the name of the definition to the `@docs` code block. For example, with a docstring added to a function `bar` + + ```julia + "..." + function bar(args...) + # ... + end + ``` + + you would add the name `bar` to a `@docs` block in `doc/src/stdlib/` + + ```@docs + foo + bar # <-- Added this one. + baz + ``` + +5. run `make docs` from the root directory; +6. check the output in `doc/_build/html` to make sure the changes are correct; +7. commit your changes and open a pull request. + +#### Doctests + +Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`. + + ```jldoctest + julia> uppercase("Docstring test") + "DOCSTRING TEST" + ``` + +A doctest needs to match an interactive REPL including the `julia>` prompt. It is recommended to add the header `# Examples` above the doctests. + +See the documentation of [writing jldoctests](@ref writing-jldoctests) for best +practices on how to write doctests for common scenarios and the `doc/README.md` +file for how to run the doctests. diff --git a/doc/src/devdocs/contributing/formatting.md b/doc/src/devdocs/contributing/formatting.md new file mode 100644 index 0000000000000..e26bf865876f7 --- /dev/null +++ b/doc/src/devdocs/contributing/formatting.md @@ -0,0 +1,22 @@ +### Code Formatting Guidelines + +#### General Formatting Guidelines for Julia code contributions + + - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). + - use whitespace to make the code more readable + - no whitespace at the end of a line (trailing whitespace) + - comments are good, especially when they explain the algorithm + - try to adhere to a 92 character line length limit + - it is generally preferred to use ASCII operators and identifiers over + Unicode equivalents whenever possible + - in docstrings refer to the language as "Julia" and the executable as "`julia`" + +#### General Formatting Guidelines For C code contributions + + - 4 spaces per indentation level, no tabs + - space between `if` and `(` (`if (x) ...`) + - newline before opening `{` in function definitions + - `f(void)` for 0-argument function declarations + - newline between `}` and `else` instead of `} else {` + - if one part of an `if..else` chain uses `{ }` then all should + - no whitespace at the end of a line diff --git a/doc/src/devdocs/contributing/git-workflow.md b/doc/src/devdocs/contributing/git-workflow.md new file mode 100644 index 0000000000000..85fe9fa5e63fd --- /dev/null +++ b/doc/src/devdocs/contributing/git-workflow.md @@ -0,0 +1,19 @@ +# Git workflow recommendations + +### Git Recommendations For Pull Requests + + - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes. + - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes. + - If any conflicts arise due to changes in Julia's `master`, prefer updating your pull request branch with `git rebase` versus `git merge` or `git pull`, since the latter will introduce merge commits that clutter the git history with noise that makes your changes more difficult to review. + - Descriptive commit messages are good. + - Using `git add -p` or `git add -i` can be useful to avoid accidentally committing unrelated changes. + - When linking to specific lines of code in discussion of an issue or pull request, hit the `y` key while viewing code on GitHub to reload the page with a URL that includes the specific version that you're viewing. That way any lines of code that you refer to will still make sense in the future, even if the content of the file changes. + - Whitespace can be automatically removed from existing commits with `git rebase`. + - To remove whitespace for the previous commit, run + `git rebase --whitespace=fix HEAD~1`. + - To remove whitespace relative to the `master` branch, run + `git rebase --whitespace=fix master`. + +#### Git Recommendations For Pull Request Reviewers + +- When merging, we generally like `squash+merge`. Unless it is the rare case of a PR with carefully staged individual commits that you want in the history separately, in which case `merge` is acceptable, but usually prefer `squash+merge`. diff --git a/doc/src/devdocs/jldoctests.md b/doc/src/devdocs/contributing/jldoctests.md similarity index 98% rename from doc/src/devdocs/jldoctests.md rename to doc/src/devdocs/contributing/jldoctests.md index 07f2ec4c75f81..0542e461c265f 100644 --- a/doc/src/devdocs/jldoctests.md +++ b/doc/src/devdocs/contributing/jldoctests.md @@ -1,6 +1,6 @@ -# Best practices for jldoctests -This page describes how to write and maintain `jldoctest` blocks in the documentation. Following these guidelines helps keep doctests reliable and easy to read. +# [Writing jldoctests](@id writing-jldoctests) +This page describes how to write and maintain `jldoctest` blocks in the documentation. Following these guidelines helps keep doctests reliable and easy to read. ## Filters diff --git a/doc/src/devdocs/contributing/patch-releases.md b/doc/src/devdocs/contributing/patch-releases.md new file mode 100644 index 0000000000000..dcbef6056fcce --- /dev/null +++ b/doc/src/devdocs/contributing/patch-releases.md @@ -0,0 +1,45 @@ + +### Contributing to patch releases + +The process of [creating a patch release](https://docs.julialang.org/en/v1/devdocs/build/distributing/#Point-releasing-101) is roughly as follows: + +1. Create a new branch (e.g. `backports-release-1.10`) against the relevant minor release + branch (e.g. `release-1.10`). Usually a corresponding pull request is created as well. + +2. Add commits, nominally from `master` (hence "backports"), to that branch. + See below for more information on this process. + +3. Run the [BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) benchmark + suite and [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) package ecosystem + exerciser against that branch. Nominally BaseBenchmarks.jl and PkgEval.jl are + invoked via [Nanosoldier.jl](https://github.com/JuliaCI/Nanosoldier.jl) from + the pull request associated with the backports branch. Fix any issues. + +4. Once all test and benchmark reports look good, merge the backports branch into + the corresponding release branch (e.g. merge `backports-release-1.10` into + `release-1.10`). + +5. Open a pull request that bumps the version of the relevant minor release to the + next patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37718). + +6. Ping `@JuliaLang/releases` to tag the patch release and update the website. + +7. Open a pull request that bumps the version of the relevant minor release to the + next prerelease patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37724). + +Step 2 above, i.e. backporting commits to the `backports-release-X.Y` branch, has largely +been automated via [`Backporter`](https://github.com/KristofferC/Backporter): Backporter +searches for merged pull requests with the relevant `backport-X.Y` tag, and attempts to +cherry-pick the commits from those pull requests onto the `backports-release-X.Y` branch. +Some commits apply successfully without intervention, others not so much. The latter +commits require "manual" backporting, with which help is generally much appreciated. +Backporter generates a report identifying those commits it managed to backport automatically +and those that require manual backporting; this report is usually copied into the first +post of the pull request associated with `backports-release-X.Y` and maintained as +additional commits are automatically and/or manually backported. + +When contributing a manual backport, if you have the necessary permissions, please push the +backport directly to the `backports-release-X.Y` branch. If you lack the relevant +permissions, please open a pull request against the `backports-release-X.Y` branch with the +manual backport. Once the manual backport is live on the `backports-release-X.Y` branch, +please remove the `backport-X.Y` tag from the originating pull request for the commits. diff --git a/doc/src/devdocs/contributing/tests.md b/doc/src/devdocs/contributing/tests.md new file mode 100644 index 0000000000000..7df203b96ffb6 --- /dev/null +++ b/doc/src/devdocs/contributing/tests.md @@ -0,0 +1,19 @@ +### Writing tests + +There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it. + +1. Go visit https://codecov.io/github/JuliaLang/julia. + +2. Browse through the source files and find some untested functionality (highlighted in red) that you think you might be able to write a test for. + +3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/v1/stdlib/Test/ may be helpful in explaining how the testing infrastructure works. + +4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace. + +5. Submit the test as a pull request (PR). + +* Code for the buildbot configuration is maintained at: https://github.com/staticfloat/julia-buildbot +* You can see the current buildbot setup at: https://build.julialang.org/builders +* [Issue 9493](https://github.com/JuliaLang/julia/issues/9493) and [issue 11885](https://github.com/JuliaLang/julia/issues/11885) have more detailed discussion on code coverage. + +Code coverage shows functionality that still needs "proof of concept" tests. These are important, as are tests for tricky edge cases, such as converting between integer types when the number to convert is near the maximum of the range of one of the integer types. Even if a function already has some coverage on Codecov, it may still benefit from tests for edge cases. From 6fa002d99e2a2986d1343383e0ecb130fff6b608 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 26 May 2025 22:49:41 +0000 Subject: [PATCH 313/662] Fix typos --- CONTRIBUTING.md | 2 +- doc/make.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 475294baaa00d..8dbd456d35e8d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ developer documentation. Here you can find more in-depth guidance for how to wri specific kinds of changes. In particular, you want want to read: - [How to contribute code changes](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/code-changes.md) -- [How to contribute additional tests](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/code-changes.md) +- [How to contribute additional tests](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/tests.md) - [How to work on documentation](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/documentation.md) - [Workflow tips for working with git](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/git-workflow.md) diff --git a/doc/make.jl b/doc/make.jl index bf56e8ae90d27..5bf19a254a5cb 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -261,7 +261,7 @@ DevDocs = [ "devdocs/contributing/code-changes.md", "devdocs/contributing/tests.md", "devdocs/contributing/documentation.md", - "devdocs/contributing/jldoctest.md", + "devdocs/contributing/jldoctests.md", "devdocs/contributing/patch-releases.md", "devdocs/contributing/formatting.md", "devdocs/contributing/git-workflow.md", From 0a970d81aa9d5da08968084510cd54d532664f03 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 26 May 2025 23:57:20 -0400 Subject: [PATCH 314/662] Fix out-of-tree docs build (#58529) The `doc` build had only limited out-of-tree support. In particular, it was relying on creating in-tree `stdlib` symbolic links (due to a missing feature in Documenter), as well as pointing external stdlibs to the in-tree references rather than the out-of-tree ones. Fix this by building up the reading of the buildroot and splitting `@__DIR__` references appropriately, depending on whether they refer to the source or the output. Fixes #53039. --- doc/Makefile | 7 +++++++ doc/make.jl | 26 ++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 0bed117989bdb..9dc32d8013971 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -30,6 +30,13 @@ $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt: @mkdir -p "$(SRCCACHE)" $(JLDOWNLOAD) "$@" https://www.unicode.org/Public/$(UNICODE_DATA_VERSION)/ucd/UnicodeData.txt +# NEWS.md and stdlib are in-tree build artifacts - don't link them for oot builds. +DOC_FILES=$(filter-out NEWS.md stdlib,$(notdir $(wildcard $(SRCDIR)/src/*))) +src/%: + @mkdir -p src + ln -s $(SRCDIR)/src/$* $@ +src: $(addprefix src/,$(DOC_FILES)) + deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" cp "$<" UnicodeData.txt diff --git a/doc/make.jl b/doc/make.jl index b8e908a42c3ea..fd81ce877d905 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -1,9 +1,13 @@ +let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) + global const buildrootdoc = i === nothing ? (@__DIR__) : joinpath(first(match(r, ARGS[i]).captures), "doc") +end + # Install dependencies needed to build the documentation. Base.ACTIVE_PROJECT[] = nothing empty!(LOAD_PATH) push!(LOAD_PATH, @__DIR__, "@stdlib") empty!(DEPOT_PATH) -push!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, joinpath(buildrootdoc, "deps")) push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg Pkg.instantiate() @@ -26,7 +30,7 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest) const STDLIB_DOCS = [] const STDLIB_DIR = Sys.STDLIB const EXT_STDLIB_DOCS = ["Pkg"] -cd(joinpath(@__DIR__, "src")) do +cd(joinpath(buildrootdoc, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) @@ -70,7 +74,8 @@ function parse_stdlib_version_file(path) end # This generates the value that will be passed to the `remotes` argument of makedocs(), # by looking through all *.version files in stdlib/. -documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib")) +documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib")), + stdlib_build_dir = joinpath(buildrootdoc, "..", "stdlib") # Get a list of all *.version files in stdlib/.. version_files = filter(readdir(stdlib_dir)) do fname isfile(joinpath(stdlib_dir, fname)) && endswith(fname, ".version") @@ -97,7 +102,7 @@ documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", " versionfile[sha_key] end # Construct the absolute (local) path to the stdlib package's root directory - package_root_dir = joinpath(stdlib_dir, "$(package)-$(package_sha)") + package_root_dir = joinpath(stdlib_build_dir, "$(package)-$(package_sha)") # Documenter needs package_root_dir to exist --- it's just a sanity check it does on the remotes= keyword. # In normal (local) builds, this will be the case, since the Makefiles will have unpacked the standard # libraries. However, on CI we do this thing where we actually build docs in a clean worktree, just @@ -127,7 +132,7 @@ function generate_markdown(basename) @assert length(splitted) == 2 replaced_links = replace(splitted[1], r"\[\#([0-9]*?)\]" => s"[#\g<1>](https://github.com/JuliaLang/julia/issues/\g<1>)") write( - joinpath(@__DIR__, "src", "$basename.md"), + joinpath(buildrootdoc, "src", "$basename.md"), """ ```@meta EditURL = "https://github.com/JuliaLang/julia/blob/master/$basename.md" @@ -281,7 +286,7 @@ end const use_revise = "revise=true" in ARGS if use_revise - let revise_env = joinpath(@__DIR__, "deps", "revise") + let revise_env = joinpath(buildrootdoc, "deps", "revise") Pkg.activate(revise_env) Pkg.add("Revise"; preserve=Pkg.PRESERVE_NONE) Base.ACTIVE_PROJECT[] = nothing @@ -351,10 +356,6 @@ DocMeta.setdocmeta!( recursive=true, warn=false, ) -let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) - global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures) -end - const format = if render_pdf Documenter.LaTeX( platform = "texplatform=docker" in ARGS ? "docker" : "native" @@ -377,8 +378,9 @@ else ) end -const output_path = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en") +const output_path = joinpath(buildrootdoc, "_build", (render_pdf ? "pdf" : "html"), "en") makedocs( + source = joinpath(buildrootdoc, "src"), build = output_path, modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], clean = true, @@ -479,7 +481,7 @@ if "deploy" in ARGS deploydocs( repo = "github.com/JuliaLang/docs.julialang.org.git", deploy_config = BuildBotConfig(), - target = joinpath(buildroot, "doc", "_build", "html", "en"), + target = joinpath(buildrootdoc, "_build", "html", "en"), dirname = "en", devurl = devurl, versions = Versions(["v#.#", devurl => devurl]), From 8c6e4ad4a223b96685609eaaa7481ea523e693e2 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 27 May 2025 08:52:30 -0400 Subject: [PATCH 315/662] doc: Get stdlib dir from make system rather than executable (#58530) The `make.jl` docsystem driver was using `Sys.STDLIB` as the canonical list and location of stdlibs. When `make.jl` is invoked as part of the make system (e.g. `make -C doc`), I think it makes more sense to use make's idea of where the stdlibs are. Of course, if you're using a just-built julia to create the docs, these two notions are identical. However, the docsystem build supports (explicitly via the `JULIA_EXECUTABLE` option), being built with an external julia executable. In this case, we should still use the stdlib source from in tree (as we do with the ordinary base sources). A primary motivation is to let AI agents run the doctests before they submit PRs without having to do a whole julia build. --- doc/Makefile | 4 ++-- doc/make.jl | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 9dc32d8013971..ccb6af38c9d80 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -21,9 +21,9 @@ help: @echo "To fix outdated doctests, use 'make doctest=fix'" @echo "To run doctests using Revise (to test changes without rebuilding the sysimage), use 'make doctest=true revise=true'" - +VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ - texplatform=$(texplatform) revise=$(revise) + texplatform=$(texplatform) revise=$(revise) stdlibdir=$(call cygpath_w,$(build_datarootdir)/julia/stdlib/$(VERSDIR)/) UNICODE_DATA_VERSION=13.0.0 $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt: diff --git a/doc/make.jl b/doc/make.jl index fd81ce877d905..b974ac30e7e75 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -1,5 +1,24 @@ +# Get the buildroot and stdlibdir from the make environment to make sure we're +# generating docs for the current julia source tree, regardless of what julia +# executable we're using. If these arguments are not passed, fall back to +# assuming that we're running a just-built version of julia and generating docs +# in tree. let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) - global const buildrootdoc = i === nothing ? (@__DIR__) : joinpath(first(match(r, ARGS[i]).captures), "doc") + if i === nothing + global const buildrootdoc = @__DIR__ + global const buildroot = abspath(joinpath(buildrootdoc, "..")) + else + global const buildroot = first(match(r, ARGS[i]).captures) + global const buildrootdoc = joinpath(buildroot, "doc") + end +end + +let r = r"stdlibdir=(.+)", i = findfirst(x -> occursin(r, x), ARGS) + if i === nothing + global const STDLIB_DIR = Sys.STDLIB + else + global const STDLIB_DIR = first(match(r, ARGS[i]).captures) + end end # Install dependencies needed to build the documentation. @@ -28,7 +47,6 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest) # make links for stdlib package docs, this is needed until #552 in Documenter.jl is finished const STDLIB_DOCS = [] -const STDLIB_DIR = Sys.STDLIB const EXT_STDLIB_DOCS = ["Pkg"] cd(joinpath(buildrootdoc, "src")) do Base.rm("stdlib"; recursive=true, force=true) From f8ece052b0ad98ed27316b63218b32c0877e69da Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 27 May 2025 09:00:44 -0400 Subject: [PATCH 316/662] Don't filter `Core` methods from newly-inferred list (#58510) This allows constructors like `Tuple{Type{Vector{Foo}}, UndefInitializer, Tuple{Int}}` to precompile as usual Appears to have a minimal effect on the stdlib pkgimages: ```julia --- before.txt 2025-05-23 08:36:20.171870043 -0400 +++ after.txt 2025-05-22 14:48:49.003869097 -0400 @@ -47,7 +47,7 @@ 20K ../julia/usr/share/julia/compiled/v1.13/Logging/pkgimage.so 20K ../julia/usr/share/julia/compiled/v1.13/Logging/pkgimage.so 3.5M ../julia/usr/share/julia/compiled/v1.13/Markdown/pkgimage.so -3.5M ../julia/usr/share/julia/compiled/v1.13/Markdown/pkgimage.so +3.6M ../julia/usr/share/julia/compiled/v1.13/Markdown/pkgimage.so 184K ../julia/usr/share/julia/compiled/v1.13/Mmap/pkgimage.so 184K ../julia/usr/share/julia/compiled/v1.13/Mmap/pkgimage.so 28K ../julia/usr/share/julia/compiled/v1.13/MozillaCACerts_jll/pkgimage.so ``` Resolves #58497. --- Compiler/src/cicache.jl | 2 +- test/precompile.jl | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Compiler/src/cicache.jl b/Compiler/src/cicache.jl index 9c528bc0ae822..4d188dbbc3450 100644 --- a/Compiler/src/cicache.jl +++ b/Compiler/src/cicache.jl @@ -14,7 +14,7 @@ end function setindex!(cache::InternalCodeCache, ci::CodeInstance, mi::MethodInstance) @assert ci.owner === cache.owner m = mi.def - if isa(m, Method) && m.module != Core + if isa(m, Method) ccall(:jl_push_newly_inferred, Cvoid, (Any,), ci) end ccall(:jl_mi_cache_insert, Cvoid, (Any, Any), mi, ci) diff --git a/test/precompile.jl b/test/precompile.jl index f92a2cc62ef0d..2ee24a71dc62e 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2145,6 +2145,29 @@ precompile_test_harness("No backedge precompile") do load_path end end +precompile_test_harness("Pre-compile Core methods") do load_path + # Core methods should support pre-compilation as external CI's like anything else + # https://github.com/JuliaLang/julia/issues/58497 + write(joinpath(load_path, "CorePrecompilation.jl"), + """ + module CorePrecompilation + struct Foo end + precompile(Tuple{Type{Vector{Foo}}, UndefInitializer, Tuple{Int}}) + end + """) + ji, ofile = Base.compilecache(Base.PkgId("CorePrecompilation")) + @eval using CorePrecompilation + invokelatest() do + let tt = Tuple{Type{Vector{CorePrecompilation.Foo}}, UndefInitializer, Tuple{Int}}, + match = first(Base._methods_by_ftype(tt, -1, Base.get_world_counter())), + mi = Base.specialize_method(match) + @test isdefined(mi, :cache) + @test mi.cache.max_world === typemax(UInt) + @test mi.cache.invoke != C_NULL + end + end +end + # Test precompilation of generated functions that return opaque closures # (with constprop marker set to false). precompile_test_harness("Generated Opaque") do load_path From d5442db00dbdc36d8bffbb528207b95fcceb48ef Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 27 May 2025 10:39:35 -0400 Subject: [PATCH 317/662] avoid error just computing coverage of generated functions (#58488) We only put debuginfo here if we ran the optimizer, so if there isn't debuginfo here, then we either didn't run the optimizer or were the result of const-prop. In the former case, we don't need to invalidate the code for instrumentation (it cannot have code from it). In the later case, we should already have an edge from the non-const-prop result. This only matters for generated functions, since otherwise we have Method's source's accurate debuginfo already and this is just a duplicate reference to it. Fix #58227 --- base/staticdata.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index 928372154653f..04fb6f0cfa263 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -82,9 +82,9 @@ end function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def::Method, validation_world::UInt) if JLOptions().code_coverage != 0 || JLOptions().malloc_log != 0 # test if the code needs to run with instrumentation, in which case we cannot use existing generated code - if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower (and less accurate?) + if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower and less reliable Compiler.should_instrument(def.module, def.debuginfo) : - Compiler.should_instrument(def.module, codeinst.debuginfo) + isdefined(codeinst, :debuginfo) && Compiler.should_instrument(def.module, codeinst.debuginfo) return true end gensig = gen_staged_sig(def, mi) From 989973adfc7a1d4eac26e4e6e3dbcd84b2c5f9bc Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 27 May 2025 12:31:32 -0400 Subject: [PATCH 318/662] doc: Fix version reading in Makefile on Windows (#58536) I copied this expression from the toplevel Makefile, but here we need to read the version eargerly, because otherwise cygpath_w does not work. This had broken the windows build, but I did not notice due to the ongoing CI difficulties. --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index ccb6af38c9d80..2b95cc9f96e9b 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -21,7 +21,7 @@ help: @echo "To fix outdated doctests, use 'make doctest=fix'" @echo "To run doctests using Revise (to test changes without rebuilding the sysimage), use 'make doctest=true revise=true'" -VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` +VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ texplatform=$(texplatform) revise=$(revise) stdlibdir=$(call cygpath_w,$(build_datarootdir)/julia/stdlib/$(VERSDIR)/) From 953903b05d16bba5ebab10eec85e83b791eea5c3 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 28 May 2025 00:46:03 -0400 Subject: [PATCH 319/662] docs: Add missing compat annotation for `isdefinedglobal` (#58542) Add missing compatibility annotation for `isdefinedglobal`. Fixes #58528. --- base/docs/basedocs.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 1445eee7e28b4..a6abf3e384cfb 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -2855,6 +2855,9 @@ a value set. If `allow_import` is `false`, the global variable must be defined inside `m` and may not be imported from another module. +!!! compat "Julia 1.12" + This function requires Julia 1.12 or later. + See also [`@isdefined`](@ref). # Examples From 8ac92910eb4d92f7b31f024ee6eff6f7dd5de748 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Wed, 28 May 2025 08:39:06 -0500 Subject: [PATCH 320/662] CI: Automatically assign a committer to PRs not opened by committers (#58303) This is the first step in implementing @StefanKarpinski's [state machine](https://discourse.julialang.org/t/suggestion-to-slightly-improve-julia-development/50916/82), as prototyped here: https://github.com/LilithHafner/AutomationTesting/. The goal is to make sure all PRs have a committer tracking them, but without adding much cognitive load to committers. This alone will be IMO slightly helpful but not super impactful. Some next steps that will increase impact are - tracking when responsibility for continued progress lies with the PR author vs with the committer - gentle reminders to the person responsible, with an option to de-assign self and/or close the PR - automatically close PR after a long period of inactivity while the PR author is responsible for progress - automatically re-assign a new committer after a long period of inactivity while the committer is responsible We chose to use random assignment to an op-in pool of reviewers at https://github.com/JuliaLang/pr-assignment/blob/main/users.txt because - The github suggested reviewer option is not available via API - Using git blame (or "suggested reviewer") will have a tendency to assign the most busy people to the most PRs which is not good - Not all committers will be willing to triage pull requests --------- Co-authored-by: Dilum Aluthge Co-authored-by: Andy Dienes <51664769+adienes@users.noreply.github.com> --- .github/CODEOWNERS | 1 + .github/workflows/PrAssignee.yml | 206 +++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 .github/workflows/PrAssignee.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a152517b78c1..3df89f0f4d096 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,7 @@ CODEOWNERS @JuliaLang/github-actions /.github/workflows/rerun_failed.yml @DilumAluthge /.github/workflows/statuses.yml @DilumAluthge +/.github/workflows/PrAssignee.yml @LilithHafner @DilumAluthge /base/special/ @oscardssmith /base/sort.jl @LilithHafner /test/sorting.jl @LilithHafner diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml new file mode 100644 index 0000000000000..0202ae20d8da7 --- /dev/null +++ b/.github/workflows/PrAssignee.yml @@ -0,0 +1,206 @@ +name: PR Assignee +on: + # Important security note: Do NOT use `actions/checkout` + # or any other method for checking out the pull request's source code. + # This is because the pull request's source code is untrusted, but the + # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). + # + # Quoting from the GitHub Docs: + # > For workflows that are triggered by the pull_request_target event, the GITHUB_TOKEN is granted + # > read/write repository permission unless the permissions key is specified and the workflow can access secrets, + # > even when it is triggered from a fork. + # > + # > Although the workflow runs in the context of the base of the pull request, + # > you should make sure that you do not check out, build, or run untrusted code from the pull request with this event. + # + # Source: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target + # + # See also: https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ + pull_request_target: + types: [opened, reopened, ready_for_review] + +# Permissions for the `GITHUB_TOKEN`: +permissions: + pull-requests: write # Needed in order to assign a user as the PR assignee + +jobs: + pr-assignee: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft != true }} + steps: + # Important security note: As discussed above, do NOT use `actions/checkout` + # or any other method for checking out the pull request's source code. + # This is because the pull request's source code is untrusted, but the + # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). + - name: Add Assignee + # We pin all third-party actions to a full length commit SHA + # https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-third-party-actions + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + retries: 5 # retry GitHub API requests up to 5 times, with exponential backoff + retry-exempt-status-codes: 403 + script: | + const oldPrAssignees = context.payload.pull_request.assignees + .map(obj => obj.login) + console.log('oldPrAssignees: ', oldPrAssignees); + const prAuthor = context.payload.pull_request.user.login; + + // Check if the PR is opened by a collaborator on the repo (aka a committer). + // We use the /repos/{owner}/{repo}/collaborators/{username} endpoint to avoid + // neeing org scope permissions. + const isCollaboratorResponseObj = await github.request('GET /repos/{owner}/{repo}/collaborators/{username}', { + owner: context.repo.owner, + repo: context.repo.repo, + username: prAuthor, + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + if (isCollaboratorResponseObj.status == 204) { + var isCollaborator = true; + } else if (isCollaboratorResponseObj.status == 404) { + var isCollaborator = false; + } else { + console.error('Unable to process the response from checkCollaborator'); + console.error('isCollaboratorResponseObj: ', isCollaboratorResponseObj); + var isCollaborator = false; + } + + console.log('prAuthor: ', prAuthor); + console.log('isCollaborator: ', isCollaborator); + + // Load the list of assignable reviewers from the JuliaLang/pr-assignment repo at: + // https://github.com/JuliaLang/pr-assignment/blob/main/users.txt + // + // NOTE to JuliaLang committers: If you want to be assigned to new PRs, please add your + // GitHub username to that file. + + // Load file contents + const { data: fileContentsObj } = await github.rest.repos.getContent({ + owner: 'JuliaLang', + repo: 'pr-assignment', + path: 'users.txt', + ref: 'main', + }); + + const fileContentsBufferObj = Buffer.from(fileContentsObj.content, "base64"); + const fileContentsText = fileContentsBufferObj.toString("utf8"); + + // Find lines that match the following regex, and extract the usernames: + const regex = /^@([a-zA-Z0-9\-]+)(\s*?)?(#[\S]*?)?$/; + const assigneeCandidates = fileContentsText + .split('\n') + .map(line => line.trim()) + .map(line => line.match(regex)) + .filter(match => match !== null) + .map(match => match[1]); + + console.log('assigneeCandidates: ', assigneeCandidates); + if (assigneeCandidates.length < 1) { + const msg = 'ERROR: Could not find any assigneeCandidates'; + console.error(msg); + throw new Error(msg); + } + + if (oldPrAssignees.length >= 1) { + console.log('Skipping this PR, because it already has at least one assignee'); + return; + } + + + const RUNNER_DEBUG_original = process.env.RUNNER_DEBUG; + console.log('RUNNER_DEBUG_original: ', RUNNER_DEBUG_original); + if (RUNNER_DEBUG_original === undefined) { + var thisIsActionsRunnerDebugMode = false; + } else { + const RUNNER_DEBUG_trimmed = RUNNER_DEBUG_original.trim().toLowerCase() + if (RUNNER_DEBUG_trimmed.length < 1) { + var thisIsActionsRunnerDebugMode = false; + } else { + var thisIsActionsRunnerDebugMode = (RUNNER_DEBUG_trimmed == 'true') || (RUNNER_DEBUG_trimmed == '1'); + } + } + console.log('thisIsActionsRunnerDebugMode: ', thisIsActionsRunnerDebugMode); + + if (isCollaborator == true) { + + if (thisIsActionsRunnerDebugMode) { + // The PR author is a committer + // But thisIsActionsRunnerDebugMode is true, so we proceed to still run the rest of the script + console.log('PR is authored by JuliaLang committer, but thisIsActionsRunnerDebugMode is true, so we will still run the rest of the script: ', prAuthor); + } else { + // The PR author is a committer, so we skip assigning them + console.log('Skipping PR authored by JuliaLang committer: ', prAuthor); + console.log('Note: If you want to run the full script (even though the PR author is a committer), simply re-run this job with Actions debug logging enabled'); + return; + } + } + + var weDidEncounterError = false; + + // Assign random committer + const selectedAssignee = assigneeCandidates[Math.floor(Math.random()*assigneeCandidates.length)] + console.log('selectedAssignee: ', selectedAssignee); + console.log(`Attempting to assign @${selectedAssignee} to this PR...`); + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + assignees: selectedAssignee, + }); + + // Add the "pr review" label + const prReviewLabel = 'status: waiting for PR reviewer'; + console.log('Attempting to add prReviewLabel to this PR...'); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + labels: [prReviewLabel], + }); + + // Now get the updated PR info, and see if we were successful: + const updatedPrData = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + const newPrAssignees = updatedPrData + .data + .assignees + .map(element => element.login) + console.log('newPrAssignees: ', newPrAssignees); + if (newPrAssignees.includes(selectedAssignee)) { + console.log(`Successfully assigned @${selectedAssignee}`); + } else { + weDidEncounterError = true; + console.log(`ERROR: Failed to assign @${selectedAssignee}`); + } + const newPrLabels = updatedPrData + .data + .labels + .map(element => element.name) + console.log('newPrLabels: ', newPrLabels); + if (newPrLabels.includes(prReviewLabel)) { + console.log('Successfully added prReviewLabel'); + } else { + weDidEncounterError = true; + console.log('ERROR: Failed to add add prReviewLabel'); + } + + // Post a comment + const commentBody = `Hello! I am a bot.\n\nThank you for your pull request!\n\nI have assigned \`@${selectedAssignee}\` to this pull request.\n\n\`@${selectedAssignee}\` can either choose to review this pull request themselves, or they can choose to find someone else to review this pull request.` + console.log('Attempting to post bot comment on the PR...'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: commentBody, + }); + + // Exit with error if any problems were encountered earlier + if (weDidEncounterError) { + const msg = 'ERROR: Encountered at least one problem while running the script'; + console.error(msg); + throw new Error(msg); + } From 1735d8f03c9e4fedba652e4e562c049ac48c93fd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 28 May 2025 11:24:04 -0400 Subject: [PATCH 321/662] make just one MethodTable (#58131) Instead of hiding the fragments of the method table in each TypeName, make one variable `Core.GlobalMethods` with access to all methods. The need to split them early for performance apparently had been long past since kwargs and constructors were already using a single table and cache anyways. Some new concepts introduced here: - A single Method can now be added to multiple functions. So instead of using eval in a for loop, we could define it just once (see example below). - Several fields (`max_args`, `name`, and `backedges`) were moved from MethodTable to their TypeName. - TypeName currently has a (user-modifiable) field called `singletonname`. If set to something other than `name`, it may be used for pretty printing of a singleton object using its "canonical" (unmangled) name, particularly for `function`. - `Core.Builtin` method table entries are even more normal now, with valid `sig` fields, and special logic to specifically prevent adding methods which would become ambiguous with them (as that would violate the tfuncs we have for them). - `Core.GlobalMethods` is a `Base.Experimental.@MethodTable GlobalMethods`. - Each `MethodTable` contains a separate `MethodCache` object for managing fast dispatch lookups. We may want to use this for the `Method` field containing the `invokes` list so that lookups there get more of the same optimizations as global calls. - Methods could be put into any number of different MethodTables (or none). The `Method.primary_world` field is intended to reflect whether it is currently put into the GlobalMethods table, and what world to use in the GlobalMethods table for running its generator, and otherwise is meaningless. - The lock for TypeName backedges is a single global lock now, in `Core.GlobalMethods.mc`. - The `backedges` in TypeName are stored on the "top-most" typename in the hierarchy, to enable efficient lookup (although we might want to consider replacing this entirely with a TypeMap). The "top-most" typename is the typename of the type closest to Any, after union-splitting, which doesn't have an intersection with Builtin (so Function and Any by implication continue to not require scanning for missing backedges since it is not permitted to add a Method applicable to all functions). - Support for having backedges from experimental method tables was removed since it was unsound and had been already replaced with staticdata.jl several months ago. - Documentation lookup for `IncludeInto` is fixed (previously attached only to `Main.include` instead of all `include` functions). Example: given this existing code in base/operators: for op in (:+, :*, :&, :|, :xor, :min, :max, :kron) @eval begin ($op)(a, b, c, xs...) = (@inline; afoldl($op, ($op)(($op)(a,b),c), xs...)) end end It could now instead be equivalently written as: let ops = Union{typeof(+), typeof(*), typeof(&), typeof(|), typeof(xor), typeof(min), typeof(max), typeof(kron)} (op::ops)(a, b, c, xs...) = (@inline; afoldl(op, (op)((op)(a,b),c), xs...)) end Fixes #57560 --- Compiler/src/Compiler.jl | 2 +- Compiler/src/abstractinterpretation.jl | 10 +- Compiler/src/stmtinfo.jl | 5 +- Compiler/src/tfuncs.jl | 5 +- Compiler/src/typeinfer.jl | 2 +- Compiler/src/utilities.jl | 6 +- base/Base_compiler.jl | 6 +- base/deprecated.jl | 4 +- base/docs/bindings.jl | 2 +- base/errorshow.jl | 2 +- base/invalidation.jl | 30 - base/methodshow.jl | 29 +- base/operators.jl | 2 +- base/reflection.jl | 8 +- base/runtime_internals.jl | 20 +- base/show.jl | 44 +- base/strings/io.jl | 6 +- base/summarysize.jl | 2 +- base/sysimg.jl | 13 +- doc/src/devdocs/functions.md | 29 +- doc/src/devdocs/types.md | 1 - src/builtins.c | 29 +- src/clangsa/GCChecker.cpp | 1 + src/codegen.cpp | 17 +- src/datatype.c | 50 +- src/gc-stock.c | 2 + src/gf.c | 817 ++++++++++++---------- src/interpreter.c | 3 +- src/ircode.c | 2 - src/jl_exported_data.inc | 5 +- src/jltypes.c | 130 ++-- src/julia.h | 38 +- src/julia_internal.h | 22 +- src/method.c | 106 +-- src/module.c | 10 +- src/precompile_utils.c | 110 +-- src/rtutils.c | 16 +- src/staticdata.c | 148 ++-- src/staticdata_utils.c | 8 +- stdlib/Serialization/src/Serialization.jl | 87 ++- stdlib/Test/src/Test.jl | 50 +- test/clangsa/MissingRoots.c | 14 - test/core.jl | 25 +- test/misc.jl | 4 +- test/precompile.jl | 8 +- test/reflection.jl | 4 +- test/show.jl | 8 +- test/stacktraces.jl | 6 +- test/syntax.jl | 5 +- test/worlds.jl | 39 +- 50 files changed, 982 insertions(+), 1010 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 68eab073b7e2d..e1c167e57ed08 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -38,7 +38,7 @@ else using Core.Intrinsics, Core.IR using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, - MethodTable, PartialOpaque, SimpleVector, TypeofVararg, + MethodTable, MethodCache, PartialOpaque, SimpleVector, TypeofVararg, _apply_iterate, apply_type, compilerbarrier, donotdelete, memoryref_isassigned, memoryrefget, memoryrefnew, memoryrefoffset, memoryrefset!, print, println, show, svec, typename, unsafe_write, write diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index a16bc06dca103..8d64575331e4a 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -363,15 +363,13 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: arg_n = split_argtypes[i]::Vector{Any} sig_n = argtypes_to_type(arg_n) sig_n === Bottom && continue - mt = ccall(:jl_method_table_for, Any, (Any,), sig_n) - mt === nothing && return FailedMethodMatch("Could not identify method table for call") - mt = mt::MethodTable thismatches = findall(sig_n, method_table(interp); limit = max_methods) if thismatches === nothing return FailedMethodMatch("For one of the union split cases, too many methods matched") end valid_worlds = intersect(valid_worlds, thismatches.valid_worlds) thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches) + mt = Core.GlobalMethods thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) for idx = 1:length(thismatches) @@ -385,11 +383,6 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: end function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(atype), max_methods::Int) - mt = ccall(:jl_method_table_for, Any, (Any,), atype) - if mt === nothing - return FailedMethodMatch("Could not identify method table for call") - end - mt = mt::MethodTable matches = findall(atype, method_table(interp); limit = max_methods) if matches === nothing # this means too many methods matched @@ -397,6 +390,7 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a return FailedMethodMatch("Too many methods matched") end fullmatch = any(match::MethodMatch->match.fully_covers, matches) + mt = Core.GlobalMethods info = MethodMatchInfo(matches, mt, atype, fullmatch) applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] return MethodMatches(applicable, info, matches.valid_worlds) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index d108c671301b9..8f08748e1bc57 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -47,17 +47,16 @@ end add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) = _add_edges_impl(edges, info) function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Bool=false) if !fully_covering(info) - # add legacy-style missing backedge info also exists = false for i in 2:length(edges) - if edges[i] === info.mt && edges[i-1] == info.atype + if edges[i] === Core.GlobalMethods && edges[i-1] == info.atype exists = true break end end if !exists push!(edges, info.atype) - push!(edges, info.mt) + push!(edges, Core.GlobalMethods) end end nmatches = length(info.results) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index f3ce3b010a345..9e07567a39adc 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -3199,15 +3199,12 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv isdispatchelem(ft) || return CallMeta(Bool, Any, Effects(), NoCallInfo()) # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below types = rewrap_unionall(Tuple{ft, unwrapped.parameters...}, types)::Type end - mt = ccall(:jl_method_table_for, Any, (Any,), types) - if !isa(mt, MethodTable) - return CallMeta(Bool, Any, EFFECTS_THROWS, NoCallInfo()) - end match, valid_worlds = findsup(types, method_table(interp)) update_valid_age!(sv, valid_worlds) if match === nothing rt = Const(false) vresults = MethodLookupResult(Any[], valid_worlds, true) + mt = Core.GlobalMethods vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index f8ea506dd3c99..dacde593fe1f2 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -767,7 +767,7 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) if item isa Core.Binding maybe_add_binding_backedge!(item, caller) elseif item isa MethodTable - ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) + ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any), invokesig, caller) else item::MethodInstance ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index a5f271b0c3ef9..fe8966c32fc17 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -158,10 +158,8 @@ end function get_compileable_sig(method::Method, @nospecialize(atype), sparams::SimpleVector) isa(atype, DataType) || return nothing - mt = ccall(:jl_method_get_table, Any, (Any,), method) - mt === nothing && return nothing - return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Any, Cint), - mt, atype, sparams, method, #=int return_if_compileable=#1) + return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Cint), + atype, sparams, method, #=int return_if_compileable=#1) end diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index c85ad4547379c..91e765bac8a13 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -218,7 +218,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke), f, T, args...) return invoke(Core.kwcall, T, kwargs, f, args...) end # invoke does not have its own call cache, but kwcall for invoke does -setfield!(typeof(invoke).name.mt, :max_args, 3, :monotonic) # invoke, f, T, args... +setfield!(typeof(invoke).name, :max_args, Int32(3), :monotonic) # invoke, f, T, args... # define applicable(f, T, args...; kwargs...), without kwargs wrapping # to forward to applicable @@ -252,7 +252,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invokelatest), f, args...) @inline return Core.invokelatest(Core.kwcall, kwargs, f, args...) end -setfield!(typeof(invokelatest).name.mt, :max_args, 2, :monotonic) # invokelatest, f, args... +setfield!(typeof(invokelatest).name, :max_args, Int32(2), :monotonic) # invokelatest, f, args... """ invoke_in_world(world, f, args...; kwargs...) @@ -286,7 +286,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke_in_world), world::UInt, @inline return Core.invoke_in_world(world, Core.kwcall, kwargs, f, args...) end -setfield!(typeof(invoke_in_world).name.mt, :max_args, 3, :monotonic) # invoke_in_world, world, f, args... +setfield!(typeof(invoke_in_world).name, :max_args, Int32(3), :monotonic) # invoke_in_world, world, f, args... # core operations & types include("promotion.jl") diff --git a/base/deprecated.jl b/base/deprecated.jl index c5701adf1a420..e7ea1e15e7b50 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -211,7 +211,7 @@ macro deprecate(old, new, export_old=true) maybe_export, :($(esc(old)) = begin $meta - depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.mt.name) + depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.singletonname) $(esc(new)) end)) else @@ -222,7 +222,7 @@ macro deprecate(old, new, export_old=true) export_old ? Expr(:export, esc(old)) : nothing, :(function $(esc(old))(args...; kwargs...) $meta - depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.mt.name) + depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.singletonname) $(esc(new))(args...; kwargs...) end)) end diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index 5c65a35659f81..5a0e8e01762e2 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -42,6 +42,6 @@ end aliasof(b::Binding) = defined(b) ? (a = aliasof(resolve(b), b); defined(a) ? a : b) : b aliasof(d::DataType, b) = Binding(d.name.module, d.name.name) -aliasof(λ::Function, b) = (m = typeof(λ).name.mt; Binding(m.module, m.name)) +aliasof(λ::Function, b) = (m = typeof(λ).name; Binding(m.module, m.singletonname)) aliasof(m::Module, b) = Binding(m, nameof(m)) aliasof(other, b) = b diff --git a/base/errorshow.jl b/base/errorshow.jl index 4a8003ad82b6f..8d90aae79be82 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -328,7 +328,7 @@ function showerror(io::IO, ex::MethodError) print(io, "\nIn case you're trying to index into the array, use square brackets [] instead of parentheses ().") end # Check for local functions that shadow methods in Base - let name = ft.name.mt.name + let name = ft.name.singletonname if f_is_function && isdefined(Base, name) basef = getfield(Base, name) if basef !== f && hasmethod(basef, arg_types) diff --git a/base/invalidation.jl b/base/invalidation.jl index f26e7968d8a2a..34f260f7379fd 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -15,36 +15,6 @@ function iterate(gri::GlobalRefIterator, i = 1) return ((b::Core.Binding).globalref, i+1) end -const TYPE_TYPE_MT = Type.body.name.mt -const NONFUNCTION_MT = Core.MethodTable.name.mt -function foreach_module_mtable(visit, m::Module, world::UInt) - for gb in globalrefs(m) - binding = gb.binding - bpart = lookup_binding_partition(world, binding) - if is_defined_const_binding(binding_kind(bpart)) - v = partition_restriction(bpart) - uw = unwrap_unionall(v) - name = gb.name - if isa(uw, DataType) - tn = uw.name - if tn.module === m && tn.name === name && tn.wrapper === v && isdefined(tn, :mt) - # this is the original/primary binding for the type (name/wrapper) - mt = tn.mt - if mt !== nothing && mt !== TYPE_TYPE_MT && mt !== NONFUNCTION_MT - @assert mt.module === m - visit(mt) || return false - end - end - elseif isa(v, Core.MethodTable) && v.module === m && v.name === name - # this is probably an external method table here, so let's - # assume so as there is no way to precisely distinguish them - visit(v) || return false - end - end - end - return true -end - function foreachgr(visit, src::CodeInfo) stmts = src.code for i = 1:length(stmts) diff --git a/base/methodshow.jl b/base/methodshow.jl index 7fdefc9b7311f..dc3f564d70db7 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -81,7 +81,7 @@ function kwarg_decl(m::Method, kwtype = nothing) if m.sig !== Tuple # OpaqueClosure or Builtin kwtype = typeof(Core.kwcall) sig = rewrap_unionall(Tuple{kwtype, NamedTuple, (unwrap_unionall(m.sig)::DataType).parameters...}, m.sig) - kwli = ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), kwtype.name.mt, sig, get_world_counter()) + kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, get_world_counter()) if kwli !== nothing kwli = kwli::Method slotnames = ccall(:jl_uncompress_argnames, Vector{Symbol}, (Any,), kwli.slot_syms) @@ -259,10 +259,10 @@ function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_ end function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) - mt = ms.mt - name = mt.name - hasname = isdefined(mt.module, name) && - typeof(getfield(mt.module, name)) <: Function + tn = ms.tn + name = tn.singletonname + hasname = isdefined(tn.module, name) && + typeof(getfield(tn.module, name)) <: Function n = length(ms) m = n==1 ? "method" : "methods" print(io, "# $n $m") @@ -271,18 +271,18 @@ function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) if hasname what = (startswith(sname, '@') ? "macro" - : mt.module === Core && mt.defs isa Core.TypeMapEntry && (mt.defs.func::Method).sig === Tuple ? + : tn.module === Core && tn.wrapper <: Core.Builtin ? "builtin function" : # else "generic function") print(io, " for ", what, " ", namedisplay, " from ") - col = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, parentmodule_before_main(ms.mt.module)) + col = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, parentmodule_before_main(tn.module)) - printstyled(io, ms.mt.module, color=col) + printstyled(io, tn.module, color=col) elseif '#' in sname print(io, " for anonymous function ", namedisplay) - elseif mt === _TYPE_NAME.mt + elseif tn === _TYPE_NAME || iskindtype(tn.wrapper) print(io, " for type constructor") else print(io, " for callable object") @@ -293,6 +293,8 @@ end # Determine the `modulecolor` value to pass to `show_method` function _modulecolor(method::Method) mmt = get_methodtable(method) + # TODO: this looks like a buggy bit of internal hacking, so disable for now + return nothing if mmt === nothing || mmt.module === parentmodule(method) return nothing end @@ -314,10 +316,10 @@ function _modulecolor(method::Method) end function show_method_table(io::IO, ms::MethodList, max::Int=-1, header::Bool=true) - mt = ms.mt - name = mt.name - hasname = isdefined(mt.module, name) && - typeof(getfield(mt.module, name)) <: Function + tn = ms.tn + name = tn.singletonname + hasname = isdefined(tn.module, name) && + typeof(getfield(tn.module, name)) <: Function if header show_method_list_header(io, ms, str -> "\""*str*"\"") end @@ -458,7 +460,6 @@ function show(io::IO, ::MIME"text/html", m::Method) end function show(io::IO, mime::MIME"text/html", ms::MethodList) - mt = ms.mt show_method_list_header(io, ms, str -> ""*str*"") print(io, "
    ") for meth in ms diff --git a/base/operators.jl b/base/operators.jl index d87e55498bd75..51729b852070d 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -633,7 +633,7 @@ function afoldl(op, a, bs...) end return y end -setfield!(typeof(afoldl).name.mt, :max_args, 34, :monotonic) +setfield!(typeof(afoldl).name, :max_args, Int32(34), :monotonic) for op in (:+, :*, :&, :|, :xor, :min, :max, :kron) @eval begin diff --git a/base/reflection.jl b/base/reflection.jl index 2e976add50190..304683639b6d3 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -933,13 +933,7 @@ this is a compiler-generated name. For explicitly-declared subtypes of `Function`, it is the name of the function's type. """ function nameof(f::Function) - t = typeof(f) - mt = t.name.mt - if mt === Symbol.name.mt - # uses shared method table, so name is not unique to this function type - return nameof(t) - end - return mt.name + return typeof(f).name.singletonname end function nameof(f::Core.IntrinsicFunction) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index dc78d35a0bc0d..b50d0c7b23881 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1356,14 +1356,14 @@ hasproperty(x, s::Symbol) = s in propertynames(x) Make method `m` uncallable and force recompilation of any methods that use(d) it. """ function delete_method(m::Method) - ccall(:jl_method_table_disable, Cvoid, (Any, Any), get_methodtable(m), m) + ccall(:jl_method_table_disable, Cvoid, (Any,), m) end # type for reflecting and pretty-printing a subset of methods mutable struct MethodList <: AbstractArray{Method,1} ms::Array{Method,1} - mt::Core.MethodTable + tn::Core.TypeName # contains module.singletonname globalref for altering some aspects of printing end size(m::MethodList) = size(m.ms) @@ -1374,10 +1374,10 @@ function MethodList(mt::Core.MethodTable) visit(mt) do m push!(ms, m) end - return MethodList(ms, mt) + return MethodList(ms, Any.name) end -function matches_to_methods(ms::Array{Any,1}, mt::Core.MethodTable, mod) +function matches_to_methods(ms::Array{Any,1}, tn::Core.TypeName, mod) # Lack of specialization => a comprehension triggers too many invalidations via _collect, so collect the methods manually ms = Method[(ms[i]::Core.MethodMatch).method for i in 1:length(ms)] # Remove shadowed methods with identical type signatures @@ -1392,7 +1392,7 @@ function matches_to_methods(ms::Array{Any,1}, mt::Core.MethodTable, mod) mod === nothing || filter!(ms) do m return parentmodule(m) ∈ mod end - return MethodList(ms, mt) + return MethodList(ms, tn) end """ @@ -1414,7 +1414,7 @@ function methods(@nospecialize(f), @nospecialize(t), world = get_world_counter() world == typemax(UInt) && error("code reflection cannot be used from generated functions") ms = _methods(f, t, -1, world)::Vector{Any} - return matches_to_methods(ms, typeof(f).name.mt, mod) + return matches_to_methods(ms, typeof(f).name, mod) end methods(@nospecialize(f), @nospecialize(t), mod::Module) = methods(f, t, (mod,)) @@ -1425,7 +1425,7 @@ function methods_including_ambiguous(@nospecialize(f), @nospecialize(t)) min = RefValue{UInt}(typemin(UInt)) max = RefValue{UInt}(typemax(UInt)) ms = _methods_by_ftype(tt, nothing, -1, world, true, min, max, Ptr{Int32}(C_NULL))::Vector{Any} - return matches_to_methods(ms, typeof(f).name.mt, nothing) + return matches_to_methods(ms, typeof(f).name, nothing) end function methods(@nospecialize(f), @@ -1623,10 +1623,8 @@ end function get_nospecializeinfer_sig(method::Method, @nospecialize(atype), sparams::SimpleVector) isa(atype, DataType) || return method.sig - mt = ccall(:jl_method_get_table, Any, (Any,), method) - mt === nothing && return method.sig - return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Any, Cint), - mt, atype, sparams, method, #=int return_if_compileable=#0) + return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Cint), + atype, sparams, method, #=int return_if_compileable=#0) end is_nospecialized(method::Method) = method.nospecialize ≠ 0 diff --git a/base/show.jl b/base/show.jl index 9864fc7e5dec8..14518be0f153e 100644 --- a/base/show.jl +++ b/base/show.jl @@ -39,16 +39,16 @@ end function _isself(ft::DataType) ftname = ft.name - isdefined(ftname, :mt) || return false - name = ftname.mt.name - mod = parentmodule(ft) # NOTE: not necessarily the same as ft.name.mt.module - return invokelatest(isdefinedglobal, mod, name) && ft == typeof(invokelatest(getglobal, mod, name)) + name = ftname.singletonname + ftname.name === name && return false + mod = parentmodule(ft) + return invokelatest(isdefinedglobal, mod, name) && ft === typeof(invokelatest(getglobal, mod, name)) end function show(io::IO, ::MIME"text/plain", f::Function) get(io, :compact, false)::Bool && return show(io, f) ft = typeof(f) - name = ft.name.mt.name + name = ft.name.singletonname if isa(f, Core.IntrinsicFunction) print(io, f) id = Core.Intrinsics.bitcast(Int32, f) @@ -542,22 +542,20 @@ module UsesCoreAndBaseOnly end function show_function(io::IO, f::Function, compact::Bool, fallback::Function) - ft = typeof(f) - mt = ft.name.mt - if mt === Symbol.name.mt - # uses shared method table + fname = typeof(f).name + if fname.name === fname.singletonname fallback(io, f) elseif compact - print(io, mt.name) - elseif isdefined(mt, :module) && isdefinedglobal(mt.module, mt.name) && - getglobal(mt.module, mt.name) === f + print(io, fname.singletonname) + elseif isdefined(fname, :module) && isdefinedglobal(fname.module, fname.singletonname) && isconst(fname.module, fname.singletonname) && + getglobal(fname.module, fname.singletonname) === f # this used to call the removed internal function `is_exported_from_stdlib`, which effectively # just checked for exports from Core and Base. mod = get(io, :module, UsesCoreAndBaseOnly) - if !(isvisible(mt.name, mt.module, mod) || mt.module === mod) - print(io, mt.module, ".") + if !(isvisible(fname.singletonname, fname.module, mod) || fname.module === mod) + print(io, fname.module, ".") end - show_sym(io, mt.name) + show_sym(io, fname.singletonname) else fallback(io, f) end @@ -965,7 +963,7 @@ function show(io::IO, ::MIME"text/plain", @nospecialize(x::Type)) # give a helpful hint for function types if x isa DataType && x !== UnionAll && !(get(io, :compact, false)::Bool) tn = x.name::Core.TypeName - globname = isdefined(tn, :mt) ? tn.mt.name : nothing + globname = tn.singletonname if is_global_function(tn, globname) print(io, " (singleton type of function ") show_sym(io, globname) @@ -1048,11 +1046,11 @@ function isvisible(sym::Symbol, parent::Module, from::Module) end function is_global_function(tn::Core.TypeName, globname::Union{Symbol,Nothing}) - if globname !== nothing + if globname !== nothing && isconcretetype(tn.wrapper) && tn !== DataType.name # ignore that typeof(DataType)===DataType, since it is valid but not useful globname_str = string(globname::Symbol) - if ('#' ∉ globname_str && '@' ∉ globname_str && isdefined(tn, :module) && - isdefinedglobal(tn.module, globname) && - isconcretetype(tn.wrapper) && isa(getglobal(tn.module, globname), tn.wrapper)) + if '#' ∉ globname_str && '@' ∉ globname_str && isdefined(tn, :module) && + isdefinedglobal(tn.module, globname) && isconst(tn.module, globname) && + isa(getglobal(tn.module, globname), tn.wrapper) return true end end @@ -1083,7 +1081,7 @@ function show_type_name(io::IO, tn::Core.TypeName) # intercept this case and print `UnionAll` instead. return print(io, "UnionAll") end - globname = isdefined(tn, :mt) ? tn.mt.name : nothing + globname = tn.singletonname globfunc = is_global_function(tn, globname) sym = (globfunc ? globname : tn.name)::Symbol globfunc && print(io, "typeof(") @@ -2567,10 +2565,10 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg uw = unwrap_unionall(ft) if ft <: Function && isa(uw, DataType) && isempty(uw.parameters) && _isself(uw) uwmod = parentmodule(uw) - if qualified && !isexported(uwmod, uw.name.mt.name) && uwmod !== Main + if qualified && !isexported(uwmod, uw.name.singletonname) && uwmod !== Main print_within_stacktrace(io, uwmod, '.', bold=true) end - s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io) + s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.singletonname), context=io) print_within_stacktrace(io, s, bold=true) elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar)) uwf = unwrap_unionall(f) diff --git a/base/strings/io.jl b/base/strings/io.jl index c9c5de1d791d5..f3a0783e98a9b 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -51,7 +51,7 @@ function print(io::IO, xs...) return nothing end -setfield!(typeof(print).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(print).name, :max_args, Int32(10), :monotonic) """ println([io::IO], xs...) @@ -76,7 +76,7 @@ julia> takestring!(io) """ println(io::IO, xs...) = print(io, xs..., "\n") -setfield!(typeof(println).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(println).name, :max_args, Int32(10), :monotonic) ## conversion of general objects to strings ## """ @@ -148,7 +148,7 @@ function print_to_string(xs...) end takestring!(s) end -setfield!(typeof(print_to_string).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(print_to_string).name, :max_args, Int32(10), :monotonic) function string_with_env(env, xs...) if isempty(xs) diff --git a/base/summarysize.jl b/base/summarysize.jl index 1e4d546e675aa..9dfd1431b84c7 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -139,7 +139,7 @@ end function (ss::SummarySize)(obj::Core.TypeName) key = pointer_from_objref(obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - return Core.sizeof(obj) + (isdefined(obj, :mt) ? ss(obj.mt) : 0) + return Core.sizeof(obj) end function (ss::SummarySize)(obj::GenericMemory) diff --git a/base/sysimg.jl b/base/sysimg.jl index 8adb05ece0b2c..f5354c6ebea1f 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -15,8 +15,8 @@ ccall(:jl_init_restored_module, Cvoid, (Any,), Base) include([mapexpr::Function,] path::AbstractString) Evaluate the contents of the input source file in the global scope of the containing module. -Every module (except those defined with `baremodule`) has its own -definition of `include`, which evaluates the file in that module. +Every `Module` (except those defined with `baremodule`) has a private 1-argument definition +of `include`, which evaluates the file in that module, for use inside that module. Returns the result of the last evaluated expression of the input file. During including, a task-local include path is set to the directory containing the file. Nested calls to `include` will search relative to that path. This function is typically used to load source @@ -40,15 +40,18 @@ Use [`Base.include`](@ref) to evaluate a file into another module. !!! compat "Julia 1.5" Julia 1.5 is required for passing the `mapexpr` argument. """ -const include = Base.IncludeInto(Main) +Base.IncludeInto """ eval(expr) Evaluate an expression in the global scope of the containing module. -Every `Module` (except those defined with `baremodule`) has its own 1-argument -definition of `eval`, which evaluates expressions in that module. +Every `Module` (except those defined with `baremodule`) has a private 1-argument definition +of `eval`, which evaluates expressions in that module, for use inside that module. """ +Core.EvalInto + +const include = Base.IncludeInto(Main) const eval = Core.EvalInto(Main) # Ensure this file is also tracked diff --git a/doc/src/devdocs/functions.md b/doc/src/devdocs/functions.md index fb67dfd17c3e2..51df95ba0bc42 100644 --- a/doc/src/devdocs/functions.md +++ b/doc/src/devdocs/functions.md @@ -1,5 +1,6 @@ # Julia Functions + This document will explain how functions, method definitions, and method tables work. ## Method Tables @@ -15,7 +16,7 @@ has a `TypeName`. ## [Function calls](@id Function-calls) -Given the call `f(x, y)`, the following steps are performed: first, the method table to use is +Given the call `f(x, y)`, the following steps are performed: first, the method cache to use is accessed as `typeof(f).name.mt`. Second, an argument tuple type is formed, `Tuple{typeof(f), typeof(x), typeof(y)}`. Note that the type of the function itself is the first element. This is because the type might have parameters, and so needs to take part in dispatch. This tuple type is looked up in the method @@ -187,7 +188,7 @@ is absent. Finally there is the kwsorter definition: ``` -function (::Core.kwftype(typeof(circle)))(kws, circle, center, radius) +function (::Core.kwcall)(kws, circle, center, radius) if haskey(kws, :color) color = kws.color else @@ -205,30 +206,6 @@ function (::Core.kwftype(typeof(circle)))(kws, circle, center, radius) end ``` -The function `Core.kwftype(t)` creates the field `t.name.mt.kwsorter` (if it hasn't been created -yet), and returns the type of that function. - -This design has the feature that call sites that don't use keyword arguments require no special -handling; everything works as if they were not part of the language at all. Call sites that do -use keyword arguments are dispatched directly to the called function's kwsorter. For example the -call: - -```julia -circle((0, 0), 1.0, color = red; other...) -``` - -is lowered to: - -```julia -kwcall(merge((color = red,), other), circle, (0, 0), 1.0) -``` - -`kwcall` (also in`Core`) denotes a kwcall signature and dispatch. -The keyword splatting operation (written as `other...`) calls the named tuple `merge` function. -This function further unpacks each *element* of `other`, expecting each one to contain two values -(a symbol and a value). -Naturally, a more efficient implementation is available if all splatted arguments are named tuples. -Notice that the original `circle` function is passed through, to handle closures. ## [Compiler efficiency issues](@id compiler-efficiency-issues) diff --git a/doc/src/devdocs/types.md b/doc/src/devdocs/types.md index a09df61e4881d..b63f1c315f457 100644 --- a/doc/src/devdocs/types.md +++ b/doc/src/devdocs/types.md @@ -199,7 +199,6 @@ TypeName name: Symbol Array defs: Nothing nothing cache: Nothing nothing - max_args: Int64 0 module: Module Core : Int64 0 : Int64 0 diff --git a/src/builtins.c b/src/builtins.c index 36b5d79ec0851..0e309ab912f9c 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2476,14 +2476,11 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1); jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0); jl_mk_builtin_func(jl_intrinsic_type, jl_symbol("IntrinsicFunction"), jl_f_intrinsic_call); - jl_mk_builtin_func( - (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type), - jl_symbol("OpaqueClosure"), jl_f_opaque_closure_call); + jl_datatype_t *oc = (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type); // Save a reference to the just created OpaqueClosure method, so we can provide special // codegen for it later. - jl_opaque_closure_method = (jl_method_t*)jl_methtable_lookup(jl_opaque_closure_typename->mt, - (jl_value_t*)jl_anytuple_type, 1); + jl_opaque_closure_method = jl_mk_builtin_func(oc, jl_symbol("OpaqueClosure"), jl_f_opaque_closure_call); // TODO: awkwardly not actually declared a Builtin, even though it relies on being handled by the special cases for Builtin everywhere else #define ADD_I(name, nargs) add_intrinsic(inm, #name, name); #define ADD_HIDDEN(name, nargs) @@ -2533,6 +2530,8 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("Module", (jl_value_t*)jl_module_type); add_builtin("MethodTable", (jl_value_t*)jl_methtable_type); + add_builtin("GlobalMethods", (jl_value_t*)jl_method_table); + add_builtin("MethodCache", (jl_value_t*)jl_methcache_type); add_builtin("Method", (jl_value_t*)jl_method_type); add_builtin("CodeInstance", (jl_value_t*)jl_code_instance_type); add_builtin("TypeMapEntry", (jl_value_t*)jl_typemap_entry_type); @@ -2597,6 +2596,26 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("AbstractString", (jl_value_t*)jl_abstractstring_type); add_builtin("String", (jl_value_t*)jl_string_type); + + // ensure that primitive types are fully allocated (since jl_init_types is incomplete) + assert(jl_atomic_load_relaxed(&jl_world_counter) == 1); + jl_module_t *core = jl_core_module; + jl_svec_t *bindings = jl_atomic_load_relaxed(&core->bindings); + jl_value_t **table = jl_svec_data(bindings); + for (size_t i = 0; i < jl_svec_len(bindings); i++) { + if (table[i] != jl_nothing) { + jl_binding_t *b = (jl_binding_t*)table[i]; + jl_value_t *v = jl_get_binding_value_in_world(b, 1); + if (v) { + if (jl_is_unionall(v)) + v = jl_unwrap_unionall(v); + if (jl_is_datatype(v)) { + jl_datatype_t *tt = (jl_datatype_t*)v; + tt->name->module = core; + } + } + } + } } #ifdef __cplusplus diff --git a/src/clangsa/GCChecker.cpp b/src/clangsa/GCChecker.cpp index af07ca2227839..09a034a9549d8 100644 --- a/src/clangsa/GCChecker.cpp +++ b/src/clangsa/GCChecker.cpp @@ -836,6 +836,7 @@ bool GCChecker::isGCTrackedType(QualType QT) { Name.ends_with_insensitive("jl_typemap_t") || Name.ends_with_insensitive("jl_unionall_t") || Name.ends_with_insensitive("jl_methtable_t") || + Name.ends_with_insensitive("jl_methcache_t") || Name.ends_with_insensitive("jl_cgval_t") || Name.ends_with_insensitive("jl_codectx_t") || Name.ends_with_insensitive("jl_ast_context_t") || diff --git a/src/codegen.cpp b/src/codegen.cpp index c0aeb5bac1915..5aa175df270f3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7561,16 +7561,11 @@ static const char *derive_sigt_name(jl_value_t *jargty) jl_datatype_t *dt = (jl_datatype_t*)jl_argument_datatype(jargty); if ((jl_value_t*)dt == jl_nothing) return NULL; - jl_sym_t *name = dt->name->name; - // if we have a kwcall, use that as the name anyways - jl_methtable_t *mt = dt->name->mt; - if (mt == jl_type_type_mt || mt == jl_nonfunction_mt || mt == NULL) { - // our value for `name` from MethodTable is not good, try to come up with something better - if (jl_is_type_type((jl_value_t*)dt)) { - dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); - if ((jl_value_t*)dt != jl_nothing) { - name = dt->name->name; - } + jl_sym_t *name = dt->name->singletonname; + if (jl_is_type_type((jl_value_t*)dt)) { + dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); + if ((jl_value_t*)dt != jl_nothing) { + name = dt->name->singletonname; } } return jl_symbol_name(name); @@ -7747,7 +7742,7 @@ const char *jl_generate_ccallable(Module *llvmmod, jl_value_t *nameval, jl_value assert(jl_is_datatype(ft)); jl_value_t *ff = ft->instance; assert(ff); - const char *name = !jl_is_string(nameval) ? jl_symbol_name(ft->name->mt->name) : jl_string_data(nameval); + const char *name = !jl_is_string(nameval) ? jl_symbol_name(ft->name->singletonname) : jl_string_data(nameval); jl_value_t *crt = declrt; if (jl_is_abstract_ref_type(declrt)) { declrt = jl_tparam0(declrt); diff --git a/src/datatype.c b/src/datatype.c index 0ccdd0b61d06f..eb25907647157 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -39,22 +39,30 @@ static jl_sym_t *jl_demangle_typename(jl_sym_t *s) JL_NOTSAFEPOINT return _jl_symbol(&n[1], len); } +JL_DLLEXPORT jl_methcache_t *jl_new_method_cache(void) +{ + jl_task_t *ct = jl_current_task; + jl_methcache_t *mc = + (jl_methcache_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methcache_t), + jl_methcache_type); + jl_atomic_store_relaxed(&mc->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&mc->cache, jl_nothing); + JL_MUTEX_INIT(&mc->writelock, "methodtable->writelock"); + return mc; +} + JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { + jl_methcache_t *mc = jl_new_method_cache(); + JL_GC_PUSH1(&mc); jl_task_t *ct = jl_current_task; jl_methtable_t *mt = - (jl_methtable_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methtable_t), - jl_methtable_type); - mt->name = jl_demangle_typename(name); - mt->module = module; + (jl_methtable_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methtable_t), jl_methtable_type); jl_atomic_store_relaxed(&mt->defs, jl_nothing); - jl_atomic_store_relaxed(&mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); - jl_atomic_store_relaxed(&mt->cache, jl_nothing); - jl_atomic_store_relaxed(&mt->max_args, 0); - mt->backedges = NULL; - JL_MUTEX_INIT(&mt->writelock, "methodtable->writelock"); - mt->offs = 0; - mt->frozen = 0; + mt->cache = mc; + mt->name = name; + mt->module = module; + JL_GC_POP(); return mt; } @@ -67,21 +75,23 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->name = name; tn->module = module; tn->wrapper = NULL; + tn->singletonname = jl_demangle_typename(name); jl_atomic_store_relaxed(&tn->Typeofwrapper, NULL); jl_atomic_store_relaxed(&tn->cache, jl_emptysvec); jl_atomic_store_relaxed(&tn->linearcache, jl_emptysvec); tn->names = NULL; tn->hash = bitmix(bitmix(module ? module->build_id.lo : 0, name->hash), 0xa1ada1da); - tn->_reserved = 0; + tn->_unused = 0; tn->abstract = abstract; tn->mutabl = mutabl; tn->mayinlinealloc = 0; - tn->mt = NULL; tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; - jl_atomic_store_relaxed(&tn->cache_entry_count, 0); + tn->backedges = NULL; tn->max_methods = 0; + jl_atomic_store_relaxed(&tn->max_args, 0); + jl_atomic_store_relaxed(&tn->cache_entry_count, 0); tn->constprop_heustic = 0; return tn; } @@ -861,18 +871,6 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( } else { tn = jl_new_typename_in((jl_sym_t*)name, module, abstract, mutabl); - if (super == jl_function_type || super == jl_builtin_type || is_anonfn_typename(jl_symbol_name(name))) { - // Callable objects (including compiler-generated closures) get independent method tables - // as an optimization - tn->mt = jl_new_method_table(name, module); - jl_gc_wb(tn, tn->mt); - if (jl_svec_len(parameters) == 0 && !abstract) - tn->mt->offs = 1; - } - else { - // Everything else, gets to use the unified table - tn->mt = jl_nonfunction_mt; - } } t->name = tn; jl_gc_wb(t, t->name); diff --git a/src/gc-stock.c b/src/gc-stock.c index 01453a30b2a4b..ce0502058baa7 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -2845,6 +2845,8 @@ static void gc_mark_roots(jl_gc_markqueue_t *mq) gc_try_claim_and_push(mq, jl_main_module, NULL); gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_main_module, "main_module"); // invisible builtin values + gc_try_claim_and_push(mq, jl_method_table, NULL); + gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_method_table, "global_method_table"); gc_try_claim_and_push(mq, jl_an_empty_vec_any, NULL); gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_an_empty_vec_any, "an_empty_vec_any"); gc_try_claim_and_push(mq, jl_module_init_order, NULL); diff --git a/src/gf.c b/src/gf.c index 5a960c33ef503..4bfe910ac6cae 100644 --- a/src/gf.c +++ b/src/gf.c @@ -27,6 +27,8 @@ extern "C" { _Atomic(int) allow_new_worlds = 1; JL_DLLEXPORT _Atomic(size_t) jl_world_counter = 1; // uses atomic acquire/release jl_mutex_t world_counter_lock; +jl_methtable_t *jl_method_table; + JL_DLLEXPORT size_t jl_get_world_counter(void) JL_NOTSAFEPOINT { jl_task_t *ct = jl_current_task; @@ -41,31 +43,41 @@ JL_DLLEXPORT size_t jl_get_tls_world_age(void) JL_NOTSAFEPOINT } // Compute the maximum number of times to unroll Varargs{T}, based on -// m->max_varargs (if specified) or a heuristic based on the maximum -// number of non-varargs arguments in the provided method table. +// m->max_varargs (if specified) or a heuristic based on the maximum number of +// non-varargs arguments for the function type of the method signature. // // If provided, `may_increase` is set to 1 if the returned value is // heuristic-based and has a chance of increasing in the future. static size_t get_max_varargs( jl_method_t *m, - jl_methtable_t *kwmt, - jl_methtable_t *mt, uint8_t *may_increase) JL_NOTSAFEPOINT { size_t max_varargs = 1; if (may_increase != NULL) *may_increase = 0; - if (m->max_varargs != UINT8_MAX) + if (m->max_varargs != UINT8_MAX) { max_varargs = m->max_varargs; - else if (kwmt != NULL && kwmt != jl_type_type_mt && kwmt != jl_nonfunction_mt && kwmt != jl_kwcall_mt) { - if (may_increase != NULL) - *may_increase = 1; // `max_args` can increase as new methods are inserted - - max_varargs = jl_atomic_load_relaxed(&kwmt->max_args) + 2; - if (mt == jl_kwcall_mt) - max_varargs += 2; - max_varargs -= m->nargs; + } + else { + jl_datatype_t *dt1 = jl_nth_argument_datatype(m->sig, 1); + jl_datatype_t *dt; + if (jl_kwcall_type && dt1 == jl_kwcall_type) + dt = jl_nth_argument_datatype(m->sig, 3); + else + dt = dt1; + if (dt != NULL && !jl_is_type_type((jl_value_t*)dt) && dt != jl_kwcall_type) { + if (may_increase != NULL) + *may_increase = 1; // `max_args` can increase as new methods are inserted + + max_varargs = jl_atomic_load_relaxed(&dt->name->max_args) + 2; + if (jl_kwcall_type && dt1 == jl_kwcall_type) + max_varargs += 2; + if (max_varargs > m->nargs) + max_varargs -= m->nargs; + else + max_varargs = 0; + } } return max_varargs; } @@ -104,9 +116,9 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee) /// ----- Definitions for various internal TypeMaps ----- /// -static int8_t jl_cachearg_offset(jl_methtable_t *mt) +static int8_t jl_cachearg_offset(void) { - return mt->offs; + return 0; } /// ----- Insertion logic for special entries ----- /// @@ -274,13 +286,13 @@ JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_value_t *t return mi; } -JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *type, size_t world) +JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_value_t *type, size_t world) { // TODO: this is sort of an odd lookup strategy (and the only user of // jl_typemap_assoc_by_type with subtype=0), while normally jl_gf_invoke_lookup would be // expected to be used instead struct jl_typemap_assoc search = {type, world, NULL}; - jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->defs), &search, jl_cachearg_offset(mt), /*subtype*/0); + jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&jl_method_table->defs), &search, 0, /*subtype*/0); if (!sf) return jl_nothing; return sf->func.value; @@ -288,7 +300,7 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ // ----- MethodInstance specialization instantiation ----- // -void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED +jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED { jl_method_t *m = jl_new_method_uninit(jl_core_module); m->name = sname; @@ -302,31 +314,36 @@ void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) m->nospecialize = 0; m->nospecialize = ~m->nospecialize; - jl_methtable_t *mt = dt->name->mt; jl_typemap_entry_t *newentry = NULL; - JL_GC_PUSH2(&m, &newentry); + jl_datatype_t *tuptyp = NULL; + JL_GC_PUSH3(&m, &newentry, &tuptyp); - newentry = jl_typemap_alloc(jl_anytuple_type, NULL, jl_emptysvec, - (jl_value_t*)m, 1, ~(size_t)0); - jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); + jl_value_t *params[2]; + params[0] = dt->name->wrapper; + params[1] = jl_tparam0(jl_anytuple_type); + tuptyp = (jl_datatype_t*)jl_apply_tuple_type_v(params, 2); - jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)jl_anytuple_type, jl_emptysvec); + jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)tuptyp, jl_emptysvec); jl_atomic_store_relaxed(&m->unspecialized, mi); jl_gc_wb(m, mi); jl_code_instance_t *codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, jl_nothing, 0, 1, ~(size_t)0, 0, jl_nothing, NULL, NULL); - jl_mi_cache_insert(mi, codeinst); jl_atomic_store_relaxed(&codeinst->specptr.fptr1, fptr); jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_args); + jl_mi_cache_insert(mi, codeinst); + + newentry = jl_typemap_alloc(tuptyp, NULL, jl_emptysvec, + (jl_value_t*)m, 1, ~(size_t)0); + jl_typemap_insert(&jl_method_table->defs, (jl_value_t*)jl_method_table, newentry, 0); - newentry = jl_typemap_alloc(jl_anytuple_type, NULL, jl_emptysvec, + newentry = jl_typemap_alloc(tuptyp, NULL, jl_emptysvec, (jl_value_t*)mi, 1, ~(size_t)0); - jl_typemap_insert(&mt->cache, (jl_value_t*)mt, newentry, 0); + jl_typemap_insert(&jl_method_table->cache->cache, (jl_value_t*)jl_method_table->cache, newentry, 0); - mt->frozen = 1; JL_GC_POP(); + return m; } // only relevant for bootstrapping. otherwise fairly broken. @@ -537,7 +554,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( { jl_value_t *owner = jl_nothing; // TODO: owner should be arg jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - while (codeinst) { + for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { if (jl_atomic_load_relaxed(&codeinst->min_world) == min_world && jl_atomic_load_relaxed(&codeinst->max_world) == max_world && jl_egal(codeinst->owner, owner) && @@ -555,7 +572,6 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( if (e && jl_egal((jl_value_t*)e, (jl_value_t*)edges)) return codeinst; } - codeinst = jl_atomic_load_relaxed(&codeinst->next); } codeinst = jl_new_codeinst( mi, owner, rettype, (jl_value_t*)jl_any_type, NULL, NULL, @@ -760,9 +776,9 @@ JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, return ret; } -int foreach_mtable_in_module( +static int foreach_typename_in_module( jl_module_t *m, - int (*visit)(jl_methtable_t *mt, void *env), + int (*visit)(jl_typename_t *tn, void *env), void *env) { jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); @@ -778,15 +794,50 @@ int foreach_mtable_in_module( jl_typename_t *tn = ((jl_datatype_t*)uw)->name; if (tn->module == m && tn->name == name && tn->wrapper == v) { // this is the original/primary binding for the type (name/wrapper) - jl_methtable_t *mt = tn->mt; - if (mt != NULL && (jl_value_t*)mt != jl_nothing && mt != jl_type_type_mt && mt != jl_nonfunction_mt) { - assert(mt->module == m); - if (!visit(mt, env)) - return 0; - } + if (!visit(((jl_datatype_t*)uw)->name, env)) + return 0; } } else if (jl_is_module(v)) { + jl_module_t *child = (jl_module_t*)v; + if (child != m && child->parent == m && child->name == name) { + // this is the original/primary binding for the submodule + if (!foreach_typename_in_module(child, visit, env)) + return 0; + } + } + } + table = jl_atomic_load_relaxed(&m->bindings); + } + return 1; +} + +static int jl_foreach_reachable_typename(int (*visit)(jl_typename_t *tn, void *env), jl_array_t *mod_array, void *env) +{ + for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { + jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); + assert(jl_is_module(m)); + if (m->parent == m) // some toplevel modules (really just Base) aren't actually + if (!foreach_typename_in_module(m, visit, env)) + return 0; + } + return 1; +} + +int foreach_mtable_in_module( + jl_module_t *m, + int (*visit)(jl_methtable_t *mt, void *env), + void *env) +{ + jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); + for (size_t i = 0; i < jl_svec_len(table); i++) { + jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); + if ((void*)b == jl_nothing) + break; + jl_sym_t *name = b->globalref->name; + jl_value_t *v = jl_get_latest_binding_value_if_const(b); + if (v) { + if (jl_is_module(v)) { jl_module_t *child = (jl_module_t*)v; if (child != m && child->parent == m && child->name == name) { // this is the original/primary binding for the submodule @@ -796,9 +847,7 @@ int foreach_mtable_in_module( } else if (jl_is_mtable(v)) { jl_methtable_t *mt = (jl_methtable_t*)v; - if (mt->module == m && mt->name == name) { - // this is probably an external method table here, so let's - // assume so as there is no way to precisely distinguish them + if (mt && mt != jl_method_table) { if (!visit(mt, env)) return 0; } @@ -809,37 +858,23 @@ int foreach_mtable_in_module( return 1; } -int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), void *env) + +int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_array_t *mod_array, void *env) { - if (!visit(jl_type_type_mt, env)) + if (!visit(jl_method_table, env)) return 0; - if (!visit(jl_nonfunction_mt, env)) - return 0; - jl_array_t *mod_array = jl_get_loaded_modules(); if (mod_array) { - JL_GC_PUSH1(&mod_array); - int i; - for (i = 0; i < jl_array_nrows(mod_array); i++) { + for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); assert(jl_is_module(m)); if (m->parent == m) // some toplevel modules (really just Base) aren't actually - if (!foreach_mtable_in_module(m, visit, env)) { - JL_GC_POP(); + if (!foreach_mtable_in_module(m, visit, env)) return 0; - } } - JL_GC_POP(); - } - else { - if (!foreach_mtable_in_module(jl_main_module, visit, env)) - return 0; - if (!foreach_mtable_in_module(jl_core_module, visit, env)) - return 0; } return 1; } - jl_function_t *jl_typeinf_func JL_GLOBALLY_ROOTED = NULL; JL_DLLEXPORT size_t jl_typeinf_world = 1; @@ -923,7 +958,7 @@ static jl_value_t *inst_varargp_in_env(jl_value_t *decl, jl_svec_t *sparams) return vm; } -static jl_value_t *ml_matches(jl_methtable_t *mt, +static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, jl_tupletype_t *type, int lim, int include_ambiguous, int intersections, size_t world, int cache_result, size_t *min_valid, size_t *max_valid, int *ambig); @@ -1130,9 +1165,10 @@ static void jl_compilation_sig( // and the types we find should be bigger. if (np >= nspec && jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND) { if (!*newparams) *newparams = tt->parameters; - if (max_varargs > 0) { + if (max_varargs > 0 && nspec >= 2) { type_i = jl_svecref(*newparams, nspec - 2); - } else { + } + else { // If max varargs is zero, always specialize to (Any...) since // there is no preceding parameter to use for `type_i` type_i = jl_bottom_type; @@ -1207,15 +1243,11 @@ JL_DLLEXPORT int jl_isa_compileable_sig( if (definition->isva) { unsigned nspec_min = nargs + 1; // min number of arg values (including tail vararg) unsigned nspec_max = INT32_MAX; // max number of arg values (including tail vararg) - jl_methtable_t *mt = jl_method_table_for(decl); - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(decl) : mt; - if ((jl_value_t*)mt != jl_nothing) { - // try to refine estimate of min and max - uint8_t heuristic_used = 0; - nspec_max = nspec_min = nargs + get_max_varargs(definition, kwmt, mt, &heuristic_used); - if (heuristic_used) - nspec_max = INT32_MAX; // new methods may be added, increasing nspec_min later - } + // try to refine estimate of min and max + uint8_t heuristic_used = 0; + nspec_max = nspec_min = nargs + get_max_varargs(definition, &heuristic_used); + if (heuristic_used) + nspec_max = INT32_MAX; // new methods may be added, increasing nspec_min later int isunbound = (jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND); if (jl_is_vararg(jl_tparam(type, np - 1))) { if (!isunbound || np < nspec_min || np > nspec_max) @@ -1404,18 +1436,18 @@ static inline jl_typemap_entry_t *lookup_leafcache(jl_genericmemory_t *leafcache return NULL; } jl_method_instance_t *cache_method( - jl_methtable_t *mt, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, + jl_methtable_t *mt, jl_methcache_t *mc, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, jl_tupletype_t *tt, // the original tupletype of the signature jl_method_t *definition, size_t world, size_t min_valid, size_t max_valid, jl_svec_t *sparams) { - // caller must hold the mt->writelock + // caller must hold the parent->writelock // short-circuit (now that we hold the lock) if this entry is already present - int8_t offs = mt ? jl_cachearg_offset(mt) : 1; + int8_t offs = mc ? jl_cachearg_offset() : 1; { // scope block - if (mt) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + if (mc) { + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); if (entry) return entry->func.linfo; @@ -1428,10 +1460,23 @@ jl_method_instance_t *cache_method( return entry->func.linfo; } + jl_method_instance_t *newmeth = NULL; + if (definition->sig == (jl_value_t*)jl_anytuple_type && definition != jl_opaque_closure_method && !definition->is_for_opaque_closure) { + newmeth = jl_atomic_load_relaxed(&definition->unspecialized); + if (newmeth != NULL) { // handle builtin methods de-specialization (for invoke, or if the global cache entry somehow gets lost) + jl_tupletype_t *cachett = (jl_tupletype_t*)newmeth->specTypes; + assert(cachett != jl_anytuple_type); + jl_typemap_entry_t *newentry = jl_typemap_alloc(cachett, NULL, jl_emptysvec, (jl_value_t*)newmeth, min_valid, max_valid); + JL_GC_PUSH1(&newentry); + jl_typemap_insert(cache, parent, newentry, offs); + JL_GC_POP(); + return newmeth; + } + } + jl_value_t *temp = NULL; jl_value_t *temp2 = NULL; jl_value_t *temp3 = NULL; - jl_method_instance_t *newmeth = NULL; jl_svec_t *newparams = NULL; JL_GC_PUSH5(&temp, &temp2, &temp3, &newmeth, &newparams); @@ -1439,8 +1484,7 @@ jl_method_instance_t *cache_method( // so that we can minimize the number of required cache entries. int cache_with_orig = 1; jl_tupletype_t *compilationsig = tt; - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(definition->sig) : mt; - intptr_t max_varargs = get_max_varargs(definition, kwmt, mt, NULL); + intptr_t max_varargs = get_max_varargs(definition, NULL); jl_compilation_sig(tt, sparams, definition, max_varargs, &newparams); if (newparams) { temp2 = jl_apply_tuple_type(newparams, 1); @@ -1476,7 +1520,7 @@ jl_method_instance_t *cache_method( // now examine what will happen if we chose to use this sig in the cache size_t min_valid2 = 1; size_t max_valid2 = ~(size_t)0; - temp = ml_matches(mt, compilationsig, MAX_UNSPECIALIZED_CONFLICTS, 1, 1, world, 0, &min_valid2, &max_valid2, NULL); + temp = ml_matches(mt, mc, compilationsig, MAX_UNSPECIALIZED_CONFLICTS, 1, 1, world, 0, &min_valid2, &max_valid2, NULL); int guards = 0; if (temp == jl_nothing) { cache_with_orig = 1; @@ -1524,7 +1568,7 @@ jl_method_instance_t *cache_method( guards++; // alternative approach: insert sentinel entry //jl_typemap_insert(cache, parent, (jl_tupletype_t*)matc->spec_types, - // NULL, jl_emptysvec, /*guard*/NULL, jl_cachearg_offset(mt), other->min_world, other->max_world); + // NULL, jl_emptysvec, /*guard*/NULL, jl_cachearg_offset(), other->min_world, other->max_world); } } assert(guards == jl_svec_len(guardsigs)); @@ -1584,7 +1628,7 @@ jl_method_instance_t *cache_method( jl_typemap_entry_t *newentry = jl_typemap_alloc(cachett, simplett, guardsigs, (jl_value_t*)newmeth, min_valid, max_valid); temp = (jl_value_t*)newentry; - if (mt && cachett == tt && jl_svec_len(guardsigs) == 0 && tt->hash && !tt->hasfreetypevars) { + if (mc && cachett == tt && jl_svec_len(guardsigs) == 0 && tt->hash && !tt->hasfreetypevars) { // we check `tt->hash` exists, since otherwise the NamedTuple // constructor and `structdiff` method pollutes this lookup with a lot // of garbage in the linear table search @@ -1597,14 +1641,14 @@ jl_method_instance_t *cache_method( jl_cache_type_(tt); JL_UNLOCK(&typecache_lock); // Might GC } - jl_genericmemory_t *oldcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *oldcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *old = (jl_typemap_entry_t*)jl_eqtable_get(oldcache, (jl_value_t*)tt, jl_nothing); jl_atomic_store_relaxed(&newentry->next, old); jl_gc_wb(newentry, old); - jl_genericmemory_t *newcache = jl_eqtable_put(jl_atomic_load_relaxed(&mt->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); + jl_genericmemory_t *newcache = jl_eqtable_put(jl_atomic_load_relaxed(&mc->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); if (newcache != oldcache) { - jl_atomic_store_release(&mt->leafcache, newcache); - jl_gc_wb(mt, newcache); + jl_atomic_store_release(&mc->leafcache, newcache); + jl_gc_wb(mc, newcache); } } else { @@ -1624,50 +1668,52 @@ jl_method_instance_t *cache_method( return newmeth; } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_value_t *mt, size_t world, size_t *min_valid, size_t *max_valid); +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, size_t *min_valid, size_t *max_valid); + +JL_DLLEXPORT jl_typemap_entry_t *jl_mt_find_cache_entry(jl_methcache_t *mc JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED JL_ROOTS_TEMPORARILY, size_t world) +{ // exported only for debugging purposes, not for casual use + if (tt->isdispatchtuple) { + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); + jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); + if (entry) + return entry; + } + JL_GC_PUSH1(&tt); + struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mc->cache), &search, jl_cachearg_offset(), /*subtype*/1); + JL_GC_POP(); + return entry; +} -static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED, size_t world) +static jl_method_instance_t *jl_mt_assoc_by_type(jl_methcache_t *mc JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED, size_t world) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); + jl_typemap_entry_t *entry = jl_mt_find_cache_entry(mc, tt, world); if (entry) return entry->func.linfo; + assert(tt->isdispatchtuple || tt->hasfreetypevars); JL_TIMING(METHOD_LOOKUP_SLOW, METHOD_LOOKUP_SLOW); jl_method_match_t *matc = NULL; JL_GC_PUSH2(&tt, &matc); - JL_LOCK(&mt->writelock); - assert(tt->isdispatchtuple || tt->hasfreetypevars); + JL_LOCK(&mc->writelock); jl_method_instance_t *mi = NULL; - if (tt->isdispatchtuple) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); - if (entry) - mi = entry->func.linfo; - } - - if (!mi) { - struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->cache), &search, jl_cachearg_offset(mt), /*subtype*/1); - if (entry) - mi = entry->func.linfo; - } - + entry = jl_mt_find_cache_entry(mc, tt, world); + if (entry) + mi = entry->func.linfo; if (!mi) { size_t min_valid = 0; size_t max_valid = ~(size_t)0; - matc = _gf_invoke_lookup((jl_value_t*)tt, jl_nothing, world, &min_valid, &max_valid); + matc = _gf_invoke_lookup((jl_value_t*)tt, jl_method_table, world, &min_valid, &max_valid); if (matc) { jl_method_t *m = matc->method; jl_svec_t *env = matc->sparams; - mi = cache_method(mt, &mt->cache, (jl_value_t*)mt, tt, m, world, min_valid, max_valid, env); + mi = cache_method(jl_method_table, mc, &mc->cache, (jl_value_t*)mc, tt, m, world, min_valid, max_valid, env); } } - JL_UNLOCK(&mt->writelock); + JL_UNLOCK(&mc->writelock); JL_GC_POP(); return mi; } - struct matches_env { struct typemap_intersection_env match; jl_typemap_entry_t *newentry; @@ -1705,7 +1751,7 @@ static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_in return 1; } -static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t *newentry, jl_typemap_entry_t **replaced, int8_t offs, size_t world) +static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t *newentry, jl_typemap_entry_t **replaced, size_t world) { jl_tupletype_t *type = newentry->sig; jl_tupletype_t *ttypes = (jl_tupletype_t*)jl_unwrap_unionall((jl_value_t*)type); @@ -1724,7 +1770,7 @@ static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t /* .ti = */ NULL, /* .env = */ jl_emptysvec, /* .issubty = */ 0}, /* .newentry = */ newentry, /* .shadowed */ NULL, /* .replaced */ NULL}; JL_GC_PUSH3(&env.match.env, &env.match.ti, &env.shadowed); - jl_typemap_intersection_visitor(defs, offs, &env.match); + jl_typemap_intersection_visitor(defs, 0, &env.match); env.match.env = NULL; env.match.ti = NULL; *replaced = env.replaced; @@ -1748,7 +1794,7 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue jl_module_t *newmod = method->module; jl_module_t *oldmod = oldvalue->module; jl_datatype_t *dt = jl_nth_argument_datatype(oldvalue->sig, 1); - if (dt == (jl_datatype_t*)jl_typeof(jl_kwcall_func)) + if (jl_kwcall_type && dt == jl_kwcall_type) dt = jl_nth_argument_datatype(oldvalue->sig, 3); int anon = dt && is_anonfn_typename(jl_symbol_name(dt->name->name)); if ((jl_options.warn_overwrite == JL_OPTIONS_WARN_OVERWRITE_ON) || @@ -1774,18 +1820,20 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue } } -static void update_max_args(jl_methtable_t *mt, jl_value_t *type) +static void update_max_args(jl_value_t *type) { - if (mt == jl_type_type_mt || mt == jl_nonfunction_mt || mt == jl_kwcall_mt) - return; type = jl_unwrap_unionall(type); + jl_datatype_t *dt = jl_nth_argument_datatype(type, 1); + if (dt == NULL || dt == jl_kwcall_type || jl_is_type_type((jl_value_t*)dt)) + return; + jl_typename_t *tn = dt->name; assert(jl_is_datatype(type)); size_t na = jl_nparams(type); if (jl_va_tuple_kind((jl_datatype_t*)type) == JL_VARARG_UNBOUND) na--; - // update occurs inside mt->writelock - if (na > jl_atomic_load_relaxed(&mt->max_args)) - jl_atomic_store_relaxed(&mt->max_args, na); + // update occurs inside global writelock + if (na > jl_atomic_load_relaxed(&tn->max_args)) + jl_atomic_store_relaxed(&tn->max_args, na); } jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED = NULL; @@ -1829,7 +1877,9 @@ static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_wo jl_atomic_store_release(&replaced->max_world, max_world); // recurse to all backedges to update their valid range also _invalidate_backedges(replaced_mi, replaced, max_world, depth + 1); - } else { + // TODO: should we visit all forward edges now and delete ourself from all of those lists too? + } + else { assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); } JL_UNLOCK(&replaced_mi->def.method->writelock); @@ -1893,6 +1943,33 @@ static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_ins JL_GC_POP(); } +static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) +{ + *isect2 = NULL; + int is_subty = 0; + *isect = jl_type_intersection_env_s(t1, t2, NULL, &is_subty); + if (*isect == jl_bottom_type) + return 0; + if (is_subty) + return 1; + // TODO: sometimes type intersection returns types with free variables + if (jl_has_free_typevars(t1) || jl_has_free_typevars(t2)) + return 1; + // determine if type-intersection can be convinced to give a better, non-bad answer + // if the intersection was imprecise, see if we can do better by switching the types + *isect2 = jl_type_intersection(t2, t1); + if (*isect2 == jl_bottom_type) { + *isect = jl_bottom_type; + *isect2 = NULL; + return 0; + } + if (jl_types_egal(*isect2, *isect)) { + *isect2 = NULL; + } + return 1; +} + + enum morespec_options { morespec_unknown, morespec_isnot, @@ -2010,39 +2087,128 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, JL_UNLOCK(&callee->def.method->writelock); } -// add a backedge from a non-existent signature to caller -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller) + +struct _typename_add_backedge { + jl_value_t *typ; + jl_value_t *caller; +}; + +static void _typename_add_backedge(jl_typename_t *tn, void *env0) { - assert(jl_is_code_instance(caller)); - if (!jl_atomic_load_relaxed(&allow_new_worlds)) - return; - JL_LOCK(&mt->writelock); + struct _typename_add_backedge *env = (struct _typename_add_backedge*)env0; + JL_GC_PROMISE_ROOTED(env->typ); + JL_GC_PROMISE_ROOTED(env->caller); if (jl_atomic_load_relaxed(&allow_new_worlds)) { - if (!mt->backedges) { + if (!tn->backedges) { // lazy-init the backedges array - mt->backedges = jl_alloc_vec_any(2); - jl_gc_wb(mt, mt->backedges); - jl_array_ptr_set(mt->backedges, 0, typ); - jl_array_ptr_set(mt->backedges, 1, caller); + tn->backedges = jl_alloc_vec_any(2); + jl_gc_wb(tn, tn->backedges); + jl_array_ptr_set(tn->backedges, 0, env->typ); + jl_array_ptr_set(tn->backedges, 1, env->caller); } else { // check if the edge is already present and avoid adding a duplicate - size_t i, l = jl_array_nrows(mt->backedges); + size_t i, l = jl_array_nrows(tn->backedges); // reuse an already cached instance of this type, if possible // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) != (jl_value_t*)caller) { - if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { - typ = jl_array_ptr_ref(mt->backedges, i - 1); + if (jl_array_ptr_ref(tn->backedges, i) != env->caller) { + if (jl_types_equal(jl_array_ptr_ref(tn->backedges, i - 1), env->typ)) { + env->typ = jl_array_ptr_ref(tn->backedges, i - 1); break; } } } - jl_array_ptr_1d_push(mt->backedges, typ); - jl_array_ptr_1d_push(mt->backedges, (jl_value_t*)caller); + jl_array_ptr_1d_push(tn->backedges, env->typ); + jl_array_ptr_1d_push(tn->backedges, env->caller); } } - JL_UNLOCK(&mt->writelock); +} + +// add a backedge from a non-existent signature to caller +JL_DLLEXPORT void jl_method_table_add_backedge(jl_value_t *typ, jl_code_instance_t *caller) +{ + assert(jl_is_code_instance(caller)); + if (!jl_atomic_load_relaxed(&allow_new_worlds)) + return; + // try to pick the best cache(s) for this typ edge + struct _typename_add_backedge env = {typ, (jl_value_t*)caller}; + jl_methcache_t *mc = jl_method_table->cache; + JL_LOCK(&mc->writelock); + if (jl_atomic_load_relaxed(&allow_new_worlds)) + jl_foreach_top_typename_for(_typename_add_backedge, typ, &env); + JL_UNLOCK(&mc->writelock); +} + +struct _typename_invalidate_backedge { + jl_value_t *type; + jl_value_t **isect; + jl_value_t **isect2; + jl_method_t *const *d; + size_t n; + size_t max_world; + int invalidated; +}; + +static void _typename_invalidate_backedges(jl_typename_t *tn, void *env0) +{ + struct _typename_invalidate_backedge *env = (struct _typename_invalidate_backedge*)env0; + JL_GC_PROMISE_ROOTED(env->type); + JL_GC_PROMISE_ROOTED(env->isect); // isJuliaType considers jl_value_t** to be a julia object too + JL_GC_PROMISE_ROOTED(env->isect2); // isJuliaType considers jl_value_t** to be a julia object too + if (tn->backedges) { + jl_value_t **backedges = jl_array_ptr_data(tn->backedges); + size_t i, na = jl_array_nrows(tn->backedges); + size_t ins = 0; + for (i = 1; i < na; i += 2) { + jl_value_t *backedgetyp = backedges[i - 1]; + JL_GC_PROMISE_ROOTED(backedgetyp); + int missing = 0; + if (jl_type_intersection2(backedgetyp, (jl_value_t*)env->type, env->isect, env->isect2)) { + // See if the intersection was actually already fully + // covered, but that the new method is ambiguous. + // -> no previous method: now there is one, need to update the missing edge + // -> one+ previously matching method(s): + // -> more specific then all of them: need to update the missing edge + // -> some may have been ambiguous: now there is a replacement + // -> some may have been called: now there is a replacement (also will be detected in the loop later) + // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) + // -> some may have been ambiguous: still are + // -> some may have been called: they may be partly replaced (will be detected in the loop later) + // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against + missing = 1; + for (size_t j = 0; j < env->n; j++) { + jl_method_t *m = env->d[j]; + JL_GC_PROMISE_ROOTED(m); + if (jl_subtype(*env->isect, m->sig) || (*env->isect2 && jl_subtype(*env->isect2, m->sig))) { + // We now know that there actually was a previous + // method for this part of the type intersection. + if (!jl_type_morespecific(env->type, m->sig)) { + missing = 0; + break; + } + } + } + } + *env->isect = *env->isect2 = NULL; + if (missing) { + jl_code_instance_t *backedge = (jl_code_instance_t*)backedges[i]; + JL_GC_PROMISE_ROOTED(backedge); + invalidate_code_instance(backedge, env->max_world, 0); + env->invalidated = 1; + if (_jl_debug_method_invalidation) + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); + } + else { + backedges[ins++] = backedges[i - 1]; + backedges[ins++] = backedges[i - 0]; + } + } + if (ins == 0) + tn->backedges = NULL; + else + jl_array_del_end(tn->backedges, na - ins); + } } struct invalidate_mt_env { @@ -2128,35 +2294,36 @@ static jl_typemap_entry_t *do_typemap_search(jl_methtable_t *mt JL_PROPAGATES_RO return (jl_typemap_entry_t *)closure; } -static void jl_method_table_invalidate(jl_methtable_t *mt, jl_method_t *replaced, size_t max_world) +static void _method_table_invalidate(jl_methcache_t *mc, void *env0) { - if (jl_options.incremental && jl_generating_output()) - jl_error("Method deletion is not possible during Module precompile."); - assert(!replaced->is_for_opaque_closure); - assert(jl_atomic_load_relaxed(&jl_world_counter) == max_world); - // drop this method from mt->cache - struct disable_mt_env mt_cache_env; - mt_cache_env.max_world = max_world; - mt_cache_env.replaced = replaced; - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), disable_mt_cache, (void*)&mt_cache_env); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + // drop this method from mc->cache + jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), disable_mt_cache, env0); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); size_t i, l = leafcache->length; for (i = 1; i < l; i += 2) { jl_typemap_entry_t *oldentry = (jl_typemap_entry_t*)jl_genericmemory_ptr_ref(leafcache, i); if (oldentry) { while ((jl_value_t*)oldentry != jl_nothing) { - disable_mt_cache(oldentry, (void*)&mt_cache_env); + disable_mt_cache(oldentry, env0); oldentry = jl_atomic_load_relaxed(&oldentry->next); } } } +} + +static void jl_method_table_invalidate(jl_method_t *replaced, size_t max_world) +{ + if (jl_options.incremental && jl_generating_output()) + jl_error("Method deletion is not possible during Module precompile."); + assert(!replaced->is_for_opaque_closure); + assert(jl_atomic_load_relaxed(&jl_world_counter) == max_world); // Invalidate the backedges int invalidated = 0; jl_value_t *specializations = jl_atomic_load_relaxed(&replaced->specializations); JL_GC_PUSH1(&specializations); if (!jl_is_svec(specializations)) specializations = (jl_value_t*)jl_svec1(specializations); - l = jl_svec_len(specializations); + size_t i, l = jl_svec_len(specializations); for (i = 0; i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i); if ((jl_value_t*)mi != jl_nothing) { @@ -2164,6 +2331,12 @@ static void jl_method_table_invalidate(jl_methtable_t *mt, jl_method_t *replaced invalidate_backedges(mi, max_world, "jl_method_table_disable"); } } + + jl_methtable_t *mt = jl_method_get_table(replaced); + struct disable_mt_env mt_cache_env; + mt_cache_env.max_world = max_world; + mt_cache_env.replaced = replaced; + _method_table_invalidate(mt->cache, &mt_cache_env); JL_GC_POP(); // XXX: this might have resolved an ambiguity, for which we have not tracked the edge here, // and thus now introduce a mistake into inference @@ -2200,15 +2373,17 @@ static int erase_method_backedges(jl_typemap_entry_t *def, void *closure) static int erase_all_backedges(jl_methtable_t *mt, void *env) { - // removes all method caches - // this might not be entirely safe (GC or MT), thus we only do it very early in bootstrapping - JL_LOCK(&mt->writelock); - mt->backedges = NULL; - JL_UNLOCK(&mt->writelock); - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), erase_method_backedges, env); + return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), erase_method_backedges, env); +} + +static int erase_all_mc_backedges(jl_typename_t *tn, void *env) +{ + tn->backedges = NULL; return 1; } +static int jl_foreach_reachable_typename(int (*visit)(jl_typename_t *tn, void *env), jl_array_t *mod_array, void *env); + JL_DLLEXPORT void jl_disable_new_worlds(void) { if (jl_generating_output()) @@ -2216,59 +2391,39 @@ JL_DLLEXPORT void jl_disable_new_worlds(void) JL_LOCK(&world_counter_lock); jl_atomic_store_relaxed(&allow_new_worlds, 0); JL_UNLOCK(&world_counter_lock); - jl_foreach_reachable_mtable(erase_all_backedges, (void*)NULL); + jl_array_t *mod_array = jl_get_loaded_modules(); + JL_GC_PUSH1(&mod_array); + jl_foreach_reachable_mtable(erase_all_backedges, mod_array, (void*)NULL); + + JL_LOCK(&jl_method_table->cache->writelock); + jl_foreach_reachable_typename(erase_all_mc_backedges, mod_array, (void*)NULL); + JL_UNLOCK(&jl_method_table->cache->writelock); + JL_GC_POP(); } -JL_DLLEXPORT void jl_method_table_disable(jl_methtable_t *mt, jl_method_t *method) +JL_DLLEXPORT void jl_method_table_disable(jl_method_t *method) { + jl_methtable_t *mt = jl_method_get_table(method); jl_typemap_entry_t *methodentry = do_typemap_search(mt, method); JL_LOCK(&world_counter_lock); if (!jl_atomic_load_relaxed(&allow_new_worlds)) jl_error("Method changes have been disabled via a call to disable_new_worlds."); int enabled = jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0; if (enabled) { - JL_LOCK(&mt->writelock); - // Narrow the world age on the method to make it uncallable + // Narrow the world age on the method to make it uncallable size_t world = jl_atomic_load_relaxed(&jl_world_counter); assert(method == methodentry->func.method); jl_atomic_store_relaxed(&method->dispatch_status, 0); assert(jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0); jl_atomic_store_relaxed(&methodentry->max_world, world); - jl_method_table_invalidate(mt, method, world); + jl_method_table_invalidate(method, world); jl_atomic_store_release(&jl_world_counter, world + 1); - JL_UNLOCK(&mt->writelock); - } + } JL_UNLOCK(&world_counter_lock); if (!enabled) jl_errorf("Method of %s already disabled", jl_symbol_name(method->name)); } -static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) -{ - *isect2 = NULL; - int is_subty = 0; - *isect = jl_type_intersection_env_s(t1, t2, NULL, &is_subty); - if (*isect == jl_bottom_type) - return 0; - if (is_subty) - return 1; - // TODO: sometimes type intersection returns types with free variables - if (jl_has_free_typevars(t1) || jl_has_free_typevars(t2)) - return 1; - // determine if type-intersection can be convinced to give a better, non-bad answer - // if the intersection was imprecise, see if we can do better by switching the types - *isect2 = jl_type_intersection(t2, t1); - if (*isect2 == jl_bottom_type) { - *isect = jl_bottom_type; - *isect2 = NULL; - return 0; - } - if (jl_types_egal(*isect2, *isect)) { - *isect2 = NULL; - } - return 1; -} - jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) { JL_TIMING(ADD_METHOD, ADD_METHOD); @@ -2277,30 +2432,32 @@ jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_timing_show_method(method, JL_TIMING_DEFAULT_BLOCK); jl_typemap_entry_t *newentry = NULL; JL_GC_PUSH1(&newentry); - JL_LOCK(&mt->writelock); // add our new entry assert(jl_atomic_load_relaxed(&method->primary_world) == ~(size_t)0); // min-world assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_ONLY) == 0); + JL_LOCK(&mt->cache->writelock); newentry = jl_typemap_alloc((jl_tupletype_t*)method->sig, simpletype, jl_emptysvec, (jl_value_t*)method, ~(size_t)0, 1); - jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); - update_max_args(mt, method->sig); - JL_UNLOCK(&mt->writelock); + jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, 0); + + if (mt == jl_method_table) + update_max_args(method->sig); + JL_UNLOCK(&mt->cache->writelock); JL_GC_POP(); return newentry; } -void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) +void jl_method_table_activate(jl_typemap_entry_t *newentry) { JL_TIMING(ADD_METHOD, ADD_METHOD); jl_method_t *method = newentry->func.method; + jl_methtable_t *mt = jl_method_get_table(method); assert(jl_is_mtable(mt)); assert(jl_is_method(method)); jl_timing_show_method(method, JL_TIMING_DEFAULT_BLOCK); jl_value_t *type = (jl_value_t*)newentry->sig; jl_value_t *oldvalue = NULL; jl_array_t *oldmi = NULL; - JL_LOCK(&mt->writelock); size_t world = jl_atomic_load_relaxed(&method->primary_world); assert(world == jl_atomic_load_relaxed(&jl_world_counter) + 1); // min-world assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); @@ -2317,7 +2474,8 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) JL_GC_PUSH6(&oldvalue, &oldmi, &loctag, &isect, &isect2, &isect3); jl_typemap_entry_t *replaced = NULL; // then check what entries we replaced - oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, jl_cachearg_offset(mt), max_world); + oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, max_world); + int invalidated = 0; int only = !(jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_PRECOMPILE_MANY); // will compute if this will be currently the only result that would returned from `ml_matches` given `sig` if (replaced) { @@ -2326,7 +2484,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) invalidated = 1; method_overwrite(newentry, m); // this is an optimized version of below, given we know the type-intersection is exact - jl_method_table_invalidate(mt, m, max_world); + jl_method_table_invalidate(m, max_world); int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); jl_atomic_store_relaxed(&m->dispatch_status, 0); only = m_dispatch & METHOD_SIG_LATEST_ONLY; @@ -2342,60 +2500,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) assert(jl_is_array(oldvalue)); d = (jl_method_t**)jl_array_ptr_data(oldvalue); n = jl_array_nrows(oldvalue); - } - if (mt->backedges) { - jl_value_t **backedges = jl_array_ptr_data(mt->backedges); - size_t i, na = jl_array_nrows(mt->backedges); - size_t ins = 0; - for (i = 1; i < na; i += 2) { - jl_value_t *backedgetyp = backedges[i - 1]; - JL_GC_PROMISE_ROOTED(backedgetyp); - int missing = 0; - if (jl_type_intersection2(backedgetyp, (jl_value_t*)type, &isect, &isect2)) { - // See if the intersection was actually already fully - // covered, but that the new method is ambiguous. - // -> no previous method: now there is one, need to update the missing edge - // -> one+ previously matching method(s): - // -> more specific then all of them: need to update the missing edge - // -> some may have been ambiguous: now there is a replacement - // -> some may have been called: now there is a replacement (also will be detected in the loop later) - // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) - // -> some may have been ambiguous: still are - // -> some may have been called: they may be partly replaced (will be detected in the loop later) - // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against - missing = 1; - size_t j; - for (j = 0; j < n; j++) { - jl_method_t *m = d[j]; - if (jl_subtype(isect, m->sig) || (isect2 && jl_subtype(isect2, m->sig))) { - // We now know that there actually was a previous - // method for this part of the type intersection. - if (!jl_type_morespecific(type, m->sig)) { - missing = 0; - break; - } - } - } - } - if (missing) { - jl_code_instance_t *backedge = (jl_code_instance_t*)backedges[i]; - JL_GC_PROMISE_ROOTED(backedge); - invalidate_code_instance(backedge, max_world, 0); - invalidated = 1; - if (_jl_debug_method_invalidation) - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); - } - else { - backedges[ins++] = backedges[i - 1]; - backedges[ins++] = backedges[i - 0]; - } - } - if (ins == 0) - mt->backedges = NULL; - else - jl_array_del_end(mt->backedges, na - ins); - } - if (oldvalue) { + oldmi = jl_alloc_vec_any(0); char *morespec = (char*)alloca(n); memset(morespec, morespec_unknown, n); @@ -2466,29 +2571,37 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) } } } - if (jl_array_nrows(oldmi)) { - // search mt->cache and leafcache and drop anything that might overlap with the new method - // this is very cheap, so we don't mind being fairly conservative at over-approximating this - struct invalidate_mt_env mt_cache_env; - mt_cache_env.max_world = max_world; - mt_cache_env.shadowed = oldmi; - mt_cache_env.newentry = newentry; - mt_cache_env.invalidated = 0; - - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), invalidate_mt_cache, (void*)&mt_cache_env); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - size_t i, l = leafcache->length; - for (i = 1; i < l; i += 2) { - jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); - if (entry) { - while (entry != jl_nothing) { - invalidate_mt_cache((jl_typemap_entry_t*)entry, (void*)&mt_cache_env); - entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); - } + } + + jl_methcache_t *mc = jl_method_table->cache; + JL_LOCK(&mc->writelock); + struct _typename_invalidate_backedge typename_env = {type, &isect, &isect2, d, n, max_world, invalidated}; + jl_foreach_top_typename_for(_typename_invalidate_backedges, type, &typename_env); + invalidated |= typename_env.invalidated; + if (oldmi && jl_array_nrows(oldmi)) { + // search mc->cache and leafcache and drop anything that might overlap with the new method + // this is very cheap, so we don't mind being fairly conservative at over-approximating this + struct invalidate_mt_env mt_cache_env; + mt_cache_env.max_world = max_world; + mt_cache_env.shadowed = oldmi; + mt_cache_env.newentry = newentry; + mt_cache_env.invalidated = 0; + + jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), invalidate_mt_cache, (void*)&mt_cache_env); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); + size_t i, l = leafcache->length; + for (i = 1; i < l; i += 2) { + jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); + if (entry) { + while (entry != jl_nothing) { + invalidate_mt_cache((jl_typemap_entry_t*)entry, (void*)&mt_cache_env); + entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); } } } + invalidated |= mt_cache_env.invalidated; } + JL_UNLOCK(&mc->writelock); } if (invalidated && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)method); @@ -2497,7 +2610,6 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) } jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); jl_atomic_store_relaxed(&method->dispatch_status, METHOD_SIG_LATEST_WHICH | (only ? METHOD_SIG_LATEST_ONLY : 0)); // TODO: this should be sequenced fully after the world counter store - JL_UNLOCK(&mt->writelock); JL_GC_POP(); } @@ -2510,7 +2622,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method jl_error("Method changes have been disabled via a call to disable_new_worlds."); size_t world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_atomic_store_relaxed(&method->primary_world, world); - jl_method_table_activate(mt, newentry); + jl_method_table_activate(newentry); jl_atomic_store_release(&jl_world_counter, world); JL_UNLOCK(&world_counter_lock); JL_GC_POP(); @@ -2562,13 +2674,15 @@ static jl_tupletype_t *lookup_arg_type_tuple(jl_value_t *arg1 JL_PROPAGATES_ROOT JL_DLLEXPORT jl_value_t *jl_method_lookup_by_tt(jl_tupletype_t *tt, size_t world, jl_value_t *_mt) { jl_methtable_t *mt = NULL; - if (_mt == jl_nothing) - mt = jl_gf_ft_mtable(jl_tparam0(tt)); + if (_mt == jl_nothing) { + mt = jl_method_table; + } else { - assert(jl_isa(_mt, (jl_value_t*)jl_methtable_type)); + assert(jl_is_mtable(_mt)); mt = (jl_methtable_t*) _mt; } - jl_method_instance_t* mi = jl_mt_assoc_by_type(mt, tt, world); + jl_methcache_t *mc = mt->cache; + jl_method_instance_t *mi = jl_mt_assoc_by_type(mc, tt, world); if (!mi) return jl_nothing; return (jl_value_t*) mi; @@ -2577,13 +2691,13 @@ JL_DLLEXPORT jl_value_t *jl_method_lookup_by_tt(jl_tupletype_t *tt, size_t world JL_DLLEXPORT jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t world) { assert(nargs > 0 && "expected caller to handle this case"); - jl_methtable_t *mt = jl_gf_mtable(args[0]); - jl_typemap_t *cache = jl_atomic_load_relaxed(&mt->cache); // XXX: gc root for this? - jl_typemap_entry_t *entry = jl_typemap_assoc_exact(cache, args[0], &args[1], nargs, jl_cachearg_offset(mt), world); + jl_methcache_t *mc = jl_method_table->cache; + jl_typemap_t *cache = jl_atomic_load_relaxed(&mc->cache); // XXX: gc root for this? + jl_typemap_entry_t *entry = jl_typemap_assoc_exact(cache, args[0], &args[1], nargs, jl_cachearg_offset(), world); if (entry) return entry->func.linfo; jl_tupletype_t *tt = arg_type_tuple(args[0], &args[1], nargs); - return jl_mt_assoc_by_type(mt, tt, world); + return jl_mt_assoc_by_type(mc, tt, world); } // return a Vector{Any} of svecs, each describing a method match: @@ -2606,10 +2720,9 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t * if (unw == (jl_value_t*)jl_emptytuple_type || jl_tparam0(unw) == jl_bottom_type) return (jl_value_t*)jl_an_empty_vec_any; if (mt == jl_nothing) - mt = (jl_value_t*)jl_method_table_for(unw); - if (mt == jl_nothing) - mt = NULL; - return ml_matches((jl_methtable_t*)mt, types, lim, include_ambiguous, 1, world, 1, min_valid, max_valid, ambig); + mt = (jl_value_t*)jl_method_table; + jl_methcache_t *mc = ((jl_methtable_t*)mt)->cache; + return ml_matches((jl_methtable_t*)mt, mc, types, lim, include_ambiguous, 1, world, 1, min_valid, max_valid, ambig); } JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT) @@ -3133,14 +3246,13 @@ JL_DLLEXPORT int32_t jl_invoke_api(jl_code_instance_t *codeinst) return -1; } -JL_DLLEXPORT jl_value_t *jl_normalize_to_compilable_sig(jl_methtable_t *mt, jl_tupletype_t *ti, jl_svec_t *env, jl_method_t *m, +JL_DLLEXPORT jl_value_t *jl_normalize_to_compilable_sig(jl_tupletype_t *ti, jl_svec_t *env, jl_method_t *m, int return_if_compileable) { jl_tupletype_t *tt = NULL; jl_svec_t *newparams = NULL; JL_GC_PUSH2(&tt, &newparams); - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(m->sig) : mt; - intptr_t max_varargs = get_max_varargs(m, kwmt, mt, NULL); + intptr_t max_varargs = get_max_varargs(m, NULL); jl_compilation_sig(ti, env, m, max_varargs, &newparams); int is_compileable = ((jl_datatype_t*)ti)->isdispatchtuple; if (newparams) { @@ -3166,10 +3278,7 @@ jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_ jl_method_t *def = mi->def.method; if (!jl_is_method(def) || !jl_is_datatype(mi->specTypes)) return mi; - jl_methtable_t *mt = jl_method_get_table(def); - if ((jl_value_t*)mt == jl_nothing) - return mi; - jl_value_t *compilationsig = jl_normalize_to_compilable_sig(mt, (jl_datatype_t*)mi->specTypes, mi->sparam_vals, def, 1); + jl_value_t *compilationsig = jl_normalize_to_compilable_sig((jl_datatype_t*)mi->specTypes, mi->sparam_vals, def, 1); if (compilationsig == jl_nothing || jl_egal(compilationsig, mi->specTypes)) return mi; jl_svec_t *env = NULL; @@ -3190,30 +3299,28 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc jl_tupletype_t *ti = match->spec_types; jl_method_instance_t *mi = NULL; if (jl_is_datatype(ti)) { - jl_methtable_t *mt = jl_method_get_table(m); - assert(mt != NULL); - if ((jl_value_t*)mt != jl_nothing) { - // get the specialization, possibly also caching it - if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { - // Since we also use this presence in the cache - // to trigger compilation when producing `.ji` files, - // inject it there now if we think it will be - // used via dispatch later (e.g. because it was hinted via a call to `precompile`) - JL_LOCK(&mt->writelock); - mi = cache_method(mt, &mt->cache, (jl_value_t*)mt, ti, m, world, min_valid, max_valid, env); - JL_UNLOCK(&mt->writelock); - } - else { - jl_value_t *tt = jl_normalize_to_compilable_sig(mt, ti, env, m, 1); - if (tt != jl_nothing) { - JL_GC_PUSH2(&tt, &env); - if (!jl_egal(tt, (jl_value_t*)ti)) { - jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)m->sig, &env); - assert(ti != jl_bottom_type); (void)ti; - } - mi = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); - JL_GC_POP(); + // get the specialization, possibly also caching it + if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { + // Since we also use this presence in the cache + // to trigger compilation when producing `.ji` files, + // inject it there now if we think it will be + // used via dispatch later (e.g. because it was hinted via a call to `precompile`) + jl_methcache_t *mc = jl_method_table->cache; + assert(mc); + JL_LOCK(&mc->writelock); + mi = cache_method(jl_method_get_table(m), mc, &mc->cache, (jl_value_t*)mc, ti, m, world, min_valid, max_valid, env); + JL_UNLOCK(&mc->writelock); + } + else { + jl_value_t *tt = jl_normalize_to_compilable_sig(ti, env, m, 1); + if (tt != jl_nothing) { + JL_GC_PUSH2(&tt, &env); + if (!jl_egal(tt, (jl_value_t*)ti)) { + jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)m->sig, &env); + assert(ti != jl_bottom_type); (void)ti; } + mi = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); + JL_GC_POP(); } } } @@ -3582,7 +3689,6 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t (callsite >> 16) & (N_CALL_CACHE - 1), (callsite >> 24 | callsite << 8) & (N_CALL_CACHE - 1)}; jl_typemap_entry_t *entry = NULL; - jl_methtable_t *mt = NULL; int i; // check each cache entry to see if it matches //#pragma unroll @@ -3609,8 +3715,8 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t if (i == 4) { // if no method was found in the associative cache, check the full cache JL_TIMING(METHOD_LOOKUP_FAST, METHOD_LOOKUP_FAST); - mt = jl_gf_mtable(F); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_methcache_t *mc = jl_method_table->cache; + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); entry = NULL; int cache_entry_count = jl_atomic_load_relaxed(&((jl_datatype_t*)FT)->name->cache_entry_count); if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && (cache_entry_count == 0 || cache_entry_count >= 8)) { @@ -3620,8 +3726,8 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); } if (entry == NULL) { - jl_typemap_t *cache = jl_atomic_load_relaxed(&mt->cache); // XXX: gc root required? - entry = jl_typemap_assoc_exact(cache, F, args, nargs, jl_cachearg_offset(mt), world); + jl_typemap_t *cache = jl_atomic_load_relaxed(&mc->cache); // XXX: gc root required? + entry = jl_typemap_assoc_exact(cache, F, args, nargs, jl_cachearg_offset(), world); if (entry == NULL) { last_alloc = jl_options.malloc_log ? jl_gc_diff_total_bytes() : 0; if (tt == NULL) { @@ -3649,7 +3755,8 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t else { assert(tt); // cache miss case - mfunc = jl_mt_assoc_by_type(mt, tt, world); + jl_methcache_t *mc = jl_method_table->cache; + mfunc = jl_mt_assoc_by_type(mc, tt, world); if (jl_options.malloc_log) jl_gc_sync_total_bytes(last_alloc); // discard allocation count from compilation if (mfunc == NULL) { @@ -3690,18 +3797,15 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t *F, jl_value_t **args, uint return _jl_invoke(F, args, nargs, mfunc, world); } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_value_t *mt, size_t world, size_t *min_valid, size_t *max_valid) +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, size_t *min_valid, size_t *max_valid) { jl_value_t *unw = jl_unwrap_unionall((jl_value_t*)types); if (!jl_is_tuple_type(unw)) return NULL; if (jl_tparam0(unw) == jl_bottom_type) return NULL; - if (mt == jl_nothing) - mt = (jl_value_t*)jl_method_table_for(unw); - if (mt == jl_nothing) - mt = NULL; - jl_value_t *matches = ml_matches((jl_methtable_t*)mt, (jl_tupletype_t*)types, 1, 0, 0, world, 1, min_valid, max_valid, NULL); + jl_methcache_t *mc = ((jl_methtable_t*)mt)->cache; + jl_value_t *matches = ml_matches((jl_methtable_t*)mt, mc, (jl_tupletype_t*)types, 1, 0, 0, world, 1, min_valid, max_valid, NULL); if (matches == jl_nothing || jl_array_nrows(matches) != 1) return NULL; jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); @@ -3713,7 +3817,9 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types, jl_value_t *mt, // Deprecated: Use jl_gf_invoke_lookup_worlds for future development size_t min_valid = 0; size_t max_valid = ~(size_t)0; - jl_method_match_t *matc = _gf_invoke_lookup(types, mt, world, &min_valid, &max_valid); + if (mt == jl_nothing) + mt = (jl_value_t*)jl_method_table; + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, &min_valid, &max_valid); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc->method; @@ -3722,7 +3828,9 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types, jl_value_t *mt, JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup_worlds(jl_value_t *types, jl_value_t *mt, size_t world, size_t *min_world, size_t *max_world) { - jl_method_match_t *matc = _gf_invoke_lookup(types, mt, world, min_world, max_world); + if (mt == jl_nothing) + mt = (jl_value_t*)jl_method_table; + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, min_world, max_world); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc; @@ -3785,7 +3893,7 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value assert(sub); (void)sub; } - mfunc = cache_method(NULL, &method->invokes, (jl_value_t*)method, tt, method, 1, 1, ~(size_t)0, tpenv); + mfunc = cache_method(NULL, NULL, &method->invokes, (jl_value_t*)method, tt, method, 1, 1, ~(size_t)0, tpenv); } JL_UNLOCK(&method->writelock); JL_GC_POP(); @@ -3829,8 +3937,8 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_ 0, 0, 0); assert(jl_is_datatype(ftype)); JL_GC_PUSH1(&ftype); - ftype->name->mt->name = name; - jl_gc_wb(ftype->name->mt, name); + ftype->name->singletonname = name; + jl_gc_wb(ftype->name, name); jl_declare_constant_val3(NULL, module, tname, (jl_value_t*)ftype, PARTITION_KIND_CONST, new_world); jl_value_t *f = jl_new_struct(ftype); ftype->instance = f; @@ -3931,7 +4039,7 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio static int ml_mtable_visitor(jl_methtable_t *mt, void *closure0) { struct typemap_intersection_env* env = (struct typemap_intersection_env*)closure0; - return jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), jl_cachearg_offset(mt), env); + return jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), 0, env); } // Visit the candidate methods, starting from t[idx], to determine a possible valid sort ordering, @@ -4214,7 +4322,7 @@ static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, array // fully-covers is a Bool indicating subtyping, though temporarily it may be // tri-values, with `nothing` indicating a match that is not a subtype, but // which is dominated by one which is (and thus should be excluded unless ambiguous) -static jl_value_t *ml_matches(jl_methtable_t *mt, +static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, jl_tupletype_t *type, int lim, int include_ambiguous, int intersections, size_t world, int cache_result, size_t *min_valid, size_t *max_valid, int *ambig) @@ -4243,10 +4351,10 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_value_t *isect2 = NULL; JL_GC_PUSH6(&env.t, &env.matc, &env.match.env, &search.env, &env.match.ti, &isect2); - if (mt) { + if (mc) { // check the leaf cache if this type can be in there if (((jl_datatype_t*)unw)->isdispatchtuple) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)type, world); if (entry) { jl_method_instance_t *mi = entry->func.linfo; @@ -4279,7 +4387,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, } // then check the full cache if it seems profitable if (((jl_datatype_t*)unw)->isdispatchtuple) { - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->cache), &search, jl_cachearg_offset(mt), /*subtype*/1); + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mc->cache), &search, jl_cachearg_offset(), /*subtype*/1); if (entry && (((jl_datatype_t*)unw)->isdispatchtuple || entry->guardsigs == jl_emptysvec)) { jl_method_instance_t *mi = entry->func.linfo; jl_method_t *meth = mi->def.method; @@ -4305,23 +4413,14 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, return env.t; } } - if (!ml_mtable_visitor(mt, &env.match) && env.t == jl_an_empty_vec_any) { - JL_GC_POP(); - // if we return early without returning methods, set only the min/max valid collected from matching - *min_valid = env.match.min_valid; - *max_valid = env.match.max_valid; - return jl_nothing; - } } - else { - // else: scan everything - if (!jl_foreach_reachable_mtable(ml_mtable_visitor, &env.match) && env.t == jl_an_empty_vec_any) { - JL_GC_POP(); - // if we return early without returning methods, set only the min/max valid collected from matching - *min_valid = env.match.min_valid; - *max_valid = env.match.max_valid; - return jl_nothing; - } + // then scan everything + if (!ml_mtable_visitor(mt, &env.match) && env.t == jl_an_empty_vec_any) { + JL_GC_POP(); + // if we return early without returning methods, set only the min/max valid collected from matching + *min_valid = env.match.min_valid; + *max_valid = env.match.max_valid; + return jl_nothing; } // if we return early, set only the min/max valid collected from matching *min_valid = env.match.min_valid; @@ -4569,14 +4668,14 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, if (env.match.min_valid < min_world) env.match.min_valid = min_world; } - if (mt && cache_result && ((jl_datatype_t*)unw)->isdispatchtuple) { // cache_result parameter keeps this from being recursive + if (mc && cache_result && ((jl_datatype_t*)unw)->isdispatchtuple) { // cache_result parameter keeps this from being recursive if (len == 1 && !has_ambiguity) { env.matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, 0); jl_method_t *meth = env.matc->method; jl_svec_t *tpenv = env.matc->sparams; - JL_LOCK(&mt->writelock); - cache_method(mt, &mt->cache, (jl_value_t*)mt, (jl_tupletype_t*)unw, meth, world, env.match.min_valid, env.match.max_valid, tpenv); - JL_UNLOCK(&mt->writelock); + JL_LOCK(&mc->writelock); + cache_method(mt, mc, &mc->cache, (jl_value_t*)mc, (jl_tupletype_t*)unw, meth, world, env.match.min_valid, env.match.max_valid, tpenv); + JL_UNLOCK(&mc->writelock); } } *min_valid = env.match.min_valid; @@ -4654,7 +4753,7 @@ JL_DLLEXPORT void jl_extern_c(jl_value_t *name, jl_value_t *declrt, jl_tupletype } // save a record of this so that the alias is generated when we write an object file - jl_method_t *meth = (jl_method_t*)jl_methtable_lookup(ft->name->mt, (jl_value_t*)sigt, jl_atomic_load_acquire(&jl_world_counter)); + jl_method_t *meth = (jl_method_t*)jl_methtable_lookup((jl_value_t*)sigt, jl_atomic_load_acquire(&jl_world_counter)); if (!jl_is_method(meth)) jl_error("@ccallable: could not find requested method"); JL_GC_PUSH1(&meth); diff --git a/src/interpreter.c b/src/interpreter.c index 2f7f9f8947576..97c37cb99b9c1 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -101,9 +101,8 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) fname = eval_value(args[0], s); jl_methtable_t *mt = NULL; - if (jl_typetagis(fname, jl_methtable_type)) { + if (jl_is_mtable(fname)) mt = (jl_methtable_t*)fname; - } atypes = eval_value(args[1], s); meth = eval_value(args[2], s); jl_method_t *ret = jl_method_def((jl_svec_t*)atypes, mt, (jl_code_info_t*)meth, s->module); diff --git a/src/ircode.c b/src/ircode.c index ddd5bb29fdfac..9a94c4c62431a 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -1616,14 +1616,12 @@ void jl_init_serializer(void) jl_densearray_type, jl_function_type, jl_typename_type, jl_builtin_type, jl_task_type, jl_uniontype_type, jl_array_any_type, jl_intrinsic_type, - jl_methtable_type, jl_typemap_level_type, jl_voidpointer_type, jl_newvarnode_type, jl_abstractstring_type, jl_array_symbol_type, jl_anytuple_type, jl_tparam0(jl_anytuple_type), jl_emptytuple_type, jl_array_uint8_type, jl_array_uint32_type, jl_code_info_type, jl_typeofbottom_type, jl_typeofbottom_type->super, jl_namedtuple_type, jl_array_int32_type, jl_uint32_type, jl_uint64_type, - jl_type_type_mt, jl_nonfunction_mt, jl_opaque_closure_type, jl_memory_any_type, jl_memory_uint8_type, diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index df3b9c121837c..76e8368132424 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -65,7 +65,7 @@ XX(jl_interconditional_type) \ XX(jl_interrupt_exception) \ XX(jl_intrinsic_type) \ - XX(jl_kwcall_func) \ + XX(jl_kwcall_type) \ XX(jl_libdl_module) \ XX(jl_libdl_dlopen_func) \ XX(jl_lineinfonode_type) \ @@ -91,13 +91,13 @@ XX(jl_method_match_type) \ XX(jl_method_type) \ XX(jl_methtable_type) \ + XX(jl_methcache_type) \ XX(jl_missingcodeerror_type) \ XX(jl_module_type) \ XX(jl_n_threads_per_pool) \ XX(jl_namedtuple_type) \ XX(jl_namedtuple_typename) \ XX(jl_newvarnode_type) \ - XX(jl_nonfunction_mt) \ XX(jl_nothing) \ XX(jl_nothing_type) \ XX(jl_number_type) \ @@ -135,7 +135,6 @@ XX(jl_typename_type) \ XX(jl_typeofbottom_type) \ XX(jl_type_type) \ - XX(jl_type_type_mt) \ XX(jl_type_typename) \ XX(jl_uint16_type) \ XX(jl_uint32_type) \ diff --git a/src/jltypes.c b/src/jltypes.c index a3d10a87b8091..fcb15a1a9a69d 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2954,7 +2954,9 @@ void jl_init_types(void) JL_GC_DISABLED XX(symbol); jl_simplevector_type = jl_new_uninitialized_datatype(); XX(simplevector); + jl_methcache_type = jl_new_uninitialized_datatype(); jl_methtable_type = jl_new_uninitialized_datatype(); + jl_method_table = jl_new_method_table(jl_symbol("GlobalMethods"), core); jl_emptysvec = (jl_svec_t*)jl_gc_permobj(sizeof(void*), jl_simplevector_type, 0); jl_set_typetagof(jl_emptysvec, jl_simplevector_tag, GC_OLD_MARKED); @@ -2962,14 +2964,10 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type = (jl_datatype_t*)jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), core, NULL, jl_emptysvec); jl_any_type->super = jl_any_type; - jl_nonfunction_mt = jl_any_type->name->mt; - jl_any_type->name->mt = NULL; jl_datatype_t *type_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), core, jl_any_type, jl_emptysvec); jl_type_type = (jl_unionall_t*)type_type; jl_type_typename = type_type->name; - jl_type_type_mt = jl_new_method_table(jl_type_typename->name, core); - jl_type_typename->mt = jl_type_type_mt; // initialize them. lots of cycles. // NOTE: types are not actually mutable, but we want to ensure they are heap-allocated with stable addresses @@ -3004,56 +3002,60 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name = jl_new_typename_in(jl_symbol("TypeName"), core, 0, 1); jl_typename_type->name->wrapper = (jl_value_t*)jl_typename_type; - jl_typename_type->name->mt = jl_nonfunction_mt; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 17 - 2; - jl_typename_type->name->names = jl_perm_symsvec(17, "name", "module", + jl_typename_type->name->n_uninitialized = 19 - 2; + jl_typename_type->name->names = jl_perm_symsvec(19, "name", "module", "singletonname", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", - "mt", "partial", - "hash", "n_uninitialized", + "backedges", "partial", + "hash", "max_args", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", "cache_entry_count", "max_methods", "constprop_heuristic"); - const static uint32_t typename_constfields[1] = { 0b00011101000100111 }; // TODO: put back atomicfields and constfields in this list - const static uint32_t typename_atomicfields[1] = { 0b00100000110000000 }; + const static uint32_t typename_constfields[1] = { 0b0001101000001001011 }; // TODO: put back atomicfields and constfields in this list + const static uint32_t typename_atomicfields[1] = { 0b0010010001110000000 }; jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(17, jl_symbol_type, jl_any_type /*jl_module_type*/, - jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, + jl_typename_type->types = jl_svec(19, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, + jl_simplevector_type, + jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, - jl_methtable_type, jl_any_type, - jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/, + jl_methcache_type, jl_any_type, + jl_any_type /*jl_long_type*/, + jl_any_type /*jl_int32_type*/, + jl_any_type /*jl_int32_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/); + jl_methcache_type->name = jl_new_typename_in(jl_symbol("MethodCache"), core, 0, 1); + jl_methcache_type->name->wrapper = (jl_value_t*)jl_methcache_type; + jl_methcache_type->super = jl_any_type; + jl_methcache_type->parameters = jl_emptysvec; + jl_methcache_type->name->n_uninitialized = 4 - 2; + jl_methcache_type->name->names = jl_perm_symsvec(4, "leafcache", "cache", "", ""); + const static uint32_t methcache_atomicfields[1] = { 0b1111 }; + jl_methcache_type->name->atomicfields = methcache_atomicfields; + jl_precompute_memoized_dt(jl_methcache_type, 1); + jl_methcache_type->types = jl_svec(4, jl_any_type, jl_any_type, jl_any_type/*voidpointer*/, jl_any_type/*int32*/); + jl_methtable_type->name = jl_new_typename_in(jl_symbol("MethodTable"), core, 0, 1); jl_methtable_type->name->wrapper = (jl_value_t*)jl_methtable_type; - jl_methtable_type->name->mt = jl_nonfunction_mt; jl_methtable_type->super = jl_any_type; jl_methtable_type->parameters = jl_emptysvec; - jl_methtable_type->name->n_uninitialized = 11 - 6; - jl_methtable_type->name->names = jl_perm_symsvec(11, "name", "defs", - "leafcache", "cache", "max_args", - "module", "backedges", - "", "", "offs", ""); - const static uint32_t methtable_constfields[1] = { 0x00000020 }; // (1<<5); - const static uint32_t methtable_atomicfields[1] = { 0x0000001e }; // (1<<1)|(1<<2)|(1<<3)|(1<<4); + jl_methtable_type->name->n_uninitialized = 0; + jl_methtable_type->name->names = jl_perm_symsvec(4, "defs", "cache", "name", "module"); + const static uint32_t methtable_constfields[1] = { 0b1110 }; + const static uint32_t methtable_atomicfields[1] = { 0b0001 }; jl_methtable_type->name->constfields = methtable_constfields; jl_methtable_type->name->atomicfields = methtable_atomicfields; jl_precompute_memoized_dt(jl_methtable_type, 1); - jl_methtable_type->types = jl_svec(11, jl_symbol_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type/*jl_long*/, - jl_any_type/*module*/, jl_any_type/*any vector*/, - jl_any_type/*voidpointer*/, jl_any_type/*int32*/, - jl_any_type/*uint8*/, jl_any_type/*uint8*/); + jl_methtable_type->types = jl_svec(4, jl_any_type, jl_methcache_type, jl_symbol_type, jl_any_type /*jl_module_type*/); jl_symbol_type->name = jl_new_typename_in(jl_symbol("Symbol"), core, 0, 1); jl_symbol_type->name->wrapper = (jl_value_t*)jl_symbol_type; - jl_symbol_type->name->mt = jl_nonfunction_mt; jl_symbol_type->super = jl_any_type; jl_symbol_type->parameters = jl_emptysvec; jl_symbol_type->name->n_uninitialized = 0; @@ -3063,7 +3065,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_simplevector_type->name = jl_new_typename_in(jl_symbol("SimpleVector"), core, 0, 1); jl_simplevector_type->name->wrapper = (jl_value_t*)jl_simplevector_type; - jl_simplevector_type->name->mt = jl_nonfunction_mt; jl_simplevector_type->super = jl_any_type; jl_simplevector_type->parameters = jl_emptysvec; jl_simplevector_type->name->n_uninitialized = 0; @@ -3249,8 +3250,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_function_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Function"), core, jl_any_type, jl_emptysvec); jl_builtin_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Builtin"), core, jl_function_type, jl_emptysvec); - jl_function_type->name->mt = NULL; // subtypes of Function have independent method tables - jl_builtin_type->name->mt = NULL; // so they don't share the Any type table jl_svec_t *tv; @@ -3289,10 +3288,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(3, jl_module_type, jl_symbol_type, jl_binding_type), jl_emptysvec, 0, 0, 3); - core = jl_new_module(jl_symbol("Core"), NULL); - jl_type_typename->mt->module = core; - jl_core_module = core; - core = NULL; // not actually ready yet to use + jl_core_module = jl_new_module(jl_symbol("Core"), NULL); tv = jl_svec1(tvar("Backend")); jl_addrspace_typename = @@ -3376,11 +3372,11 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_an_empty_memory_any = (jl_value_t*)jl_alloc_memory_any(0); // used internally - jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); - jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); // finish initializing module Core core = jl_core_module; + jl_method_table->module = core; + jl_atomic_store_relaxed(&jl_method_table->cache->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); // export own name, so "using Foo" makes "Foo" itself visible jl_set_initial_const(core, core->name, (jl_value_t*)core, 1); @@ -3718,13 +3714,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(4, jl_type_type, jl_simplevector_type, jl_method_type, jl_bool_type), jl_emptysvec, 0, 0, 4); - // all Kinds share the Type method table (not the nonfunction one) - jl_unionall_type->name->mt = - jl_uniontype_type->name->mt = - jl_datatype_type->name->mt = - jl_typeofbottom_type->name->mt = - jl_type_type_mt; - jl_intrinsic_type = jl_new_primitivetype((jl_value_t*)jl_symbol("IntrinsicFunction"), core, jl_builtin_type, jl_emptysvec, 32); @@ -3848,23 +3837,20 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_datatype_type->types, 6, jl_int32_type); jl_svecset(jl_datatype_type->types, 7, jl_uint16_type); jl_svecset(jl_typename_type->types, 1, jl_module_type); - jl_svecset(jl_typename_type->types, 3, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 4, jl_voidpointer_type); - jl_svecset(jl_typename_type->types, 5, jl_type_type); + jl_svecset(jl_typename_type->types, 5, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 6, jl_type_type); - jl_svecset(jl_typename_type->types, 11, jl_long_type); - jl_svecset(jl_typename_type->types, 12, jl_int32_type); - jl_svecset(jl_typename_type->types, 13, jl_uint8_type); - jl_svecset(jl_typename_type->types, 14, jl_uint8_type); + jl_svecset(jl_typename_type->types, 7, jl_type_type); + jl_svecset(jl_typename_type->types, 12, jl_long_type); + jl_svecset(jl_typename_type->types, 13, jl_int32_type); + jl_svecset(jl_typename_type->types, 14, jl_int32_type); jl_svecset(jl_typename_type->types, 15, jl_uint8_type); jl_svecset(jl_typename_type->types, 16, jl_uint8_type); - jl_svecset(jl_methtable_type->types, 4, jl_long_type); - jl_svecset(jl_methtable_type->types, 5, jl_module_type); - jl_svecset(jl_methtable_type->types, 6, jl_array_any_type); - jl_svecset(jl_methtable_type->types, 7, jl_long_type); // voidpointer - jl_svecset(jl_methtable_type->types, 8, jl_long_type); // uint32_t plus alignment - jl_svecset(jl_methtable_type->types, 9, jl_uint8_type); - jl_svecset(jl_methtable_type->types, 10, jl_uint8_type); + jl_svecset(jl_typename_type->types, 17, jl_uint8_type); + jl_svecset(jl_typename_type->types, 18, jl_uint8_type); + jl_svecset(jl_methcache_type->types, 2, jl_long_type); // voidpointer + jl_svecset(jl_methcache_type->types, 3, jl_long_type); // uint32_t plus alignment + jl_svecset(jl_methtable_type->types, 3, jl_module_type); jl_svecset(jl_method_type->types, 13, jl_method_instance_type); //jl_svecset(jl_debuginfo_type->types, 0, jl_method_instance_type); // union(jl_method_instance_type, jl_method_type, jl_symbol_type) jl_svecset(jl_method_instance_type->types, 4, jl_code_instance_type); @@ -3879,6 +3865,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_compute_field_offsets(jl_uniontype_type); jl_compute_field_offsets(jl_tvar_type); jl_compute_field_offsets(jl_methtable_type); + jl_compute_field_offsets(jl_methcache_type); jl_compute_field_offsets(jl_method_instance_type); jl_compute_field_offsets(jl_code_instance_type); jl_compute_field_offsets(jl_unionall_type); @@ -3966,9 +3953,9 @@ void post_boot_hooks(void) jl_trimfailure_type = (jl_datatype_t*)core("TrimFailure"); jl_pair_type = core("Pair"); - jl_kwcall_func = core("kwcall"); - jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; - jl_atomic_store_relaxed(&jl_kwcall_mt->max_args, 0); + jl_value_t *kwcall_func = core("kwcall"); + jl_kwcall_type = (jl_datatype_t*)jl_typeof(kwcall_func); + jl_atomic_store_relaxed(&jl_kwcall_type->name->max_args, 0); jl_weakref_type = (jl_datatype_t*)core("WeakRef"); jl_vecelement_typename = ((jl_datatype_t*)jl_unwrap_unionall(core("VecElement")))->name; @@ -3976,27 +3963,6 @@ void post_boot_hooks(void) jl_abioverride_type = (jl_datatype_t*)core("ABIOverride"); jl_init_box_caches(); - - // set module field of primitive types - jl_svec_t *bindings = jl_atomic_load_relaxed(&jl_core_module->bindings); - jl_value_t **table = jl_svec_data(bindings); - for (size_t i = 0; i < jl_svec_len(bindings); i++) { - if (table[i] != jl_nothing) { - jl_binding_t *b = (jl_binding_t*)table[i]; - jl_value_t *v = jl_get_binding_value(b); - if (v) { - if (jl_is_unionall(v)) - v = jl_unwrap_unionall(v); - if (jl_is_datatype(v)) { - jl_datatype_t *tt = (jl_datatype_t*)v; - tt->name->module = jl_core_module; - if (tt->name->mt) - tt->name->mt->module = jl_core_module; - } - } - } - } - export_jl_small_typeof(); } diff --git a/src/julia.h b/src/julia.h index c7b19fb8b4530..70eb654d4b1f3 100644 --- a/src/julia.h +++ b/src/julia.h @@ -197,6 +197,7 @@ typedef struct _jl_datatype_t jl_tupletype_t; struct _jl_code_instance_t; typedef struct _jl_method_instance_t jl_method_instance_t; typedef struct _jl_globalref_t jl_globalref_t; +typedef struct _jl_typemap_entry_t jl_typemap_entry_t; // TypeMap is an implicitly defined type @@ -509,6 +510,7 @@ typedef struct { JL_DATA_TYPE jl_sym_t *name; struct _jl_module_t *module; + jl_sym_t *singletonname; // sometimes used for debug printing jl_svec_t *names; // field names const uint32_t *atomicfields; // if any fields are atomic, we record them here const uint32_t *constfields; // if any fields are const, we record them here @@ -518,15 +520,16 @@ typedef struct { _Atomic(jl_value_t*) Typeofwrapper; // cache for Type{wrapper} _Atomic(jl_svec_t*) cache; // sorted array _Atomic(jl_svec_t*) linearcache; // unsorted array - struct _jl_methtable_t *mt; + jl_array_t *backedges; // uncovered (sig => caller::CodeInstance) pairs with this type as the function jl_array_t *partial; // incomplete instantiations of this type intptr_t hash; + _Atomic(int32_t) max_args; // max # of non-vararg arguments in a signature with this type as the function int32_t n_uninitialized; // type properties uint8_t abstract:1; uint8_t mutabl:1; uint8_t mayinlinealloc:1; - uint8_t _reserved:5; + uint8_t _unused:5; _Atomic(uint8_t) cache_entry_count; // (approximate counter of TypeMapEntry for heuristics) uint8_t max_methods; // override for inference's max_methods setting (0 = no additional limit or relaxation) uint8_t constprop_heustic; // override for inference's constprop heuristic @@ -828,7 +831,7 @@ struct _jl_globalref_t { }; // one Type-to-Value entry -typedef struct _jl_typemap_entry_t { +struct _jl_typemap_entry_t { JL_DATA_TYPE _Atomic(struct _jl_typemap_entry_t*) next; // invasive linked list jl_tupletype_t *sig; // the type signature for this entry @@ -845,7 +848,7 @@ typedef struct _jl_typemap_entry_t { int8_t isleafsig; // isleaftype(sig) & !any(isType, sig) : unsorted and very fast int8_t issimplesig; // all(isleaftype | isAny | isType | isVararg, sig) : sorted and fast int8_t va; // isVararg(sig) -} jl_typemap_entry_t; +}; // one level in a TypeMap tree (each level splits on a type at a given offset) typedef struct _jl_typemap_level_t { @@ -865,19 +868,20 @@ typedef struct _jl_typemap_level_t { _Atomic(jl_typemap_t*) any; } jl_typemap_level_t; -// contains the TypeMap for one Type -typedef struct _jl_methtable_t { +typedef struct _jl_methcache_t { JL_DATA_TYPE - jl_sym_t *name; // sometimes used for debug printing - _Atomic(jl_typemap_t*) defs; _Atomic(jl_genericmemory_t*) leafcache; _Atomic(jl_typemap_t*) cache; - _Atomic(intptr_t) max_args; // max # of non-vararg arguments in a signature - jl_module_t *module; // sometimes used for debug printing - jl_array_t *backedges; // (sig, caller::CodeInstance) pairs jl_mutex_t writelock; - uint8_t offs; // 0, or 1 to skip splitting typemap on first (function) argument - uint8_t frozen; // whether this accepts adding new methods +} jl_methcache_t; + +// contains global MethodTable +typedef struct _jl_methtable_t { + JL_DATA_TYPE + _Atomic(jl_typemap_t*) defs; + jl_methcache_t *cache; + jl_sym_t *name; // sometimes used for debug printing + jl_module_t *module; // sometimes used for debug printing } jl_methtable_t; typedef struct { @@ -1100,16 +1104,17 @@ extern JL_DLLIMPORT jl_datatype_t *jl_upsilonnode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_quotenode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_newvarnode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_intrinsic_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_methcache_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_methtable_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_typemap_level_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_typemap_entry_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_kwcall_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_svec_t *jl_emptysvec JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_emptytuple JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_true JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_false JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_nothing JL_GLOBALLY_ROOTED; -extern JL_DLLIMPORT jl_value_t *jl_kwcall_func JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_libdl_dlopen_func JL_GLOBALLY_ROOTED; @@ -1452,9 +1457,7 @@ STATIC_INLINE void jl_array_uint32_set(void *a, size_t i, uint32_t x) JL_NOTSAFE #define jl_string_data(s) ((char*)s + sizeof(void*)) #define jl_string_len(s) (*(size_t*)s) -#define jl_gf_ft_mtable(ft) (((jl_datatype_t*)ft)->name->mt) -#define jl_gf_mtable(f) (jl_gf_ft_mtable(jl_typeof(f))) -#define jl_gf_name(f) (jl_gf_mtable(f)->name) +#define jl_gf_name(f) (((jl_datatype_t*)jl_typeof(f))->name->singletonname) // struct type info JL_DLLEXPORT jl_svec_t *jl_compute_fieldtypes(jl_datatype_t *st JL_PROPAGATES_ROOT, void *stack, int cacheable); @@ -1649,6 +1652,7 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_method(v) jl_typetagis(v,jl_method_type) #define jl_is_module(v) jl_typetagis(v,jl_module_tag<<4) #define jl_is_mtable(v) jl_typetagis(v,jl_methtable_type) +#define jl_is_mcache(v) jl_typetagis(v,jl_methcache_type) #define jl_is_task(v) jl_typetagis(v,jl_task_tag<<4) #define jl_is_string(v) jl_typetagis(v,jl_string_tag<<4) #define jl_is_cpointer(v) jl_is_cpointer_type(jl_typeof(v)) diff --git a/src/julia_internal.h b/src/julia_internal.h index 133a347bf82dc..b94011e17a3e3 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -388,9 +388,7 @@ static inline void memassign_safe(int hasptr, char *dst, const jl_value_t *src, #define GC_IN_IMAGE 4 // useful constants -extern JL_DLLIMPORT jl_methtable_t *jl_type_type_mt JL_GLOBALLY_ROOTED; -extern JL_DLLIMPORT jl_methtable_t *jl_nonfunction_mt JL_GLOBALLY_ROOTED; -extern jl_methtable_t *jl_kwcall_mt JL_GLOBALLY_ROOTED; +extern jl_methtable_t *jl_method_table JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_method_t *jl_opaque_closure_method JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT _Atomic(size_t) jl_world_counter; extern jl_debuginfo_t *jl_nulldebuginfo JL_GLOBALLY_ROOTED; @@ -806,9 +804,9 @@ int jl_has_concrete_subtype(jl_value_t *typ); jl_tupletype_t *jl_inst_arg_tuple_type(jl_value_t *arg1, jl_value_t **args, size_t nargs, int leaf); jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl_value_t **args, size_t nargs, int leaf); JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry); +void jl_method_table_activate(jl_typemap_entry_t *newentry); jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *name, jl_fptr_args_t fptr) JL_GC_DISABLED; +jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); @@ -858,7 +856,7 @@ int setonce_bits(jl_datatype_t *rty, char *p, jl_value_t *owner, jl_value_t *rhs jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module, size_t new_world); jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st, size_t new_world); -int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), void *env); +int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_array_t *mod_array, void *env); int foreach_mtable_in_module(jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), void *env); void jl_init_main_module(void); JL_DLLEXPORT int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT; @@ -928,10 +926,17 @@ jl_datatype_t *jl_nth_argument_datatype(jl_value_t *argtypes JL_PROPAGATES_ROOT, JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_table_for( jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_methcache_t *jl_method_cache_for( + jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; jl_methtable_t *jl_kwmethod_table_for( jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +jl_methcache_t *jl_kwmethod_cache_for( + jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_get_table( jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_methcache_t *jl_method_get_cache( + jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +void jl_foreach_top_typename_for(void (*f)(jl_typename_t*, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, void *env); JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t); JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; @@ -1287,17 +1292,18 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name); jl_module_t *jl_add_standard_imports(jl_module_t *m); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); +JL_DLLEXPORT jl_methcache_t *jl_new_method_cache(void); JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp) JL_PROPAGATES_ROOT; JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_value_t *type, size_t world); +JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_value_t *type, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins JL_PROPAGATES_ROOT); JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_code_instance_t *caller); -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller); +JL_DLLEXPORT void jl_method_table_add_backedge(jl_value_t *typ, jl_code_instance_t *caller); JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, jl_code_instance_t *ci JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, diff --git a/src/method.c b/src/method.c index f71f09ceb83bc..77863b27e24b6 100644 --- a/src/method.c +++ b/src/method.c @@ -16,7 +16,6 @@ extern "C" { #endif -jl_methtable_t *jl_kwcall_mt; jl_method_t *jl_opaque_closure_method; static void check_c_types(const char *where, jl_value_t *rt, jl_value_t *at) @@ -803,7 +802,8 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t else if (jl_is_mtable(kind)) { assert(i < l); ex = data[i++]; - jl_method_table_add_backedge((jl_methtable_t*)kind, ex, ci); + if ((jl_methtable_t*)kind == jl_method_table) + jl_method_table_add_backedge(ex, ci); } else { assert(i < l); @@ -1154,54 +1154,71 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) return gf; } -static jl_methtable_t *nth_methtable(jl_value_t *a JL_PROPAGATES_ROOT, int n) JL_NOTSAFEPOINT +static void foreach_top_nth_typename(void (*f)(jl_typename_t*, void*), jl_value_t *a JL_PROPAGATES_ROOT, int n, void *env) { if (jl_is_datatype(a)) { if (n == 0) { - jl_methtable_t *mt = ((jl_datatype_t*)a)->name->mt; - if (mt != NULL) - return mt; + jl_datatype_t *dt = ((jl_datatype_t*)a); + jl_typename_t *tn = NULL; + while (1) { + if (dt != jl_any_type && dt != jl_function_type) + tn = dt->name; + if (dt->super == dt) + break; + dt = dt->super; + } + if (tn) + f(tn, env); } else if (jl_is_tuple_type(a)) { if (jl_nparams(a) >= n) - return nth_methtable(jl_tparam(a, n - 1), 0); + foreach_top_nth_typename(f, jl_tparam(a, n - 1), 0, env); } } else if (jl_is_typevar(a)) { - return nth_methtable(((jl_tvar_t*)a)->ub, n); + foreach_top_nth_typename(f, ((jl_tvar_t*)a)->ub, n, env); } else if (jl_is_unionall(a)) { - return nth_methtable(((jl_unionall_t*)a)->body, n); + foreach_top_nth_typename(f, ((jl_unionall_t*)a)->body, n, env); } else if (jl_is_uniontype(a)) { jl_uniontype_t *u = (jl_uniontype_t*)a; - jl_methtable_t *m1 = nth_methtable(u->a, n); - if ((jl_value_t*)m1 != jl_nothing) { - jl_methtable_t *m2 = nth_methtable(u->b, n); - if (m1 == m2) - return m1; - } + foreach_top_nth_typename(f, u->a, n, env); + foreach_top_nth_typename(f, u->b, n, env); } - return (jl_methtable_t*)jl_nothing; } // get the MethodTable for dispatch, or `nothing` if cannot be determined JL_DLLEXPORT jl_methtable_t *jl_method_table_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - return nth_methtable(argtypes, 1); + return jl_method_table; +} + +// get a MethodCache for dispatch +JL_DLLEXPORT jl_methcache_t *jl_method_cache_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_method_table->cache; +} + +void jl_foreach_top_typename_for(void (*f)(jl_typename_t*, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, void *env) +{ + foreach_top_nth_typename(f, argtypes, 1, env); } -jl_methtable_t *jl_kwmethod_table_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +jl_methcache_t *jl_kwmethod_cache_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - jl_methtable_t *kwmt = nth_methtable(argtypes, 3); - if ((jl_value_t*)kwmt == jl_nothing) - return NULL; - return kwmt; + return jl_method_table->cache; } JL_DLLEXPORT jl_methtable_t *jl_method_get_table(jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - return method->external_mt ? (jl_methtable_t*)method->external_mt : jl_method_table_for(method->sig); + return method->external_mt ? (jl_methtable_t*)method->external_mt : jl_method_table; +} + +// get an arbitrary MethodCache for dispatch optimizations of method +JL_DLLEXPORT jl_methcache_t *jl_method_get_cache(jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_method_get_table(method)->cache; } JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, @@ -1218,25 +1235,29 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, size_t nargs = jl_svec_len(atypes); assert(nargs > 0); int isva = jl_is_vararg(jl_svecref(atypes, nargs - 1)); - if (!jl_is_type(jl_svecref(atypes, 0)) || (isva && nargs == 1)) + jl_value_t *ft = jl_svecref(atypes, 0); + if (!jl_is_type(ft) || (isva && nargs == 1)) jl_error("function type in method definition is not a type"); jl_sym_t *name; jl_method_t *m = NULL; jl_value_t *argtype = NULL; - JL_GC_PUSH3(&f, &m, &argtype); + JL_GC_PUSH4(&ft, &f, &m, &argtype); size_t i, na = jl_svec_len(atypes); argtype = jl_apply_tuple_type(atypes, 1); if (!jl_is_datatype(argtype)) jl_error("invalid type in method definition (Union{})"); - jl_methtable_t *external_mt = mt; if (!mt) - mt = jl_method_table_for(argtype); - if ((jl_value_t*)mt == jl_nothing) - jl_error("Method dispatch is unimplemented currently for this method signature"); - if (mt->frozen) - jl_error("cannot add methods to a builtin function"); + mt = jl_method_table; + jl_methtable_t *external_mt = mt == jl_method_table ? NULL : mt; + + //if (!external_mt) { + // jl_value_t **ttypes = { jl_builtin_type, jl_tparam0(jl_anytuple_type) }; + // jl_value_t *invalidt = jl_apply_tuple_type_v(ttypes, 2); // Tuple{Union{Builtin,OpaqueClosure}, Vararg} + // if (!jl_has_empty_intersection(argtype, invalidt)) + // jl_error("cannot add methods to a builtin function"); + //} assert(jl_is_linenode(functionloc)); jl_sym_t *file = (jl_sym_t*)jl_linenode_file(functionloc); @@ -1245,21 +1266,13 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, int32_t line = jl_linenode_line(functionloc); // TODO: derive our debug name from the syntax instead of the type - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(argtype) : mt; // if we have a kwcall, try to derive the name from the callee argument method table - name = (kwmt ? kwmt : mt)->name; - if (kwmt == jl_type_type_mt || kwmt == jl_nonfunction_mt || external_mt) { - // our value for `name` is bad, try to guess what the syntax might have had, - // like `jl_static_show_func_sig` might have come up with - jl_datatype_t *dt = jl_nth_argument_datatype(argtype, mt == jl_kwcall_mt ? 3 : 1); - if (dt != NULL) { - name = dt->name->name; - if (jl_is_type_type((jl_value_t*)dt)) { - dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); - if ((jl_value_t*)dt != jl_nothing) { - name = dt->name->name; - } - } + jl_datatype_t *dtname = (jl_datatype_t*)jl_argument_datatype(jl_kwcall_type && ft == (jl_value_t*)jl_kwcall_type && nargs >= 3 ? jl_svecref(atypes, 2) : ft); + name = (jl_value_t*)dtname != jl_nothing ? dtname->name->singletonname : jl_any_type->name->singletonname; + if (jl_is_type_type((jl_value_t*)dtname)) { + dtname = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dtname)); + if ((jl_value_t*)dtname != jl_nothing) { + name = dtname->name->singletonname; } } @@ -1320,6 +1333,9 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, jl_symbol_name(file), line); } + ft = jl_rewrap_unionall(ft, argtype); + if (!external_mt && !jl_has_empty_intersection(ft, (jl_value_t*)jl_builtin_type)) // disallow adding methods to Any, Function, Builtin, and subtypes, or Unions of those + jl_error("cannot add methods to a builtin function"); m = jl_new_method_uninit(module); m->external_mt = (jl_value_t*)external_mt; diff --git a/src/module.c b/src/module.c index 272748edbadb2..9c1a93d08373a 100644 --- a/src/module.c +++ b/src/module.c @@ -1157,14 +1157,14 @@ static void jl_binding_dep_message(jl_binding_t *b) jl_printf(JL_STDERR, " instead."); } else { - jl_methtable_t *mt = jl_gf_mtable(v); - if (mt != NULL) { + jl_typename_t *tn = ((jl_datatype_t*)jl_typeof(v))->name; + if (tn != NULL) { jl_printf(JL_STDERR, ", use "); - if (mt->module != jl_core_module) { - jl_static_show(JL_STDERR, (jl_value_t*)mt->module); + if (tn->module != jl_core_module) { + jl_static_show(JL_STDERR, (jl_value_t*)tn->module); jl_printf(JL_STDERR, "."); } - jl_printf(JL_STDERR, "%s", jl_symbol_name(mt->name)); + jl_printf(JL_STDERR, "%s", jl_symbol_name(tn->singletonname)); jl_printf(JL_STDERR, " instead."); } } diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 295f91ad31e67..86bb723443925 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -159,12 +159,12 @@ static int compile_all_collect_(jl_methtable_t *mt, void *env) return 1; } -static void jl_compile_all_defs(jl_array_t *mis, int all) +static void jl_compile_all_defs(jl_array_t *mis, int all, jl_array_t *mod_array) { jl_array_t *allmeths = jl_alloc_vec_any(0); JL_GC_PUSH1(&allmeths); - jl_foreach_reachable_mtable(compile_all_collect_, allmeths); + jl_foreach_reachable_mtable(compile_all_collect_, mod_array, allmeths); size_t world = jl_atomic_load_acquire(&jl_world_counter); size_t i, l = jl_array_nrows(allmeths); @@ -224,38 +224,53 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur return 1; } -static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) +struct precompile_enq_all_specializations_env { + jl_array_t *worklist; + jl_array_t *m; +}; + +static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *env) { jl_method_t *m = def->func.method; - if (m->external_mt) - return 1; + assert(!m->external_mt); + struct precompile_enq_all_specializations_env *closure = (struct precompile_enq_all_specializations_env*)env; + if (closure->worklist) { + size_t i, l = jl_array_nrows(closure->worklist); + for (i = 0; i < l; i++) { + if (m->module == (jl_module_t*)jl_array_ptr_ref(closure->worklist, i)) + break; + } + if (i == l) + return 1; + } if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + jl_array_ptr_1d_push(closure->m, (jl_value_t*)mi); } else { jl_value_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); if (!jl_is_svec(specializations)) { - precompile_enq_specialization_((jl_method_instance_t*)specializations, closure); + precompile_enq_specialization_((jl_method_instance_t*)specializations, closure->m); } else { size_t i, l = jl_svec_len(specializations); for (i = 0; i < l; i++) { jl_value_t *mi = jl_svecref(specializations, i); if (mi != jl_nothing) - precompile_enq_specialization_((jl_method_instance_t*)mi, closure); + precompile_enq_specialization_((jl_method_instance_t*)mi, closure->m); } } } if (m->ccallable) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); + jl_array_ptr_1d_push(closure->m, (jl_value_t*)m->ccallable); return 1; } -static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) +static int precompile_enq_all_specializations_(jl_array_t *worklist, jl_array_t *env) { - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); + struct precompile_enq_all_specializations_env closure = {worklist, env}; + return jl_typemap_visitor(jl_atomic_load_relaxed(&jl_method_table->defs), precompile_enq_all_specializations__, &closure); } static void *jl_precompile_(jl_array_t *m, int external_linkage) @@ -284,13 +299,13 @@ static void *jl_precompile_(jl_array_t *m, int external_linkage) return native_code; } -static void *jl_precompile(int all) +static void *jl_precompile(int all, jl_array_t *mod_array) { // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); JL_GC_PUSH1(&m); - jl_compile_all_defs(m, all); - jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); + jl_compile_all_defs(m, all, mod_array); + precompile_enq_all_specializations_(NULL, m); void *native_code = jl_precompile_(m, 0); JL_GC_POP(); return native_code; @@ -311,13 +326,8 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met jl_array_t *m = jl_alloc_vec_any(0); JL_GC_PUSH1(&m); if (!suppress_precompile) { - size_t i, n = jl_array_nrows(worklist); - for (i = 0; i < n; i++) { - jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); - assert(jl_is_module(mod)); - foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); - } - n = jl_array_nrows(extext_methods); + precompile_enq_all_specializations_(worklist, m); + size_t i, n = jl_array_nrows(extext_methods); for (i = 0; i < n; i++) { jl_method_t *method = (jl_method_t*)jl_array_ptr_ref(extext_methods, i); assert(jl_is_method(method)); @@ -350,21 +360,15 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met static int enq_ccallable_entrypoints_(jl_typemap_entry_t *def, void *closure) { jl_method_t *m = def->func.method; - if (m->external_mt) - return 1; + assert(!m->external_mt); if (m->ccallable) jl_add_entrypoint((jl_tupletype_t*)jl_svecref(m->ccallable, 1)); return 1; } -static int enq_ccallable_entrypoints(jl_methtable_t *mt, void *env) -{ - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), enq_ccallable_entrypoints_, env); -} - JL_DLLEXPORT void jl_add_ccallable_entrypoints(void) { - jl_foreach_reachable_mtable(enq_ccallable_entrypoints, NULL); + jl_typemap_visitor(jl_atomic_load_relaxed(&jl_method_table->defs), enq_ccallable_entrypoints_, NULL); } static void *jl_precompile_trimmed(size_t world) @@ -402,35 +406,39 @@ static void *jl_precompile_trimmed(size_t world) return native_code; } -static void jl_rebuild_methtables(arraylist_t* MIs, htable_t* mtables) +static void jl_rebuild_methtables(arraylist_t *MIs, htable_t *mtables) JL_GC_DISABLED { - size_t i; - for (i = 0; i < MIs->len; i++) { + // Rebuild MethodTable to contain only those methods for which we compiled code. + // This can have significant soundness problems if there previously existed + // any ambiguous methods, but it would probably be pretty hard to do this + // fully correctly (with the necessary inserted guard entries). + htable_t ms; + htable_new(&ms, 0); + for (size_t i = 0; i < MIs->len; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)MIs->items[i]; jl_method_t *m = mi->def.method; + // Check if the method is already in the new table, if not then insert it there + void **inserted = ptrhash_bp(&ms, m); + if (*inserted != HT_NOTFOUND) + continue; + *inserted = (void*)m; jl_methtable_t *old_mt = jl_method_get_table(m); if ((jl_value_t *)old_mt == jl_nothing) continue; - jl_sym_t *name = old_mt->name; if (!ptrhash_has(mtables, old_mt)) - ptrhash_put(mtables, old_mt, jl_new_method_table(name, m->module)); + ptrhash_put(mtables, old_mt, jl_new_method_table(old_mt->name, old_mt->module)); jl_methtable_t *mt = (jl_methtable_t*)ptrhash_get(mtables, old_mt); - size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *lookup = jl_methtable_lookup(mt, m->sig, world); - // Check if the method is already in the new table, if not then insert it there - if (lookup == jl_nothing || (jl_method_t*)lookup != m) { - //TODO: should this be a function like unsafe_insert_method? - size_t min_world = jl_atomic_load_relaxed(&m->primary_world); - size_t max_world = ~(size_t)0; - assert(min_world == jl_atomic_load_relaxed(&m->primary_world)); - int dispatch_status = jl_atomic_load_relaxed(&m->dispatch_status); - jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); - jl_atomic_store_relaxed(&m->dispatch_status, 0); - jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); - jl_atomic_store_relaxed(&m->primary_world, min_world); - jl_atomic_store_relaxed(&m->dispatch_status, dispatch_status); - jl_atomic_store_relaxed(&newentry->min_world, min_world); - jl_atomic_store_relaxed(&newentry->max_world, max_world); // short-circuit jl_method_table_insert - } + //TODO: should this be a function like unsafe_insert_method, since all that is wanted is the jl_typemap_insert on a copy of the existing entry + size_t min_world = jl_atomic_load_relaxed(&m->primary_world); + size_t max_world = ~(size_t)0; + int dispatch_status = jl_atomic_load_relaxed(&m->dispatch_status); + jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); + jl_atomic_store_relaxed(&m->dispatch_status, 0); + jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); + jl_atomic_store_relaxed(&m->primary_world, min_world); + jl_atomic_store_relaxed(&m->dispatch_status, dispatch_status); + jl_atomic_store_relaxed(&newentry->min_world, min_world); + jl_atomic_store_relaxed(&newentry->max_world, max_world); // short-circuit jl_method_table_insert } + htable_free(&ms); } diff --git a/src/rtutils.c b/src/rtutils.c index 4baf0ee5e6e9c..2f3bd4e8a0074 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -671,7 +671,7 @@ JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROO static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT { - jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; + jl_sym_t *globname = dv->name->singletonname; if (globname && dv->name->module) { jl_binding_t *b = jl_get_module_binding(dv->name->module, globname, 0); jl_value_t *bv = jl_get_latest_binding_value_if_resolved_and_const_debug_only(b); @@ -683,7 +683,7 @@ static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT static int is_globfunction(jl_value_t *v, jl_datatype_t *dv, jl_sym_t **globname_out) JL_NOTSAFEPOINT { - jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; + jl_sym_t *globname = dv->name->singletonname; *globname_out = globname; if (globname && !strchr(jl_symbol_name(globname), '#') && !strchr(jl_symbol_name(globname), '@')) { return 1; @@ -814,6 +814,9 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (v == (jl_value_t*)jl_methtable_type) { n += jl_printf(out, "Core.MethodTable"); } + else if (v == (jl_value_t*)jl_methcache_type) { + n += jl_printf(out, "Core.MethodCache"); + } else if (v == (jl_value_t*)jl_any_type) { n += jl_printf(out, "Any"); } @@ -997,6 +1000,9 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (v == jl_nothing || (jl_nothing && (jl_value_t*)vt == jl_typeof(jl_nothing))) { n += jl_printf(out, "nothing"); } + else if (v == (jl_value_t*)jl_method_table) { + n += jl_printf(out, "Core.GlobalMethods"); + } else if (vt == jl_string_type) { n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1); } @@ -1426,10 +1432,8 @@ size_t jl_static_show_func_sig_(JL_STREAM *s, jl_value_t *type, jl_static_show_c return n; } if ((jl_nparams(ftype) == 0 || ftype == ((jl_datatype_t*)ftype)->name->wrapper) && - ((jl_datatype_t*)ftype)->name->mt && - ((jl_datatype_t*)ftype)->name->mt != jl_type_type_mt && - ((jl_datatype_t*)ftype)->name->mt != jl_nonfunction_mt) { - n += jl_static_show_symbol(s, ((jl_datatype_t*)ftype)->name->mt->name); + !jl_is_type_type(ftype) && !jl_is_type_type((jl_value_t*)((jl_datatype_t*)ftype)->super)) { // aka !iskind + n += jl_static_show_symbol(s, ((jl_datatype_t*)ftype)->name->singletonname); } else { n += jl_printf(s, "(::"); diff --git a/src/staticdata.c b/src/staticdata.c index 5465cb1da0218..eb503fe0ffa78 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -116,7 +116,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 152 +#define NUM_TAGS 151 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -182,6 +182,7 @@ static void get_tags(jl_value_t **tags[NUM_TAGS]) INSERT_TAG(jl_array_any_type); INSERT_TAG(jl_intrinsic_type); INSERT_TAG(jl_methtable_type); + INSERT_TAG(jl_methcache_type); INSERT_TAG(jl_typemap_level_type); INSERT_TAG(jl_typemap_entry_type); INSERT_TAG(jl_voidpointer_type); @@ -229,6 +230,7 @@ static void get_tags(jl_value_t **tags[NUM_TAGS]) INSERT_TAG(jl_addrspacecore_type); INSERT_TAG(jl_debuginfo_type); INSERT_TAG(jl_abioverride_type); + INSERT_TAG(jl_kwcall_type); // special typenames INSERT_TAG(jl_tuple_typename); @@ -277,12 +279,9 @@ static void get_tags(jl_value_t **tags[NUM_TAGS]) INSERT_TAG(jl_main_module); INSERT_TAG(jl_top_module); INSERT_TAG(jl_typeinf_func); - INSERT_TAG(jl_type_type_mt); - INSERT_TAG(jl_nonfunction_mt); - INSERT_TAG(jl_kwcall_mt); - INSERT_TAG(jl_kwcall_func); INSERT_TAG(jl_opaque_closure_method); INSERT_TAG(jl_nulldebuginfo); + INSERT_TAG(jl_method_table); // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG assert(i == NUM_TAGS - 1); @@ -785,12 +784,6 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } - if (s->incremental && jl_is_mtable(v)) { - jl_methtable_t *mt = (jl_methtable_t *)v; - // Any back-edges will be re-validated and added by staticdata.jl, so - // drop them from the image here - record_field_change((jl_value_t**)&mt->backedges, NULL); - } if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; if (s->incremental) { @@ -836,24 +829,6 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } } - if (jl_is_mtable(v)) { - jl_methtable_t *mt = (jl_methtable_t*)v; - if (jl_options.trim || jl_options.strip_ir) { - record_field_change((jl_value_t**)&mt->backedges, NULL); - } - else { - // don't recurse into all backedges memory (yet) - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); - if (backedges) { - jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); - for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { - jl_value_t *t = jl_array_ptr_ref(backedges, i); - assert(!jl_is_code_instance(t)); - jl_queue_for_serialization(s, t); - } - } - } - } if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; if (s->incremental && needs_uniquing(v, s->query_cache)) { @@ -892,6 +867,23 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->module)); assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } + // Any back-edges will be re-validated and added by staticdata.jl, so + // drop them from the image here + if (s->incremental || jl_options.trim || jl_options.strip_ir) { + record_field_change((jl_value_t**)&tn->backedges, NULL); + } + else { + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&tn->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { + jl_value_t *t = jl_array_ptr_ref(backedges, i); + assert(!jl_is_code_instance(t)); + jl_queue_for_serialization(s, t); + } + } + } } if (jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; @@ -1015,7 +1007,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } } - else if (jl_typetagis(v, jl_module_tag << 4)) { + else if (jl_is_module(v)) { jl_queue_module_for_serialization(s, (jl_module_t*)v); } else if (layout->nfields > 0) { @@ -1025,15 +1017,21 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (jl_is_svec(jl_atomic_load_relaxed(&m->specializations))) jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->specializations), 0, 1); } - else if (jl_typetagis(v, jl_typename_type)) { - jl_typename_t *tn = (jl_typename_t*)v; - if (tn->mt != NULL && !tn->mt->frozen) { - jl_methtable_t * new_methtable = (jl_methtable_t *)ptrhash_get(&new_methtables, tn->mt); - if (new_methtable != HT_NOTFOUND) - record_field_change((jl_value_t **)&tn->mt, (jl_value_t*)new_methtable); - else - record_field_change((jl_value_t **)&tn->mt, NULL); + else if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + jl_methtable_t *newmt = (jl_methtable_t*)ptrhash_get(&new_methtables, mt); + if (newmt != HT_NOTFOUND) + record_field_change((jl_value_t **)&mt->defs, (jl_value_t*)jl_atomic_load_relaxed(&newmt->defs)); + else + record_field_change((jl_value_t **)&mt->defs, jl_nothing); + } + else if (jl_is_mcache(v)) { + jl_methcache_t *mc = (jl_methcache_t*)v; + jl_value_t *cache = jl_atomic_load_relaxed(&mc->cache); + if (!jl_typetagis(cache, jl_typemap_entry_type) || ((jl_typemap_entry_t*)cache)->sig != jl_tuple_type) { // aka Builtins (maybe sometimes OpaqueClosure too) + record_field_change((jl_value_t **)&mc->cache, jl_nothing); } + record_field_change((jl_value_t **)&mc->leafcache, jl_an_empty_memory_any); } // TODO: prune any partitions and partition data that has been deleted in the current world //else if (jl_is_binding(v)) { @@ -1098,7 +1096,20 @@ static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, i if (!jl_needs_serialization(s, v)) return; - jl_value_t *t = jl_typeof(v); + jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); + // check early from errors, so we have a little bit of contextual state for debugging them + if (t == jl_task_type) { + jl_error("Task cannot be serialized"); + } + if (s->incremental && needs_uniquing(v, s->query_cache) && t == jl_binding_type) { + jl_binding_t *b = (jl_binding_t*)v; + if (b->globalref == NULL) + jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity + } + if (jl_is_foreign_type(t) == 1) { + jl_error("Cannot serialize instances of foreign datatypes"); + } + // Items that require postorder traversal must visit their children prior to insertion into // the worklist/serialization_order (and also before their first use) if (s->incremental && !immediate) { @@ -1506,8 +1517,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (needs_uniquing(v, s->query_cache)) { if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; - if (b->globalref == NULL) - jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity write_pointerfield(s, (jl_value_t*)b->globalref->mod); write_pointerfield(s, (jl_value_t*)b->globalref->name); continue; @@ -1672,7 +1681,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_write_module(s, item, (jl_module_t*)v); } else if (jl_typetagis(v, jl_task_tag << 4)) { - jl_error("Task cannot be serialized"); + abort(); // unreachable } else if (jl_is_svec(v)) { assert(f == s->s); @@ -1688,7 +1697,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED write_uint8(f, '\0'); // null-terminated strings for easier C-compatibility } else if (jl_is_foreign_type(t) == 1) { - jl_error("Cannot serialize instances of foreign datatypes"); + abort(); // unreachable } else if (jl_datatype_nfields(t) == 0) { // The object has no fields, so we just snapshot its byte representation @@ -2562,7 +2571,7 @@ static void jl_prune_mi_backedges(jl_array_t *backedges) jl_array_del_end(backedges, n - ins); } -static void jl_prune_mt_backedges(jl_array_t *backedges) +static void jl_prune_tn_backedges(jl_array_t *backedges) { if (backedges == NULL) return; @@ -2738,7 +2747,7 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) JL_GC_PUSH1(&slotnames); int tostrip = jl_array_len(slotnames); // for keyword methods, strip only nargs to keep the keyword names at the end for reflection - if (jl_tparam0(jl_unwrap_unionall(m->sig)) == jl_typeof(jl_kwcall_func)) + if (jl_tparam0(jl_unwrap_unionall(m->sig)) == (jl_value_t*)jl_kwcall_type) tostrip = m->nargs; strip_slotnames(slotnames, tostrip); m->slot_syms = jl_compress_argnames(slotnames); @@ -2770,14 +2779,14 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) return 1; } -static int strip_all_codeinfos_(jl_methtable_t *mt, void *_env) +static int strip_all_codeinfos_mt(jl_methtable_t *mt, void *_env) { return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), strip_all_codeinfos__, NULL); } -static void jl_strip_all_codeinfos(void) +static void jl_strip_all_codeinfos(jl_array_t *mod_array) { - jl_foreach_reachable_mtable(strip_all_codeinfos_, NULL); + jl_foreach_reachable_mtable(strip_all_codeinfos_mt, mod_array, NULL); } static int strip_module(jl_module_t *m, jl_sym_t *docmeta_sym) @@ -2982,15 +2991,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new *extext_methods = jl_alloc_vec_any(0); internal_methods = jl_alloc_vec_any(0); JL_GC_PUSH1(&internal_methods); - jl_collect_methtable_from_mod(jl_type_type_mt, *extext_methods); - jl_collect_methtable_from_mod(jl_nonfunction_mt, *extext_methods); - size_t i, len = jl_array_len(mod_array); - for (i = 0; i < len; i++) { - jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); - assert(jl_is_module(m)); - if (m->parent == m) // some toplevel modules (really just Base) aren't actually - jl_collect_extext_methods_from_mod(*extext_methods, m); - } + jl_collect_extext_methods(*extext_methods, mod_array); if (edges) { // Extract `edges` now (from info prepared by jl_collect_methcache_from_mod) @@ -3013,7 +3014,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, htable_new(&bits_replace, 0); // strip metadata and IR when requested if (jl_options.strip_metadata || jl_options.strip_ir) { - jl_strip_all_codeinfos(); + jl_strip_all_codeinfos(mod_array); jl_strip_all_docmeta(mod_array); } // collect needed methods and replace method tables that are in the tags array @@ -3070,31 +3071,11 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, size_t num_mis; jl_get_llvm_mis(native_functions, &num_mis, NULL); arraylist_grow(&MIs, num_mis); - jl_get_llvm_mis(native_functions, &num_mis, (jl_method_instance_t *)MIs.items); + jl_get_llvm_mis(native_functions, &num_mis, (jl_method_instance_t*)MIs.items); } } if (jl_options.trim) { jl_rebuild_methtables(&MIs, &new_methtables); - jl_methtable_t *mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_type_type_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_type_type_mt = mt; - else - jl_type_type_mt = jl_new_method_table(jl_type_type_mt->name, jl_type_type_mt->module); - - mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_kwcall_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_kwcall_mt = mt; - else - jl_kwcall_mt = jl_new_method_table(jl_kwcall_mt->name, jl_kwcall_mt->module); - - mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_nonfunction_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_nonfunction_mt = mt; - else - jl_nonfunction_mt = jl_new_method_table(jl_nonfunction_mt->name, jl_nonfunction_mt->module); } nsym_tag = 0; @@ -3259,17 +3240,14 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_prune_type_cache_hash(jl_atomic_load_relaxed(&tn->cache))); jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&tn->backedges, 1); + jl_prune_tn_backedges((jl_array_t*)backedges); } else if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); jl_prune_mi_backedges((jl_array_t*)backedges); } - else if (jl_is_mtable(v)) { - jl_methtable_t *mt = (jl_methtable_t*)v; - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); - jl_prune_mt_backedges((jl_array_t*)backedges); - } else if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); @@ -3522,7 +3500,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli if (jl_options.trim) *_native_data = jl_precompile_trimmed(precompilation_world); else - *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); + *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, mod_array); } // Make sure we don't run any Julia code concurrently after this point diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 65f7dc59d9397..d699e0c262d26 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -332,9 +332,9 @@ static int jl_collect_methtable_from_mod(jl_methtable_t *mt, void *env) // Collect methods of external functions defined by modules in the worklist // "extext" = "extending external" // Also collect relevant backedges -static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) +static void jl_collect_extext_methods(jl_array_t *s, jl_array_t *mod_array) { - foreach_mtable_in_module(m, jl_collect_methtable_from_mod, s); + jl_foreach_reachable_mtable(jl_collect_methtable_from_mod, mod_array, s); } static void jl_record_edges(jl_method_instance_t *caller, jl_array_t *edges) @@ -727,9 +727,7 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size } for (i = 0; i < l; i++) { jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_array_ptr_ref(external, i); - jl_methtable_t *mt = jl_method_get_table(entry->func.method); - assert((jl_value_t*)mt != jl_nothing); - jl_method_table_activate(mt, entry); + jl_method_table_activate(entry); } } } diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 3362c9439d385..9437fedf649ec 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -467,6 +467,20 @@ function serialize(s::AbstractSerializer, meth::Method) nothing end +function serialize(s::AbstractSerializer, mt::Core.MethodTable) + serialize_type(s, typeof(mt)) + serialize(s, mt.cache) + nothing +end + +function serialize(s::AbstractSerializer, mc::Core.MethodCache) + serialize_type(s, typeof(mc)) + serialize(s, mc.name) + serialize(s, mc.module) + nothing +end + + function serialize(s::AbstractSerializer, linfo::Core.MethodInstance) serialize_cycle(s, linfo) && return writetag(s.io, METHODINSTANCE_TAG) @@ -536,11 +550,12 @@ function serialize_typename(s::AbstractSerializer, t::Core.TypeName) serialize(s, t.flags & 0x2 == 0x2) # .mutable serialize(s, Int32(length(primary.types) - t.n_uninitialized)) serialize(s, t.max_methods) - if isdefined(t, :mt) && t.mt !== Symbol.name.mt - serialize(s, t.mt.name) - serialize(s, collect(Base.MethodList(t.mt))) - serialize(s, t.mt.max_args) - kws = collect(methods(Core.kwcall, (Any, t.wrapper, Vararg))) + ms = Base.matches_to_methods(Base._methods_by_ftype(Tuple{t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms + if t.singletonname !== t.name || !isempty(ms) + serialize(s, t.singletonname) + serialize(s, ms) + serialize(s, t.max_args) + kws = Base.matches_to_methods(Base._methods_by_ftype(Tuple{typeof(Core.kwcall), Any, t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms if isempty(kws) writetag(s.io, UNDEFREF_TAG) else @@ -555,21 +570,17 @@ end # decide whether to send all data for a type (instead of just its name) function should_send_whole_type(s, t::DataType) tn = t.name - if isdefined(tn, :mt) - # TODO improve somehow - # send whole type for anonymous functions in Main - name = tn.mt.name - mod = tn.module - isanonfunction = mod === Main && # only Main - t.super === Function && # only Functions - unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type - (!isdefined(mod, name) || t != typeof(getglobal(mod, name))) # XXX: 95% accurate test for this being an inner function - # TODO: more accurate test? (tn.name !== "#" name) - #TODO: iskw = startswith(tn.name, "#kw#") && ??? - #TODO: iskw && return send-as-kwftype - return mod === __deserialized_types__ || isanonfunction - end - return false + # TODO improve somehow? + # send whole type for anonymous functions in Main + name = tn.singletonname + mod = tn.module + mod === __deserialized_types__ && return true + isanonfunction = mod === Main && # only Main + t.super === Function && # only Functions + unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type + (!isdefined(mod, name) || t != typeof(getglobal(mod, name))) # XXX: 95% accurate test for this being an inner function + # TODO: more accurate test? (tn.name !== "#" name) + return isanonfunction end function serialize_type_data(s, @nospecialize(t::DataType)) @@ -1112,8 +1123,8 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) meth.recursion_relation = recursion_relation end if !is_for_opaque_closure - mt = ccall(:jl_method_table_for, Any, (Any,), sig) - if mt !== nothing && nothing === ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), mt, sig, Base.get_world_counter()) + mt = Core.GlobalMethods + if nothing === ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, Base.get_world_counter()) # XXX: quite sketchy? ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, meth, C_NULL) end end @@ -1122,6 +1133,19 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) return meth end +function deserialize(s::AbstractSerializer, ::Type{Core.MethodTable}) + mc = deserialize(s)::Core.MethodCache + mc === Core.GlobalMethods.cache && return Core.GlobalMethods + return getglobal(mc.mod, mc.name)::Core.MethodTable +end + +function deserialize(s::AbstractSerializer, ::Type{Core.MethodCache}) + name = deserialize(s)::Symbol + mod = deserialize(s)::Module + f = Base.unwrap_unionall(getglobal(mod, name)) + return (f::Core.MethodTable).cache +end + function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance}) linfo = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, (Ptr{Cvoid},), C_NULL) deserialize_cycle(s, linfo) @@ -1471,20 +1495,10 @@ function deserialize_typename(s::AbstractSerializer, number) if tag != UNDEFREF_TAG mtname = handle_deserialize(s, tag) defs = deserialize(s) - maxa = deserialize(s)::Int + maxa = deserialize(s)::Union{Int,Int32} if makenew - mt = ccall(:jl_new_method_table, Any, (Any, Any), name, tn.module) - if !isempty(parameters) - mt.offs = 0 - end - mt.name = mtname - setfield!(mt, :max_args, maxa, :monotonic) - ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), tn, Base.fieldindex(Core.TypeName, :mt)-1, mt) - for def in defs - if isdefined(def, :sig) - ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, def, C_NULL) - end - end + tn.singletonname = mtname + setfield!(tn, :max_args, Int32(maxa), :monotonic) end tag = Int32(read(s.io, UInt8)::UInt8) if tag != UNDEFREF_TAG @@ -1494,9 +1508,6 @@ function deserialize_typename(s::AbstractSerializer, number) @eval Core.kwcall(kwargs::NamedTuple, f::$ty, args...) = $kws(kwargs, f, args...) end end - elseif makenew - mt = Symbol.name.mt - ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), tn, Base.fieldindex(Core.TypeName, :mt)-1, mt) end return tn end diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 738da88197c1c..8e15d10512029 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -2215,30 +2215,7 @@ function detect_ambiguities(mods::Module...; end end end - work = Base.loaded_modules_array() - filter!(mod -> mod === parentmodule(mod), work) # some items in loaded_modules_array are not top modules (really just Base) - while !isempty(work) - mod = pop!(work) - for n in names(mod, all = true) - Base.isdeprecated(mod, n) && continue - if !isdefined(mod, n) - if is_in_mods(mod, recursive, mods) - if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds - println("Skipping ", mod, '.', n) # typically stale exports - end - end - continue - end - f = Base.unwrap_unionall(getfield(mod, n)) - if isa(f, Module) && f !== mod && parentmodule(f) === mod && nameof(f) === n - push!(work, f) - elseif isa(f, DataType) && isdefined(f.name, :mt) && parentmodule(f) === mod && nameof(f) === n && f.name.mt !== Symbol.name.mt && f.name.mt !== DataType.name.mt - examine(f.name.mt) - end - end - end - examine(Symbol.name.mt) - examine(DataType.name.mt) + examine(Core.GlobalMethods) return collect(ambs) end @@ -2286,30 +2263,7 @@ function detect_unbound_args(mods...; push!(ambs, m) end end - work = Base.loaded_modules_array() - filter!(mod -> mod === parentmodule(mod), work) # some items in loaded_modules_array are not top modules (really just Base) - while !isempty(work) - mod = pop!(work) - for n in names(mod, all = true) - Base.isdeprecated(mod, n) && continue - if !isdefined(mod, n) - if is_in_mods(mod, recursive, mods) - if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds - println("Skipping ", mod, '.', n) # typically stale exports - end - end - continue - end - f = Base.unwrap_unionall(getfield(mod, n)) - if isa(f, Module) && f !== mod && parentmodule(f) === mod && nameof(f) === n - push!(work, f) - elseif isa(f, DataType) && isdefined(f.name, :mt) && parentmodule(f) === mod && nameof(f) === n && f.name.mt !== Symbol.name.mt && f.name.mt !== DataType.name.mt - examine(f.name.mt) - end - end - end - examine(Symbol.name.mt) - examine(DataType.name.mt) + examine(Core.GlobalMethods) return collect(ambs) end diff --git a/test/clangsa/MissingRoots.c b/test/clangsa/MissingRoots.c index 0a0d5369eba44..84341f9410e1e 100644 --- a/test/clangsa/MissingRoots.c +++ b/test/clangsa/MissingRoots.c @@ -277,20 +277,6 @@ void nonconst_loads2() static inline void look_at_value2(jl_value_t *v) { look_at_value(v); } -void mtable(jl_value_t *f) { - look_at_value2((jl_value_t*)jl_gf_mtable(f)); - jl_value_t *val = NULL; - JL_GC_PUSH1(&val); - val = (jl_value_t*)jl_gf_mtable(f); - JL_GC_POP(); -} - -void mtable2(jl_value_t **v) { - jl_value_t *val = NULL; - JL_GC_PUSH1(&val); - val = (jl_value_t*)jl_gf_mtable(v[2]); - JL_GC_POP(); -} void tparam0(jl_value_t *atype) { look_at_value(jl_tparam0(atype)); diff --git a/test/core.jl b/test/core.jl index c65a2a821e275..0c4133d949346 100644 --- a/test/core.jl +++ b/test/core.jl @@ -17,10 +17,11 @@ for (T, c) in ( (Core.CodeInstance, [:def, :owner, :rettype, :exctype, :rettype_const, :analysis_results, :time_infer_total, :time_infer_cache_saved, :time_infer_self]), (Core.Method, [#=:name, :module, :file, :line, :primary_world, :sig, :slot_syms, :external_mt, :nargs, :called, :nospecialize, :nkw, :isva, :is_for_opaque_closure, :constprop=#]), (Core.MethodInstance, [#=:def, :specTypes, :sparam_vals=#]), - (Core.MethodTable, [:module]), + (Core.MethodTable, [:cache, :module, :name]), + (Core.MethodCache, []), (Core.TypeMapEntry, [:sig, :simplesig, :guardsigs, :func, :isleafsig, :issimplesig, :va]), (Core.TypeMapLevel, []), - (Core.TypeName, [:name, :module, :names, :wrapper, :mt, :hash, :n_uninitialized, :flags]), + (Core.TypeName, [:name, :module, :names, :wrapper, :hash, :n_uninitialized, :flags]), (DataType, [:name, :super, :parameters, :instance, :hash]), (TypeVar, [:name, :ub, :lb]), (Core.Memory, [:length, :ptr]), @@ -37,10 +38,11 @@ for (T, c) in ( (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), (Core.Method, [:primary_world, :dispatch_status]), (Core.MethodInstance, [:cache, :flags]), - (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), + (Core.MethodTable, [:defs]), + (Core.MethodCache, [:leafcache, :cache, :var""]), (Core.TypeMapEntry, [:next, :min_world, :max_world]), (Core.TypeMapLevel, [:arg1, :targ, :name1, :tname, :list, :any]), - (Core.TypeName, [:cache, :linearcache, :cache_entry_count]), + (Core.TypeName, [:cache, :linearcache, :Typeofwrapper, :max_args, :cache_entry_count]), (DataType, [:types, :layout]), (Core.Memory, []), (Core.GenericMemoryRef, []), @@ -2647,11 +2649,14 @@ struct D14919 <: Function; end @test B14919()() == "It's a brand new world" @test C14919()() == D14919()() == "Boo." -for f in (:Any, :Function, :(Core.Builtin), :(Union{Nothing, Type}), :(Union{typeof(+), Type}), :(Union{typeof(+), typeof(-)}), :(Base.Callable)) - @test_throws ErrorException("Method dispatch is unimplemented currently for this method signature") @eval (::$f)() = 1 -end -for f in (:(Core.getfield), :((::typeof(Core.getfield))), :((::Core.IntrinsicFunction))) - @test_throws ErrorException("cannot add methods to a builtin function") @eval $f() = 1 +let ex = ErrorException("cannot add methods to a builtin function") + for f in (:(Core.Any), :(Core.Function), :(Core.Builtin), :(Base.Callable), :(Union{Nothing,F} where F), :(typeof(Core.getfield)), :(Core.IntrinsicFunction)) + @test_throws ex @eval (::$f)() = 1 + end + @test_throws ex @eval (::Union{Nothing,F})() where {F<:Function} = 1 + for f in (:(Core.getfield),) + @test_throws ex @eval $f() = 1 + end end # issue #33370 @@ -5092,7 +5097,7 @@ function f16340(x::T) where T return g end let g = f16340(1) - @test isa(typeof(g).name.mt.defs.sig, UnionAll) + @test isa(only(methods(g)).sig, UnionAll) end # issue #16793 diff --git a/test/misc.jl b/test/misc.jl index 64b057d1ec4fc..d72ad5b58ad12 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1632,10 +1632,10 @@ end let errs = IOBuffer() run(`$(Base.julia_cmd()) -e ' using Test - @test isdefined(DataType.name.mt, :backedges) + @test isdefined(Type.body.name, :backedges) Base.Experimental.disable_new_worlds() @test_throws "disable_new_worlds" @eval f() = 1 - @test !isdefined(DataType.name.mt, :backedges) + @test !isdefined(Type.body.name, :backedges) @test_throws "disable_new_worlds" Base.delete_method(which(+, (Int, Int))) @test 1+1 == 2 using Dates diff --git a/test/precompile.jl b/test/precompile.jl index 2ee24a71dc62e..2555086214c77 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1958,8 +1958,12 @@ precompile_test_harness("PkgCacheInspector") do load_path end modules, init_order, edges, new_ext_cis, external_methods, new_method_roots, cache_sizes = sv - m = only(external_methods).func::Method - @test m.name == :repl_cmd && m.nargs < 2 + for m in external_methods + m = m.func::Method + if m.name !== :f + @test m.name == :repl_cmd && m.nargs == 1 + end + end @test new_ext_cis === nothing || any(new_ext_cis) do ci mi = ci.def::Core.MethodInstance mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} diff --git a/test/reflection.jl b/test/reflection.jl index 6da64ae7d7031..f7c81df32f41e 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -528,13 +528,13 @@ test_typed_ir_printing(g15714, Tuple{Vector{Float32}}, #@test used_dup_var_tested15715 @test used_unique_var_tested15714 -let li = typeof(fieldtype).name.mt.cache.func::Core.MethodInstance, +let li = only(methods(fieldtype)).unspecialized, lrepr = string(li), mrepr = string(li.def), lmime = repr("text/plain", li), mmime = repr("text/plain", li.def) - @test lrepr == lmime == "MethodInstance for fieldtype(...)" + @test lrepr == lmime == "MethodInstance for fieldtype(::Vararg{Any})" @test mrepr == "fieldtype(...) @ Core none:0" # simple print @test mmime == "fieldtype(...)\n @ Core none:0" # verbose print end diff --git a/test/show.jl b/test/show.jl index fa5989d6cd91d..89560d0e908d1 100644 --- a/test/show.jl +++ b/test/show.jl @@ -858,7 +858,7 @@ struct S45879{P} end let ms = methods(S45879) @test ms isa Base.MethodList @test length(ms) == 0 - @test sprint(show, Base.MethodList(Method[], typeof(S45879).name.mt)) isa String + @test sprint(show, Base.MethodList(Method[], typeof(S45879).name)) isa String end function f49475(a=12.0; b) end @@ -1598,7 +1598,7 @@ struct f_with_params{t} <: Function end end let io = IOBuffer() - show(io, MIME"text/html"(), ModFWithParams.f_with_params.body.name.mt) + show(io, MIME"text/html"(), methods(ModFWithParams.f_with_params{Int}())) @test occursin("ModFWithParams.f_with_params", String(take!(io))) end @@ -1780,10 +1780,10 @@ end anonfn_type_repr = "$modname.var\"$(typeof(anonfn).name.name)\"" @test repr(typeof(anonfn)) == anonfn_type_repr @test repr(anonfn) == anonfn_type_repr * "()" - @test repr("text/plain", anonfn) == "$(typeof(anonfn).name.mt.name) (generic function with 1 method)" + @test repr("text/plain", anonfn) == "$(typeof(anonfn).name.singletonname) (generic function with 1 method)" mkclosure = x->y->x+y clo = mkclosure(10) - @test repr("text/plain", clo) == "$(typeof(clo).name.mt.name) (generic function with 1 method)" + @test repr("text/plain", clo) == "$(typeof(clo).name.singletonname) (generic function with 1 method)" @test repr(UnionAll) == "UnionAll" end diff --git a/test/stacktraces.jl b/test/stacktraces.jl index ca553c2a2e801..3df0998fe88f6 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -90,7 +90,7 @@ f(x) = (y = h(x); y) trace = (try; f(3); catch; stacktrace(catch_backtrace()); end)[1:3] can_inline = Bool(Base.JLOptions().can_inline) for (frame, func, inlined) in zip(trace, [g,h,f], (can_inline, can_inline, false)) - @test frame.func === typeof(func).name.mt.name + @test frame.func === typeof(func).name.singletonname # broken until #50082 can be addressed mi = isa(frame.linfo, Core.CodeInstance) ? frame.linfo.def : frame.linfo @test mi.def.module === which(func, (Any,)).module broken=inlined @@ -109,10 +109,10 @@ let src = Meta.lower(Main, quote let x = 1 end end).args[1]::Core.CodeInfo repr = string(sf) @test repr == "Toplevel MethodInstance thunk at b:3" end -let li = typeof(fieldtype).name.mt.cache.func::Core.MethodInstance, +let li = only(methods(fieldtype)).unspecialized, sf = StackFrame(:a, :b, 3, li, false, false, 0), repr = string(sf) - @test repr == "fieldtype(...) at b:3" + @test repr == "fieldtype(::Vararg{Any}) at b:3" end let ctestptr = cglobal((:ctest, "libccalltest")), diff --git a/test/syntax.jl b/test/syntax.jl index 0c56b6b74b167..01b48918b5fde 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -596,10 +596,9 @@ let thismodule = @__MODULE__, @test !isdefined(M16096, :foo16096) @test !isdefined(M16096, :it) @test typeof(local_foo16096).name.module === thismodule - @test typeof(local_foo16096).name.mt.module === thismodule - @test getfield(thismodule, typeof(local_foo16096).name.mt.name) === local_foo16096 + @test getfield(thismodule, typeof(local_foo16096).name.singletonname) === local_foo16096 @test getfield(thismodule, typeof(local_foo16096).name.name) === typeof(local_foo16096) - @test !isdefined(M16096, typeof(local_foo16096).name.mt.name) + @test !isdefined(M16096, typeof(local_foo16096).name.singletonname) @test !isdefined(M16096, typeof(local_foo16096).name.name) end diff --git a/test/worlds.jl b/test/worlds.jl index 542bbaa440362..4b0ef208f9c7e 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -194,31 +194,26 @@ f_gen265(x::Type{Int}) = 3 # would have capped those specializations if they were still valid f26506(@nospecialize(x)) = 1 g26506(x) = Base.inferencebarrier(f26506)(x[1]) -z = Any["ABC"] +z26506 = Any["ABC"] f26506(x::Int) = 2 -g26506(z) # Places an entry for f26506(::String) in mt.name.cache +g26506(z26506) # Places an entry for f26506(::String) in MethodTable cache +w26506 = Base.get_world_counter() +cache26506 = ccall(:jl_mt_find_cache_entry, Any, (Any, Any, UInt), Core.GlobalMethods.cache, Tuple{typeof(f26506),String}, w26506)::Core.TypeMapEntry +@test cache26506.max_world === typemax(UInt) +w26506 = Base.get_world_counter() f26506(x::String) = 3 -let cache = typeof(f26506).name.mt.cache - # The entry we created above should have been truncated - @test cache.min_world == cache.max_world -end -c26506_1, c26506_2 = Condition(), Condition() -# Captures the world age -result26506 = Any[] -t = Task(()->begin - wait(c26506_1) - push!(result26506, g26506(z)) - notify(c26506_2) -end) -yield(t) +@test w26506+1 === Base.get_world_counter() +# The entry we created above should have been truncated +@test cache26506.max_world == w26506 +# Captures the world age on creation +t26506 = @task g26506(z26506) f26506(x::Float64) = 4 -let cache = typeof(f26506).name.mt.cache - # The entry we created above should have been truncated - @test cache.min_world == cache.max_world -end -notify(c26506_1) -wait(c26506_2) -@test result26506[1] == 3 +@test cache26506.max_world == w26506 +f26506(x::String) = 5 +# The entry we created above should not have been changed +@test cache26506.max_world == w26506 +@test fetch(schedule(t26506)) === 3 +@test g26506(z26506) === 5 # issue #38435 f38435(::Int, ::Any) = 1 From ef1936281249ff97dcbf99951f58792474b0cfab Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 28 May 2025 16:15:06 -0400 Subject: [PATCH 322/662] CI: `PrAssignee.yml`: `.catch()` and return the error object if 404 is encountered when checking collaborator status (#58550) Follow-up to #58303. --- .github/workflows/PrAssignee.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 0202ae20d8da7..728dcbb902cfa 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -38,7 +38,11 @@ jobs: uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: retries: 5 # retry GitHub API requests up to 5 times, with exponential backoff - retry-exempt-status-codes: 403 + retry-exempt-status-codes: "403,404" + # In the above list: + # + # - 404 is in the list because we will always hit a 404 when the PR author is not + # a committer. This 404 is normal and expected. script: | const oldPrAssignees = context.payload.pull_request.assignees .map(obj => obj.login) @@ -55,7 +59,26 @@ jobs: headers: { 'X-GitHub-Api-Version': '2022-11-28' } - }) + }).catch((error) => { + // So, I'm not sure if the error is being thrown by `@octokit/request.js` or + // `@octokit/plugin-retry.js`. I suspect that the initial error is thrown by + // `request.js`, and is subsequently propagated by `plugin-retry.js`. + // + // Anyway, the docs for `request.js` say the following: + // + // > If an error occurs, the promise is rejected with an `error` object + //> containing 3 keys to help with debugging: + // > + // > - `error.status` The http response status code + // > - `error.request` The request options such as `method`, `url` and `data` + // > - `error.response` The http response object with `url`, `headers`, and `data` + // + // Source: https://github.com/octokit/request.js/blob/main/README.md#request + // + // So, inside this `.catch()`, we simply return that `error` object. + + return error; + }); if (isCollaboratorResponseObj.status == 204) { var isCollaborator = true; } else if (isCollaboratorResponseObj.status == 404) { From e984c57381a195578465156c47ff663a7804ad84 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 28 May 2025 16:41:02 -0400 Subject: [PATCH 323/662] Convert remaining JLL stdlibs to LazyLibraries (#58444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Elliot Saba Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- .../src/CompilerSupportLibraries_jll.jl | 18 +- stdlib/GMP_jll/Project.toml | 1 + stdlib/GMP_jll/src/GMP_jll.jl | 61 +++--- stdlib/GMP_jll/test/runtests.jl | 2 +- .../src/LLVMLibUnwind_jll.jl | 37 ++-- stdlib/LLVMLibUnwind_jll/test/runtests.jl | 18 +- stdlib/LibCURL_jll/src/LibCURL_jll.jl | 45 ++-- stdlib/LibGit2_jll/src/LibGit2_jll.jl | 43 ++-- stdlib/LibSSH2_jll/Project.toml | 1 + stdlib/LibSSH2_jll/src/LibSSH2_jll.jl | 48 ++-- stdlib/LibUV_jll/src/LibUV_jll.jl | 1 + stdlib/LibUnwind_jll/Project.toml | 4 +- stdlib/LibUnwind_jll/src/LibUnwind_jll.jl | 49 +++-- stdlib/LibUnwind_jll/test/runtests.jl | 2 +- stdlib/MPFR_jll/src/MPFR_jll.jl | 33 ++- stdlib/Manifest.toml | 30 +-- stdlib/OpenBLAS_jll/Project.toml | 3 +- stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl | 14 +- stdlib/OpenLibm_jll/src/OpenLibm_jll.jl | 31 ++- stdlib/OpenSSL_jll/src/OpenSSL_jll.jl | 53 ++--- stdlib/PCRE2_jll/src/PCRE2_jll.jl | 34 +-- stdlib/Project.toml | 1 + stdlib/SuiteSparse_jll/Project.toml | 6 +- stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl | 206 ++++++++++-------- stdlib/Zlib_jll/src/Zlib_jll.jl | 33 ++- stdlib/Zstd_jll/src/Zstd_jll.jl | 31 ++- stdlib/dSFMT_jll/src/dSFMT_jll.jl | 34 +-- stdlib/libLLVM_jll/Project.toml | 4 +- stdlib/libLLVM_jll/src/libLLVM_jll.jl | 50 +++-- stdlib/libLLVM_jll/test/runtests.jl | 2 +- .../src/libblastrampoline_jll.jl | 6 + stdlib/nghttp2_jll/src/nghttp2_jll.jl | 31 ++- test/choosetests.jl | 2 +- test/stdlib_dependencies.jl | 199 +++++++++++++++++ test/trimming/Project.toml | 3 + test/trimming/Zstd_jll/Project.toml | 15 ++ test/trimming/Zstd_jll/src/Zstd_jll.jl | 73 +++++++ test/trimming/Zstd_jll/test/runtests.jl | 7 + test/trimming/basic_jll.jl | 2 +- 39 files changed, 820 insertions(+), 413 deletions(-) create mode 100644 test/stdlib_dependencies.jl create mode 100644 test/trimming/Zstd_jll/Project.toml create mode 100644 test/trimming/Zstd_jll/src/Zstd_jll.jl create mode 100644 test/trimming/Zstd_jll/test/runtests.jl diff --git a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl index 1f5c34c9c10ee..b7f887da3799e 100644 --- a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl +++ b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl @@ -64,13 +64,18 @@ if @isdefined(_libatomic_path) const libatomic = LazyLibrary(_libatomic_path) end const libgcc_s = LazyLibrary(_libgcc_s_path) -libgfortran_deps = [libgcc_s] + +_libgfortran_deps = [libgcc_s] if @isdefined _libquadmath_path const libquadmath = LazyLibrary(_libquadmath_path) - push!(libgfortran_deps, libquadmath) + push!(_libgfortran_deps, libquadmath) end -const libgfortran = LazyLibrary(_libgfortran_path, dependencies=libgfortran_deps) -const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=[libgcc_s]) + +const libgfortran = LazyLibrary(_libgfortran_path, dependencies=_libgfortran_deps) + +_libstdcxx_dependencies = LazyLibrary[libgcc_s] +const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=_libstdcxx_dependencies) + const libgomp = LazyLibrary(_libgomp_path) # Some installations (such as those from-source) may not have `libssp` @@ -116,4 +121,9 @@ function __init__() push!(LIBPATH_list, LIBPATH[]) end +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end + end # module CompilerSupportLibraries_jll diff --git a/stdlib/GMP_jll/Project.toml b/stdlib/GMP_jll/Project.toml index a31688d0a9c07..c17e5311a7d80 100644 --- a/stdlib/GMP_jll/Project.toml +++ b/stdlib/GMP_jll/Project.toml @@ -4,6 +4,7 @@ version = "6.3.0+2" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] diff --git a/stdlib/GMP_jll/src/GMP_jll.jl b/stdlib/GMP_jll/src/GMP_jll.jl index ae8b3c0b3e7d5..3a2375b54e1d4 100644 --- a/stdlib/GMP_jll/src/GMP_jll.jl +++ b/stdlib/GMP_jll/src/GMP_jll.jl @@ -2,51 +2,62 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl baremodule GMP_jll -using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +using Base, Libdl, CompilerSupportLibraries_jll export libgmp, libgmpxx # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libgmp_handle::Ptr{Cvoid} = C_NULL libgmp_path::String = "" -libgmpxx_handle::Ptr{Cvoid} = C_NULL libgmpxx_path::String = "" if Sys.iswindows() - const libgmp = "libgmp-10.dll" - const libgmpxx = "libgmpxx-4.dll" + const _libgmp_path = BundledLazyLibraryPath("libgmp-10.dll") + const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx-4.dll") +elseif Sys.isapple() + const _libgmp_path = BundledLazyLibraryPath("libgmp.10.dylib") + const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.4.dylib") +else + const _libgmp_path = BundledLazyLibraryPath("libgmp.so.10") + const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.so.4") +end + +const libgmp = LazyLibrary(_libgmp_path) + +if Sys.isfreebsd() + _libgmpxx_dependencies = LazyLibrary[libgmp, libgcc_s] elseif Sys.isapple() - const libgmp = "@rpath/libgmp.10.dylib" - const libgmpxx = "@rpath/libgmpxx.4.dylib" + _libgmpxx_dependencies = LazyLibrary[libgmp] else - const libgmp = "libgmp.so.10" - const libgmpxx = "libgmpxx.so.4" + _libgmpxx_dependencies = LazyLibrary[libgmp, libstdcxx, libgcc_s] +end +const libgmpxx = LazyLibrary( + _libgmpxx_path, + dependencies=_libgmpxx_dependencies, +) + +function eager_mode() + CompilerSupportLibraries_jll.eager_mode() + dlopen(libgmp) + dlopen(libgmpxx) end +is_available() = true function __init__() - global libgmp_handle = dlopen(libgmp) - global libgmp_path = dlpath(libgmp_handle) - global libgmpxx_handle = dlopen(libgmpxx) - global libgmpxx_path = dlpath(libgmpxx_handle) + global libgmp_path = string(_libgmp_path) + global libgmpxx_path = string(_libgmpxx_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgmp_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libgmp_path() = libgmp_path -get_libgmpxx_path() = libgmpxx_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module GMP_jll diff --git a/stdlib/GMP_jll/test/runtests.jl b/stdlib/GMP_jll/test/runtests.jl index b2b35b98cbe17..ad0e2dc8a4944 100644 --- a/stdlib/GMP_jll/test/runtests.jl +++ b/stdlib/GMP_jll/test/runtests.jl @@ -3,6 +3,6 @@ using Test, Libdl, GMP_jll @testset "GMP_jll" begin - vn = VersionNumber(unsafe_string(unsafe_load(cglobal((:__gmp_version, libgmp), Ptr{Cchar})))) + vn = VersionNumber(unsafe_string(unsafe_load(cglobal(dlsym(libgmp, :__gmp_version), Ptr{Cchar})))) @test vn == v"6.3.0" end diff --git a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl index 429e35b91d3f2..4adae99c520d4 100644 --- a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl +++ b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl @@ -5,38 +5,33 @@ baremodule LLVMLibUnwind_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export llvmlibunwind # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -llvmlibunwind_handle::Ptr{Cvoid} = C_NULL llvmlibunwind_path::String = "" -const llvmlibunwind = "libunwind" +const _llvmlibunwind_path = BundledLazyLibraryPath("libunwind") +const llvmlibunwind = LazyLibrary(_llvmlibunwind_path) +function eager_mode() + dlopen(llvmlibunwind) +end +is_available() = @static Sys.isapple() ? true : false function __init__() - # We only dlopen something on MacOS - @static if Sys.isapple() - global llvmlibunwind_handle = dlopen(llvmlibunwind) - global llvmlibunwind_path = dlpath(llvmlibunwind_handle) - global artifact_dir = dirname(Sys.BINDIR) - LIBPATH[] = dirname(llvmlibunwind_path) - push!(LIBPATH_list, LIBPATH[]) - end + global llvmlibunwind_path = string(_llvmlibunwind_path) + global artifact_dir = dirname(Sys.BINDIR) + LIBPATH[] = dirname(llvmlibunwind_path) + push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = @static Sys.isapple() ? true : false -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_llvmlibunwind_path() = llvmlibunwind_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LLVMLibUnwind_jll diff --git a/stdlib/LLVMLibUnwind_jll/test/runtests.jl b/stdlib/LLVMLibUnwind_jll/test/runtests.jl index e984593ab2c25..42afe50a875f6 100644 --- a/stdlib/LLVMLibUnwind_jll/test/runtests.jl +++ b/stdlib/LLVMLibUnwind_jll/test/runtests.jl @@ -1,16 +1,14 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Libdl -using LLVMLibUnwind_jll: llvmlibunwind_handle - +using Test, Libdl, LLVMLibUnwind_jll @testset "LLVMLibUnwind_jll" begin if Sys.isapple() - @test dlsym(llvmlibunwind_handle, :unw_getcontext; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_init_local; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_init_local_dwarf; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_step; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_get_reg; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_set_reg; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_resume; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_getcontext; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_init_local; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_init_local_dwarf; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_step; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_get_reg; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_set_reg; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_resume; throw_error=false) !== nothing end end diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index 5c1c2aa14b23a..92af7af03881c 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -9,41 +9,52 @@ if !(Sys.iswindows() || Sys.isapple()) using OpenSSL_jll end -const PATH_list = String[] -const LIBPATH_list = String[] - export libcurl # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libcurl_handle::Ptr{Cvoid} = C_NULL libcurl_path::String = "" +_libcurl_dependencies = LazyLibrary[libz, libnghttp2, libssh2] +if !(Sys.iswindows() || Sys.isapple()) + append!(_libcurl_dependencies, [libssl, libcrypto]) +end + if Sys.iswindows() - const libcurl = "libcurl-4.dll" + const _libcurl_path = BundledLazyLibraryPath("libcurl-4.dll") elseif Sys.isapple() - const libcurl = "@rpath/libcurl.4.dylib" + const _libcurl_path = BundledLazyLibraryPath("libcurl.4.dylib") else - const libcurl = "libcurl.so.4" + const _libcurl_path = BundledLazyLibraryPath("libcurl.so.4") end +const libcurl = LazyLibrary( + _libcurl_path, + dependencies=_libcurl_dependencies, +) + +function eager_mode() + Zlib_jll.eager_mode() + nghttp2_jll.eager_mode() + LibSSH2_jll.eager_mode() + dlopen(libcurl) +end +is_available() = true + function __init__() - global libcurl_handle = dlopen(libcurl) - global libcurl_path = dlpath(libcurl_handle) + global libcurl_path = string(_libcurl_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libcurl_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libcurl_path() = libcurl_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibCURL_jll diff --git a/stdlib/LibGit2_jll/src/LibGit2_jll.jl b/stdlib/LibGit2_jll/src/LibGit2_jll.jl index c69deb4a9d932..26f679b2634e8 100644 --- a/stdlib/LibGit2_jll/src/LibGit2_jll.jl +++ b/stdlib/LibGit2_jll/src/LibGit2_jll.jl @@ -9,41 +9,50 @@ if !(Sys.iswindows() || Sys.isapple()) using OpenSSL_jll end -const PATH_list = String[] -const LIBPATH_list = String[] - export libgit2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libgit2_handle::Ptr{Cvoid} = C_NULL libgit2_path::String = "" if Sys.iswindows() - const libgit2 = "libgit2.dll" + const _libgit2_path = BundledLazyLibraryPath("libgit2.dll") elseif Sys.isapple() - const libgit2 = "@rpath/libgit2.1.9.dylib" + const _libgit2_path = BundledLazyLibraryPath("libgit2.1.9.dylib") +else + const _libgit2_path = BundledLazyLibraryPath("libgit2.so.1.9") +end + +if Sys.isfreebsd() + _libgit2_dependencies = LazyLibrary[libssh2, libssl, libcrypto] else - const libgit2 = "libgit2.so.1.9" + _libgit2_dependencies = LazyLibrary[libssh2] +end +const libgit2 = LazyLibrary(_libgit2_path, dependencies=_libgit2_dependencies) + +function eager_mode() + LibSSH2_jll.eager_mode() + @static if !(Sys.iswindows() || Sys.isapple()) + OpenSSL_jll.eager_mode() + end + dlopen(libgit2) end +is_available() = true function __init__() - global libgit2_handle = dlopen(libgit2) - global libgit2_path = dlpath(libgit2_handle) + global libgit2_path = string(_libgit2_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgit2_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libgit2_path() = libgit2_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibGit2_jll diff --git a/stdlib/LibSSH2_jll/Project.toml b/stdlib/LibSSH2_jll/Project.toml index 09f07b559344c..f535847ae2f29 100644 --- a/stdlib/LibSSH2_jll/Project.toml +++ b/stdlib/LibSSH2_jll/Project.toml @@ -6,6 +6,7 @@ version = "1.11.3+1" OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" [compat] julia = "1.8" diff --git a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl index e9392fe34a918..8c0884e8aca7b 100644 --- a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl +++ b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl @@ -3,48 +3,58 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibSSH2_jll.jl baremodule LibSSH2_jll -using Base, Libdl +using Base, Libdl, Zlib_jll if !Sys.iswindows() # On Windows we use system SSL/crypto libraries using OpenSSL_jll end -const PATH_list = String[] -const LIBPATH_list = String[] - export libssh2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libssh2_handle::Ptr{Cvoid} = C_NULL libssh2_path::String = "" + if Sys.iswindows() - const libssh2 = "libssh2.dll" + const _libssh2_path = BundledLazyLibraryPath("libssh2.dll") + _libssh2_dependencies = LazyLibrary[] elseif Sys.isapple() - const libssh2 = "@rpath/libssh2.1.dylib" + const _libssh2_path = BundledLazyLibraryPath("libssh2.1.dylib") + _libssh2_dependencies = LazyLibrary[libz, libcrypto] +elseif Sys.isfreebsd() + const _libssh2_path = BundledLazyLibraryPath("libssh2.so.1") + _libssh2_dependencies = LazyLibrary[libz, libcrypto] else - const libssh2 = "libssh2.so.1" + const _libssh2_path = BundledLazyLibraryPath("libssh2.so.1") + _libssh2_dependencies = LazyLibrary[libcrypto] end +const libssh2 = LazyLibrary(_libssh2_path, dependencies=_libssh2_dependencies) + +function eager_mode() + Zlib_jll.eager_mode() + @static if !Sys.iswindows() + OpenSSL_jll.eager_mode() + end + dlopen(libssh2) +end +is_available() = true + function __init__() - global libssh2_handle = dlopen(libssh2) - global libssh2_path = dlpath(libssh2_handle) + global libssh2_path = string(_libssh2_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssh2_path) push!(LIBPATH_list, LIBPATH[]) end - -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libssh2_path() = libssh2_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibSSH2_jll diff --git a/stdlib/LibUV_jll/src/LibUV_jll.jl b/stdlib/LibUV_jll/src/LibUV_jll.jl index febc47f168ab9..5bf9b7ef3b0fb 100644 --- a/stdlib/LibUV_jll/src/LibUV_jll.jl +++ b/stdlib/LibUV_jll/src/LibUV_jll.jl @@ -6,5 +6,6 @@ baremodule LibUV_jll using Base, Libdl # NOTE: This file is currently empty, as we link libuv statically for now. +is_available() = true end # module LibUV_jll diff --git a/stdlib/LibUnwind_jll/Project.toml b/stdlib/LibUnwind_jll/Project.toml index b43f1c537ce5a..8e6ba70a3deb3 100644 --- a/stdlib/LibUnwind_jll/Project.toml +++ b/stdlib/LibUnwind_jll/Project.toml @@ -3,8 +3,10 @@ uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" version = "1.8.1+2" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" [compat] julia = "1.6" diff --git a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl index f97b18443b6fd..29659a18df0e0 100644 --- a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl +++ b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl @@ -4,39 +4,46 @@ baremodule LibUnwind_jll using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +using CompilerSupportLibraries_jll +using Zlib_jll export libunwind # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libunwind_handle::Ptr{Cvoid} = C_NULL libunwind_path::String = "" -const libunwind = "libunwind.so.8" +const _libunwind_path = BundledLazyLibraryPath("libunwind.so.8") + +if Sys.isfreebsd() + _libunwind_dependencies = LazyLibrary[libz] +else + _libunwind_dependencies = LazyLibrary[libgcc_s, libz] +end +const libunwind = LazyLibrary(_libunwind_path, dependencies=_libunwind_dependencies) + + +function eager_mode() + CompilerSupportLibraries_jll.eager_mode() + Zlib_jll.eager_mode() + dlopen(libunwind) +end +is_available() = @static(Sys.islinux() || Sys.isfreebsd()) ? true : false function __init__() - # We only do something on Linux/FreeBSD - @static if Sys.islinux() || Sys.isfreebsd() - global libunwind_handle = dlopen(libunwind) - global libunwind_path = dlpath(libunwind_handle) - global artifact_dir = dirname(Sys.BINDIR) - LIBPATH[] = dirname(libunwind_path) - push!(LIBPATH_list, LIBPATH[]) - end + global libunwind_path = string(_libunwind_path) + global artifact_dir = dirname(Sys.BINDIR) + LIBPATH[] = dirname(libunwind_path) + push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = @static (Sys.islinux() || Sys.isfreebsd()) ? true : false -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libunwind_path() = libunwind_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibUnwind_jll diff --git a/stdlib/LibUnwind_jll/test/runtests.jl b/stdlib/LibUnwind_jll/test/runtests.jl index 1cb33dd6729e3..c87ccad988dec 100644 --- a/stdlib/LibUnwind_jll/test/runtests.jl +++ b/stdlib/LibUnwind_jll/test/runtests.jl @@ -4,6 +4,6 @@ using Test, Libdl, LibUnwind_jll @testset "LibUnwind_jll" begin if !Sys.isapple() && !Sys.iswindows() - @test dlsym(LibUnwind_jll.libunwind_handle, :unw_backtrace; throw_error=false) !== nothing + @test dlsym(LibUnwind_jll.libunwind, :unw_backtrace; throw_error=false) !== nothing end end diff --git a/stdlib/MPFR_jll/src/MPFR_jll.jl b/stdlib/MPFR_jll/src/MPFR_jll.jl index 219ab0cad41be..58c79d1ab87ff 100644 --- a/stdlib/MPFR_jll/src/MPFR_jll.jl +++ b/stdlib/MPFR_jll/src/MPFR_jll.jl @@ -4,41 +4,38 @@ baremodule MPFR_jll using Base, Libdl, GMP_jll -const PATH_list = String[] -const LIBPATH_list = String[] - export libmpfr # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libmpfr_handle::Ptr{Cvoid} = C_NULL libmpfr_path::String = "" if Sys.iswindows() - const libmpfr = "libmpfr-6.dll" + const _libmpfr_path = BundledLazyLibraryPath("libmpfr-6.dll") elseif Sys.isapple() - const libmpfr = "@rpath/libmpfr.6.dylib" + const _libmpfr_path = BundledLazyLibraryPath("libmpfr.6.dylib") else - const libmpfr = "libmpfr.so.6" + const _libmpfr_path = BundledLazyLibraryPath("libmpfr.so.6") +end + +_libmpfr_dependencies = LazyLibrary[libgmp] + +const libmpfr = LazyLibrary(_libmpfr_path, dependencies=_libmpfr_dependencies) + +function eager_mode() + dlopen(libmpfr) end +is_available() = true function __init__() - global libmpfr_handle = dlopen(libmpfr) - global libmpfr_path = dlpath(libmpfr_handle) + global libmpfr_path = string(_libmpfr_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libmpfr_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libmpfr_path() = libmpfr_path - end # module MPFR_jll diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 71c942549e804..2866272e90073 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" +julia_version = "1.13.0-DEV" manifest_format = "2.0" -project_hash = "1cb1aede0b4f0a2f12806233b9f188a63d6acf04" +project_hash = "6826701002e0b87f8744b1c4bf97e2cff5fc1642" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -23,7 +23,7 @@ version = "1.11.0" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.2.0+0" +version = "1.3.0+1" [[deps.Dates]] deps = ["Printf"] @@ -44,7 +44,7 @@ version = "1.11.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" +version = "1.7.0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -56,7 +56,7 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.GMP_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" version = "6.3.0+2" @@ -73,7 +73,7 @@ version = "1.12.0" [[deps.LLD_jll]] deps = ["Artifacts", "Libdl", "Zlib_jll", "libLLVM_jll"] uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" -version = "18.1.7+3" +version = "20.1.2+0" [[deps.LLVMLibUnwind_jll]] deps = ["Artifacts", "Libdl"] @@ -106,7 +106,7 @@ uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" version = "1.9.0+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -116,7 +116,7 @@ uuid = "183b4373-6708-53ba-ad28-60e28bb38547" version = "2.0.1+20" [[deps.LibUnwind_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "Zlib_jll"] uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" version = "1.8.1+2" @@ -127,7 +127,7 @@ version = "1.11.0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" +version = "1.12.0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -149,7 +149,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2024.11.26" +version = "2025.2.25" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -158,7 +158,7 @@ version = "1.3.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.28+3" +version = "0.3.29+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] @@ -178,7 +178,7 @@ version = "10.44.0+1" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.0" +version = "1.13.0" weakdeps = ["REPL"] [deps.Pkg.extensions] @@ -195,7 +195,7 @@ uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" version = "1.11.0" [[deps.REPL]] -deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" @@ -241,7 +241,7 @@ uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" version = "1.11.0" [[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" version = "7.10.1+0" @@ -285,7 +285,7 @@ uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" version = "2.2.5+2" [[deps.libLLVM_jll]] -deps = ["Artifacts", "Libdl", "Zlib_jll", "Zstd_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "Zlib_jll", "Zstd_jll"] uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" version = "20.1.2+1" diff --git a/stdlib/OpenBLAS_jll/Project.toml b/stdlib/OpenBLAS_jll/Project.toml index 07a81d3c1d547..8eafa2f2365c1 100644 --- a/stdlib/OpenBLAS_jll/Project.toml +++ b/stdlib/OpenBLAS_jll/Project.toml @@ -3,10 +3,9 @@ uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.29+0" [deps] -# See note in `src/OpenBLAS_jll.jl` about this dependency. +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] julia = "1.11" diff --git a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl index bd93b050ebbee..e3f4f14391369 100644 --- a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl +++ b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl @@ -2,7 +2,7 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenBLAS_jll.jl baremodule OpenBLAS_jll -using Base, Libdl, Base.BinaryPlatforms +using Base, Libdl using CompilerSupportLibraries_jll export libopenblas @@ -28,7 +28,17 @@ elseif Sys.isapple() else const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".so")) end -const libopenblas = LazyLibrary(_libopenblas_path, dependencies=[libgfortran]) + +_libopenblas_dependencies = LazyLibrary[libgfortran] +if Sys.isapple() + if isdefined(CompilerSupportLibraries_jll, :libquadmath) + push!(_libopenblas_dependencies, CompilerSupportLibraries_jll.libquadmath) + end + if Sys.ARCH != :aarch64 + push!(_libopenblas_dependencies, CompilerSupportLibraries_jll.libgcc_s) + end +end +const libopenblas = LazyLibrary(_libopenblas_path, dependencies=_libopenblas_dependencies) # Conform to LazyJLLWrappers API function eager_mode() diff --git a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl index 297cd25512894..54c7e6d64b7d4 100644 --- a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl +++ b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl @@ -4,41 +4,36 @@ baremodule OpenLibm_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libopenlibm # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libopenlibm_handle::Ptr{Cvoid} = C_NULL libopenlibm_path::String = "" if Sys.iswindows() - const libopenlibm = "libopenlibm.dll" + const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.dll") elseif Sys.isapple() - const libopenlibm = "@rpath/libopenlibm.4.dylib" + const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.4.dylib") else - const libopenlibm = "libopenlibm.so.4" + const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.so.4") end +const libopenlibm = LazyLibrary(_libopenlibm_path) + +function eager_mode() + dlopen(libopenlibm) +end +is_available() = true + function __init__() - global libopenlibm_handle = dlopen(libopenlibm) - global libopenlibm_path = dlpath(libopenlibm_handle) + global libopenlibm_path = string(_libopenlibm_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libopenlibm_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libopenlibm_path() = libopenlibm_path - end # module OpenLibm_jll diff --git a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl index bba9a0a299de9..b9169b5f9c5cf 100644 --- a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl +++ b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl @@ -5,54 +5,55 @@ baremodule OpenSSL_jll using Base, Libdl, Base.BinaryPlatforms -const PATH_list = String[] -const LIBPATH_list = String[] - export libcrypto, libssl # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libcrypto_handle::Ptr{Cvoid} = C_NULL libcrypto_path::String = "" -libssl_handle::Ptr{Cvoid} = C_NULL libssl_path::String = "" if Sys.iswindows() if arch(HostPlatform()) == "x86_64" - const libcrypto = "libcrypto-3-x64.dll" - const libssl = "libssl-3-x64.dll" + const _libcrypto_path = BundledLazyLibraryPath("libcrypto-3-x64.dll") + const _libssl_path = BundledLazyLibraryPath("libssl-3-x64.dll") else - const libcrypto = "libcrypto-3.dll" - const libssl = "libssl-3.dll" + const _libcrypto_path = BundledLazyLibraryPath("libcrypto-3.dll") + const _libssl_path = BundledLazyLibraryPath("libssl-3.dll") end elseif Sys.isapple() - const libcrypto = "@rpath/libcrypto.3.dylib" - const libssl = "@rpath/libssl.3.dylib" + const _libcrypto_path = BundledLazyLibraryPath("libcrypto.3.dylib") + const _libssl_path = BundledLazyLibraryPath("libssl.3.dylib") else - const libcrypto = "libcrypto.so.3" - const libssl = "libssl.so.3" + const _libcrypto_path = BundledLazyLibraryPath("libcrypto.so.3") + const _libssl_path = BundledLazyLibraryPath("libssl.so.3") +end + +const libcrypto = LazyLibrary(_libcrypto_path) + +_libssl_dependencies = LazyLibrary[libcrypto] +const libssl = LazyLibrary(_libssl_path, dependencies=_libssl_dependencies) + +function eager_mode() + dlopen(libcrypto) + dlopen(libssl) end +is_available() = true function __init__() - global libcrypto_handle = dlopen(libcrypto) - global libcrypto_path = dlpath(libcrypto_handle) - global libssl_handle = dlopen(libssl) - global libssl_path = dlpath(libssl_handle) + global libcrypto_path = string(_libcrypto_path) + global libssl_path = string(_libssl_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssl_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libcrypto_path() = libcrypto_path -get_libssl_path() = libssl_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module OpenSSL_jll diff --git a/stdlib/PCRE2_jll/src/PCRE2_jll.jl b/stdlib/PCRE2_jll/src/PCRE2_jll.jl index d825ac74db5a8..9af7fb3863a23 100644 --- a/stdlib/PCRE2_jll/src/PCRE2_jll.jl +++ b/stdlib/PCRE2_jll/src/PCRE2_jll.jl @@ -4,41 +4,41 @@ baremodule PCRE2_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libpcre2_8 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libpcre2_8_handle::Ptr{Cvoid} = C_NULL libpcre2_8_path::String = "" if Sys.iswindows() - const libpcre2_8 = "libpcre2-8-0.dll" + const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8-0.dll") elseif Sys.isapple() - const libpcre2_8 = "@rpath/libpcre2-8.0.dylib" + const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8.0.dylib") else - const libpcre2_8 = "libpcre2-8.so.0" + const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8.so.0") end +const libpcre2_8 = LazyLibrary(_libpcre2_8_path) + +function eager_mode() + dlopen(libpcre2_8) +end +is_available() = true + function __init__() - global libpcre2_8_handle = dlopen(libpcre2_8) - global libpcre2_8_path = dlpath(libpcre2_8_handle) + global libpcre2_8_path = string(_libpcre2_8_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libpcre2_8_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libpcre2_8_path() = libpcre2_8_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module PCRE2_jll diff --git a/stdlib/Project.toml b/stdlib/Project.toml index 1e03a0f474490..2051377a6967c 100644 --- a/stdlib/Project.toml +++ b/stdlib/Project.toml @@ -54,6 +54,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" dSFMT_jll = "05ff407c-b0c1-5878-9df8-858cc2e60c36" libLLVM_jll = "8f36deef-c2a5-5394-99ed-8e07531fb29a" libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" diff --git a/stdlib/SuiteSparse_jll/Project.toml b/stdlib/SuiteSparse_jll/Project.toml index cfe46b3fd9569..ff454476db148 100644 --- a/stdlib/SuiteSparse_jll/Project.toml +++ b/stdlib/SuiteSparse_jll/Project.toml @@ -3,11 +3,13 @@ uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" version = "7.10.1+0" [deps] -libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.13" [extras] diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index b0699f132eb7f..6fb4f4c53bfa6 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -2,139 +2,153 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/SuiteSparse_jll.jl baremodule SuiteSparse_jll -using Base, Libdl, libblastrampoline_jll - -const PATH_list = String[] -const LIBPATH_list = String[] +using Base, Libdl, libblastrampoline_jll, CompilerSupportLibraries_jll export libamd, libbtf, libcamd, libccolamd, libcholmod, libcolamd, libklu, libldl, librbio, libspqr, libsuitesparseconfig, libumfpack # These get calculated in __init__() # Man I can't wait until these are automatically handled by an in-Base JLLWrappers clone. const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libamd_handle::Ptr{Cvoid} = C_NULL libamd_path::String = "" -libbtf_handle::Ptr{Cvoid} = C_NULL libbtf_path::String = "" -libcamd_handle::Ptr{Cvoid} = C_NULL libcamd_path::String = "" -libccolamd_handle::Ptr{Cvoid} = C_NULL libccolamd_path::String = "" -libcholmod_handle::Ptr{Cvoid} = C_NULL libcholmod_path::String = "" -libcolamd_handle::Ptr{Cvoid} = C_NULL libcolamd_path::String = "" -libklu_handle::Ptr{Cvoid} = C_NULL libklu_path::String = "" -libldl_handle::Ptr{Cvoid} = C_NULL libldl_path::String = "" -librbio_handle::Ptr{Cvoid} = C_NULL librbio_path::String = "" -libspqr_handle::Ptr{Cvoid} = C_NULL libspqr_path::String = "" -libsuitesparseconfig_handle::Ptr{Cvoid} = C_NULL libsuitesparseconfig_path::String = "" -libumfpack_handle::Ptr{Cvoid} = C_NULL libumfpack_path::String = "" if Sys.iswindows() - const libamd = "libamd.dll" - const libbtf = "libbtf.dll" - const libcamd = "libcamd.dll" - const libccolamd = "libccolamd.dll" - const libcholmod = "libcholmod.dll" - const libcolamd = "libcolamd.dll" - const libklu = "libklu.dll" - const libldl = "libldl.dll" - const librbio = "librbio.dll" - const libspqr = "libspqr.dll" - const libsuitesparseconfig = "libsuitesparseconfig.dll" - const libumfpack = "libumfpack.dll" + const _libamd_path = BundledLazyLibraryPath("libamd.dll") + const _libbtf_path = BundledLazyLibraryPath("libbtf.dll") + const _libcamd_path = BundledLazyLibraryPath("libcamd.dll") + const _libccolamd_path = BundledLazyLibraryPath("libccolamd.dll") + const _libcholmod_path = BundledLazyLibraryPath("libcholmod.dll") + const _libcolamd_path = BundledLazyLibraryPath("libcolamd.dll") + const _libklu_path = BundledLazyLibraryPath("libklu.dll") + const _libldl_path = BundledLazyLibraryPath("libldl.dll") + const _librbio_path = BundledLazyLibraryPath("librbio.dll") + const _libspqr_path = BundledLazyLibraryPath("libspqr.dll") + const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.dll") + const _libumfpack_path = BundledLazyLibraryPath("libumfpack.dll") elseif Sys.isapple() - const libamd = "@rpath/libamd.3.dylib" - const libbtf = "@rpath/libbtf.2.dylib" - const libcamd = "@rpath/libcamd.3.dylib" - const libccolamd = "@rpath/libccolamd.3.dylib" - const libcholmod = "@rpath/libcholmod.5.dylib" - const libcolamd = "@rpath/libcolamd.3.dylib" - const libklu = "@rpath/libklu.2.dylib" - const libldl = "@rpath/libldl.3.dylib" - const librbio = "@rpath/librbio.4.dylib" - const libspqr = "@rpath/libspqr.4.dylib" - const libsuitesparseconfig = "@rpath/libsuitesparseconfig.7.dylib" - const libumfpack = "@rpath/libumfpack.6.dylib" + const _libamd_path = BundledLazyLibraryPath("libamd.3.dylib") + const _libbtf_path = BundledLazyLibraryPath("libbtf.2.dylib") + const _libcamd_path = BundledLazyLibraryPath("libcamd.3.dylib") + const _libccolamd_path = BundledLazyLibraryPath("libccolamd.3.dylib") + const _libcholmod_path = BundledLazyLibraryPath("libcholmod.5.dylib") + const _libcolamd_path = BundledLazyLibraryPath("libcolamd.3.dylib") + const _libklu_path = BundledLazyLibraryPath("libklu.2.dylib") + const _libldl_path = BundledLazyLibraryPath("libldl.3.dylib") + const _librbio_path = BundledLazyLibraryPath("librbio.4.dylib") + const _libspqr_path = BundledLazyLibraryPath("libspqr.4.dylib") + const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.7.dylib") + const _libumfpack_path = BundledLazyLibraryPath("libumfpack.6.dylib") else - const libamd = "libamd.so.3" - const libbtf = "libbtf.so.2" - const libcamd = "libcamd.so.3" - const libccolamd = "libccolamd.so.3" - const libcholmod = "libcholmod.so.5" - const libcolamd = "libcolamd.so.3" - const libklu = "libklu.so.2" - const libldl = "libldl.so.3" - const librbio = "librbio.so.4" - const libspqr = "libspqr.so.4" - const libsuitesparseconfig = "libsuitesparseconfig.so.7" - const libumfpack = "libumfpack.so.6" + const _libamd_path = BundledLazyLibraryPath("libamd.so.3") + const _libbtf_path = BundledLazyLibraryPath("libbtf.so.2") + const _libcamd_path = BundledLazyLibraryPath("libcamd.so.3") + const _libccolamd_path = BundledLazyLibraryPath("libccolamd.so.3") + const _libcholmod_path = BundledLazyLibraryPath("libcholmod.so.5") + const _libcolamd_path = BundledLazyLibraryPath("libcolamd.so.3") + const _libklu_path = BundledLazyLibraryPath("libklu.so.2") + const _libldl_path = BundledLazyLibraryPath("libldl.so.3") + const _librbio_path = BundledLazyLibraryPath("librbio.so.4") + const _libspqr_path = BundledLazyLibraryPath("libspqr.so.4") + const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.so.7") + const _libumfpack_path = BundledLazyLibraryPath("libumfpack.so.6") end -function __init__() +const libsuitesparseconfig = LazyLibrary(_libsuitesparseconfig_path) +const libldl = LazyLibrary(_libldl_path) +const libbtf = LazyLibrary(_libbtf_path) + +_libcolamd_dependencies = LazyLibrary[libsuitesparseconfig] +const libcolamd = LazyLibrary(_libcolamd_path; dependencies=_libcolamd_dependencies) + +_libamd_dependencies = LazyLibrary[libsuitesparseconfig] +const libamd = LazyLibrary(_libamd_path; dependencies=_libamd_dependencies) + +_libcamd_dependencies = LazyLibrary[libsuitesparseconfig] +const libcamd = LazyLibrary(_libcamd_path; dependencies=_libcamd_dependencies) + +_libccolamd_dependencies = LazyLibrary[libsuitesparseconfig] +const libccolamd = LazyLibrary(_libccolamd_path; dependencies=_libccolamd_dependencies) + +_librbio_dependencies = LazyLibrary[libsuitesparseconfig] +const librbio = LazyLibrary(_librbio_path; dependencies=_librbio_dependencies) + +_libcholmod_dependencies = LazyLibrary[ + libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline + ] +const libcholmod = LazyLibrary(_libcholmod_path; dependencies=_libcholmod_dependencies) + +_libklu_dependencies = LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf] +const libklu = LazyLibrary(_libklu_path; dependencies=_libklu_dependencies) + +if Sys.isfreebsd() || Sys.isapple() + _libspqr_dependencies = LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline] +else + _libspqr_dependencies = LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libstdcxx, libgcc_s] +end +const libspqr = LazyLibrary(_libspqr_path; dependencies=_libspqr_dependencies) + +_libumfpack_dependencies = LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline] +const libumfpack = LazyLibrary(_libumfpack_path; dependencies=_libumfpack_dependencies) + +function eager_mode() + CompilerSupportLibraries_jll.eager_mode() libblastrampoline_jll.eager_mode() + dlopen(libamd) + dlopen(libbtf) + dlopen(libcamd) + dlopen(libccolamd) + dlopen(libcholmod) + dlopen(libcolamd) + dlopen(libklu) + dlopen(libldl) + dlopen(librbio) + dlopen(libspqr) + dlopen(libsuitesparseconfig) + dlopen(libumfpack) +end +is_available() = true + +function __init__() # BSD-3-Clause - global libamd_handle = dlopen(libamd) - global libamd_path = dlpath(libamd_handle) - global libcamd_handle = dlopen(libcamd) - global libcamd_path = dlpath(libcamd_handle) - global libccolamd_handle = dlopen(libccolamd) - global libccolamd_path = dlpath(libccolamd_handle) - global libcolamd_handle = dlopen(libcolamd) - global libcolamd_path = dlpath(libcolamd_handle) - global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) - global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) + global libamd_path = string(_libamd_path) + global libcamd_path = string(_libcamd_path) + global libccolamd_path = string(_libccolamd_path) + global libcolamd_path = string(_libcolamd_path) + global libsuitesparseconfig_path = string(_libsuitesparseconfig_path) # LGPL-2.1+ - global libbtf_handle = dlopen(libbtf) - global libbtf_path = dlpath(libbtf_handle) - global libklu_handle = dlopen(libklu) - global libklu_path = dlpath(libklu_handle) - global libldl_handle = dlopen(libldl) - global libldl_path = dlpath(libldl_handle) + global libbtf_path = string(_libbtf_path) + global libklu_path = string(_libklu_path) + global libldl_path = string(_libldl_path) # GPL-2.0+ if Base.USE_GPL_LIBS - global libcholmod_handle = dlopen(libcholmod) - global libcholmod_path = dlpath(libcholmod_handle) - global librbio_handle = dlopen(librbio) - global librbio_path = dlpath(librbio_handle) - global libspqr_handle = dlopen(libspqr) - global libspqr_path = dlpath(libspqr_handle) - global libumfpack_handle = dlopen(libumfpack) - global libumfpack_path = dlpath(libumfpack_handle) + global libcholmod_path = string(_libcholmod_path) + global librbio_path = string(_librbio_path) + global libspqr_path = string(_libspqr_path) + global libumfpack_path = string(_libumfpack_path) end global artifact_dir = dirname(Sys.BINDIR) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libamd_path() = libamd_path -get_libbtf_path() = libbtf_path -get_libcamd_path() = libcamd_path -get_libccolamd_path() = libccolamd_path -get_libcholmod_path() = libcholmod_path -get_libcolamd_path() = libcolamd_path -get_libklu_path() = libklu_path -get_libldl_path() = libldl_path -get_librbio_path() = librbio_path -get_libspqr_path() = libspqr_path -get_libsuitesparseconfig_path() = libsuitesparseconfig_path -get_libumfpack_path() = libumfpack_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module SuiteSparse_jll diff --git a/stdlib/Zlib_jll/src/Zlib_jll.jl b/stdlib/Zlib_jll/src/Zlib_jll.jl index fb043c7143789..649cb93f862da 100644 --- a/stdlib/Zlib_jll/src/Zlib_jll.jl +++ b/stdlib/Zlib_jll/src/Zlib_jll.jl @@ -4,41 +4,40 @@ baremodule Zlib_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libz # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libz_handle::Ptr{Cvoid} = C_NULL libz_path::String = "" if Sys.iswindows() - const libz = "libz.dll" + const _libz_path = BundledLazyLibraryPath("libz.dll") elseif Sys.isapple() - const libz = "@rpath/libz.1.dylib" + const _libz_path = BundledLazyLibraryPath("libz.1.dylib") else - const libz = "libz.so.1" + const _libz_path = BundledLazyLibraryPath("libz.so.1") end +const libz = LazyLibrary(_libz_path) + +function eager_mode() + dlopen(libz) +end +is_available() = true function __init__() - global libz_handle = dlopen(libz) - global libz_path = dlpath(libz_handle) + global libz_path = string(_libz_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libz_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libz_path() = libz_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module Zlib_jll diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl index c16413f963d0b..004a884bed1ff 100644 --- a/stdlib/Zstd_jll/src/Zstd_jll.jl +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -8,16 +8,23 @@ using Base, Libdl export libzstd, zstd, zstdmt # These get calculated in __init__() -libzstd_handle::Ptr{Cvoid} = C_NULL +const PATH = Ref("") +const PATH_list = String[] +const LIBPATH = Ref("") +const LIBPATH_list = String[] +artifact_dir::String = "" +libzstd_path::String = "" if Sys.iswindows() - const libzstd = "libzstd-1.dll" + const _libzstd_path = BundledLazyLibraryPath("libzstd-1.dll") elseif Sys.isapple() - const libzstd = "@rpath/libzstd.1.dylib" + const _libzstd_path = BundledLazyLibraryPath("libzstd.1.dylib") else - const libzstd = "libzstd.so.1" + const _libzstd_path = BundledLazyLibraryPath("libzstd.so.1") end +const libzstd = LazyLibrary(_libzstd_path) + if Sys.iswindows() const zstd_exe = "zstd.exe" const zstdmt_exe = "zstdmt.exe" @@ -65,9 +72,21 @@ end zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) +# Function to eagerly dlopen our library and thus resolve all dependencies +function eager_mode() + dlopen(libzstd) +end + +is_available() = true + function __init__() - global libzstd_handle = dlopen(libzstd) - nothing + global libzstd_path = string(_libzstd_path) + global artifact_dir = dirname(Sys.BINDIR) +end + +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) end end # module Zstd_jll diff --git a/stdlib/dSFMT_jll/src/dSFMT_jll.jl b/stdlib/dSFMT_jll/src/dSFMT_jll.jl index b84bf0d8204ae..b7e79c9d587ea 100644 --- a/stdlib/dSFMT_jll/src/dSFMT_jll.jl +++ b/stdlib/dSFMT_jll/src/dSFMT_jll.jl @@ -5,41 +5,41 @@ baremodule dSFMT_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libdSFMT # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libdSFMT_handle::Ptr{Cvoid} = C_NULL libdSFMT_path::String = "" if Sys.iswindows() - const libdSFMT = "libdSFMT.dll" + const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.dll") elseif Sys.isapple() - const libdSFMT = "@rpath/libdSFMT.dylib" + const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.dylib") else - const libdSFMT = "libdSFMT.so" + const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.so") end +const libdSFMT = LazyLibrary(_libdSFMT_path) + +function eager_mode() + dlopen(libdSFMT) +end +is_available() = true + function __init__() - global libdSFMT_handle = dlopen(libdSFMT) - global libdSFMT_path = dlpath(libdSFMT_handle) + global libdSFMT_path = string(_libdSFMT_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libdSFMT_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libdSFMT_path() = libdSFMT_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module dSFMT_jll diff --git a/stdlib/libLLVM_jll/Project.toml b/stdlib/libLLVM_jll/Project.toml index 04280f23f58da..364e4338840eb 100644 --- a/stdlib/libLLVM_jll/Project.toml +++ b/stdlib/libLLVM_jll/Project.toml @@ -4,13 +4,15 @@ version = "20.1.2+1" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [compat] +CompilerSupportLibraries_jll = "1.3.0" Zlib_jll = "1" -Zstd_jll = "1.5" +Zstd_jll = "1.5.7" julia = "1.13" [extras] diff --git a/stdlib/libLLVM_jll/src/libLLVM_jll.jl b/stdlib/libLLVM_jll/src/libLLVM_jll.jl index c7d4a9128f312..c1d89d2ce886f 100644 --- a/stdlib/libLLVM_jll/src/libLLVM_jll.jl +++ b/stdlib/libLLVM_jll/src/libLLVM_jll.jl @@ -3,43 +3,57 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/libLLVM_jll.jl baremodule libLLVM_jll -using Base, Libdl, Zlib_jll, Zstd_jll - -const PATH_list = String[] -const LIBPATH_list = String[] +using Base, Libdl, Zlib_jll, Zstd_jll, CompilerSupportLibraries_jll export libLLVM # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libLLVM_handle::Ptr{Cvoid} = C_NULL libLLVM_path::String = "" if Sys.iswindows() - const libLLVM = "$(Base.libllvm_name).dll" + const _libLLVM_path = BundledLazyLibraryPath("$(Base.libllvm_name).dll") elseif Sys.isapple() - const libLLVM = "@rpath/libLLVM.dylib" + const _libLLVM_path = BundledLazyLibraryPath("libLLVM.dylib") +else + const _libLLVM_path = BundledLazyLibraryPath("$(Base.libllvm_name).so") +end + +if Sys.isapple() + _libLLVM_dependencies = LazyLibrary[libz, libzstd] +elseif Sys.isfreebsd() + _libLLVM_dependencies = LazyLibrary[libz, libzstd, libgcc_s] else - const libLLVM = "$(Base.libllvm_name).so" + _libLLVM_dependencies = LazyLibrary[libz, libzstd, libstdcxx, libgcc_s] end +const libLLVM = LazyLibrary( + _libLLVM_path, + dependencies=_libLLVM_dependencies, +) + +function eager_mode() + CompilerSupportLibraries_jll.eager_mode() + Zlib_jll.eager_mode() + # Zstd_jll.eager_mode() # Not lazy yet + dlopen(libLLVM) +end +is_available() = true + function __init__() - global libLLVM_handle = dlopen(libLLVM) - global libLLVM_path = dlpath(libLLVM_handle) + global libLLVM_path = string(_libLLVM_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libLLVM_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libLLVM_path() = libLLVM_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module libLLVM_jll diff --git a/stdlib/libLLVM_jll/test/runtests.jl b/stdlib/libLLVM_jll/test/runtests.jl index e04076f4145a5..8025c2cb2e693 100644 --- a/stdlib/libLLVM_jll/test/runtests.jl +++ b/stdlib/libLLVM_jll/test/runtests.jl @@ -4,5 +4,5 @@ using Test, Libdl, libLLVM_jll @testset "libLLVM_jll" begin # Try to find a symbol from the C API of libLLVM as a simple sanity check. - @test dlsym(libLLVM_jll.libLLVM_handle, :LLVMContextCreate; throw_error=false) !== nothing + @test dlsym(libLLVM_jll.libLLVM, :LLVMContextCreate; throw_error=false) !== nothing end diff --git a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl index 6cdc1b4ac3ce6..01d222b8a472d 100644 --- a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl +++ b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl @@ -58,4 +58,10 @@ function __init__() LIBPATH[] = dirname(libblastrampoline_path) push!(LIBPATH_list, LIBPATH[]) end + +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end + end # module libblastrampoline_jll diff --git a/stdlib/nghttp2_jll/src/nghttp2_jll.jl b/stdlib/nghttp2_jll/src/nghttp2_jll.jl index 5057299614aa5..3257c9602822b 100644 --- a/stdlib/nghttp2_jll/src/nghttp2_jll.jl +++ b/stdlib/nghttp2_jll/src/nghttp2_jll.jl @@ -4,41 +4,36 @@ baremodule nghttp2_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libnghttp2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libnghttp2_handle::Ptr{Cvoid} = C_NULL libnghttp2_path::String = "" if Sys.iswindows() - const libnghttp2 = "libnghttp2-14.dll" + const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2-14.dll") elseif Sys.isapple() - const libnghttp2 = "@rpath/libnghttp2.14.dylib" + const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2.14.dylib") else - const libnghttp2 = "libnghttp2.so.14" + const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2.so.14") end +const libnghttp2 = LazyLibrary(_libnghttp2_path) + +function eager_mode() + dlopen(libnghttp2) +end +is_available() = true + function __init__() - global libnghttp2_handle = dlopen(libnghttp2) - global libnghttp2_path = dlpath(libnghttp2_handle) + global libnghttp2_path = string(_libnghttp2_path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libnghttp2_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libnghttp2_path() = libnghttp2_path - end # module nghttp2_jll diff --git a/test/choosetests.jl b/test/choosetests.jl index b07f5e8310ee6..c68fee2e09f32 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -22,7 +22,7 @@ const TESTNAMES = [ "euler", "show", "client", "terminfo", "errorshow", "sets", "goto", "llvmcall", "llvmcall2", "ryu", "some", "meta", "stacktraces", "docs", "gc", - "misc", "threads", "stress", "binaryplatforms", "atexit", + "misc", "threads", "stress", "binaryplatforms","stdlib_dependencies", "atexit", "enums", "cmdlineargs", "int", "interpreter", "checked", "bitset", "floatfuncs", "precompile", "relocatedepot", "boundscheck", "error", "ambiguous", "cartesian", "osutils", diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl new file mode 100644 index 0000000000000..46832aa404df1 --- /dev/null +++ b/test/stdlib_dependencies.jl @@ -0,0 +1,199 @@ +using Pkg, Test, Libdl + +# Remove `.X.dylib` or just `.dylib` +function strip_soversion_macos(lib) + m = match(r"^(.*?)(\.\d+)*\.dylib$", lib) + if m !== nothing + return m.captures[1] + end + return lib +end + +# Remove `.so.X` or just `.so` +function strip_soversion_linux(lib) + m = match(r"^(.*?)\.so(\.\d+)*$", lib) + if m !== nothing + return m.captures[1] + end + return lib +end + +# Remove `-X.dll` or just `.dll` +function strip_soversion_windows(lib) + m = match(r"^(.*?)(-\d+)*\.dll$", lib) + if m !== nothing + return m.captures[1] + end + return lib +end + + +function get_deps_otool(lib_path::String) + libs = split(readchomp(`otool -L $(lib_path)`), "\n")[2:end] + # Get rid of `(compatibility version x.y.x)` at the end + libs = first.(split.(libs, (" (compatibility",))) + # Get rid of any `@rpath/` stuff at the beginning + libs = last.(split.(libs, ("@rpath/",))) + + # If there are any absolute paths left, get rid of them here + libs = basename.(strip.(libs)) + + # Now that we've got the basenames of each library, remove `.x.dylib` if it exists: + libs = strip_soversion_macos.(libs) + + # Get rid of any self-referential links + self_lib = strip_soversion_macos(basename(lib_path)) + libs = filter(!=(self_lib), libs) + return libs +end + +function is_system_lib_macos(lib) + system_libs = [ + "libSystem.B", + "libc++", # While we package libstdc++, we do NOT package libc++. + "libiconv", # some things (like git) link against system libiconv + + # macOS frameworks used by things like LibCurl + "CoreFoundation", + "CoreServices", + "Security", + "SystemConfiguration" + ] + return lib ∈ system_libs +end + +function is_system_lib_linux(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "librt", + "libpthread", + "ld-linux-x86-64", + "ld-linux-x86", + "ld-linux-aarch64", + "ld-linux-armhf", + "ld-linux-i386", + ] + return lib ∈ system_libs +end + +function is_system_lib_freebsd(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "libthr", # primary threading library + "libpthread", # alias kept for compatibility + "librt", + "libutil", + "libexecinfo", + "libc++", + "libcxxrt", + ] + return lib ∈ system_libs +end + +function get_deps_readelf(lib_path::String) + # Split into lines + libs = split(readchomp(`readelf -d $(lib_path)`), "\n") + + # Only keep `(NEEDED)` lines + needed_str = Sys.isfreebsd() ? "NEEDED" : "(NEEDED)" + libs = filter(contains(needed_str), libs) + + # Grab the SONAME from "Shared library: [$SONAME]" + libs = map(libs) do lib + m = match(r"Shared library: \[(.*)\]$", lib) + if m !== nothing + return basename(m.captures[1]) + end + return "" + end + libs = filter(!isempty, strip.(libs)) + + # Get rid of soversions in the filenames + libs = strip_soversion_linux.(libs) + return libs +end + + +skip = false +# On linux, we need `readelf` available, otherwise we refuse to attempt this +if Sys.islinux() || Sys.isfreebsd() + if Sys.which("readelf") === nothing + @debug("Silently skipping stdlib_dependencies.jl as `readelf` not available.") + skip = true + end + get_deps = get_deps_readelf + strip_soversion = strip_soversion_linux + is_system_lib = Sys.islinux() ? is_system_lib_linux : is_system_lib_freebsd +elseif Sys.isapple() + # On macOS, we need `otool` available + if Sys.which("otool") === nothing + @debug("Silently skipping stdlib_dependencies.jl as `otool` not available.") + skip = true + end + get_deps = get_deps_otool + strip_soversion = strip_soversion_macos + is_system_lib = is_system_lib_macos +else + @debug("Don't know how to run `stdlib_dependencies.jl` on this platform") + skip = true +end + +if !skip + # Iterate over all JLL stdlibs, check their lazy libraries to ensure + # that they list all valid library dependencies, avoiding a situation + # where the JLL wrapper code has fallen out of sync with the binaries + # themselves. + @testset "Stdlib JLL dependency check" begin + for (_, (stdlib_name, _)) in Pkg.Types.stdlibs() + if !endswith(stdlib_name, "_jll") + continue + end + + # Import the stdlib, skip it if it's not available on this platform + m = eval(Meta.parse("import $(stdlib_name); $(stdlib_name)")) + if !Base.invokelatest(getproperty(m, :is_available)) + continue + end + + for prop_name in names(m) + prop = getproperty(m, prop_name) + if isa(prop, Libdl.LazyLibrary) + lib_path = dlpath(prop) + lazy_lib_deps = strip_soversion.(basename.(dlpath.(prop.dependencies))) + real_lib_deps = filter(!is_system_lib, get_deps(lib_path)) + + # See if there are missing dependencies in the lazy library deps + missing_deps = setdiff(real_lib_deps, lazy_lib_deps) + extraneous_deps = setdiff(lazy_lib_deps, real_lib_deps) + + # We expect there to be no missing or extraneous deps + deps_mismatch = !isempty(missing_deps) || !isempty(extraneous_deps) + + # This is a manually-managed special case + if stdlib_name == "libblastrampoline_jll" && + prop_name == :libblastrampoline && + extraneous_deps == ["libopenblas64_"] + deps_mismatch = false + end + + @test !deps_mismatch + + # Print out the deps mismatch if we find one + if deps_mismatch + @warn("Dependency mismatch", + jll=stdlib_name, + library=string(prop_name), + missing_deps=join(missing_deps, ", "), + extraneous_deps=join(extraneous_deps, ", "), + actual_deps=join(real_lib_deps, ", "), + ) + end + end + end + end + end +end diff --git a/test/trimming/Project.toml b/test/trimming/Project.toml index a0bf6688d9dd4..183536eec9a29 100644 --- a/test/trimming/Project.toml +++ b/test/trimming/Project.toml @@ -1,3 +1,6 @@ [deps] JLLWrappers = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" + +[sources] +Zstd_jll = {path = "Zstd_jll"} diff --git a/test/trimming/Zstd_jll/Project.toml b/test/trimming/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..467516843390a --- /dev/null +++ b/test/trimming/Zstd_jll/Project.toml @@ -0,0 +1,15 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/test/trimming/Zstd_jll/src/Zstd_jll.jl b/test/trimming/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..c16413f963d0b --- /dev/null +++ b/test/trimming/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +libzstd_handle::Ptr{Cvoid} = C_NULL + +if Sys.iswindows() + const libzstd = "libzstd-1.dll" +elseif Sys.isapple() + const libzstd = "@rpath/libzstd.1.dylib" +else + const libzstd = "libzstd.so.1" +end + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) +zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) + +function __init__() + global libzstd_handle = dlopen(libzstd) + nothing +end + +end # module Zstd_jll diff --git a/test/trimming/Zstd_jll/test/runtests.jl b/test/trimming/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/test/trimming/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index 334b5466343d0..659495e3340a6 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -1,5 +1,5 @@ using Libdl -using Zstd_jll +using Zstd_jll # Note this uses the vendored older non-LazyLibrary version of Zstd_jll function @main(args::Vector{String})::Cint println(Core.stdout, "Julia! Hello, world!") From 7a87048e6d2ea3f03ca5f9abbf219219c2e7fdb5 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Wed, 28 May 2025 23:53:33 +0200 Subject: [PATCH 324/662] Restrict arg of sizehint!(::Dict, n) to Integer (#58533) But also allow other types of integers than Int. Closes #58531 (for Dict) --- base/dict.jl | 3 ++- test/dict.jl | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/base/dict.jl b/base/dict.jl index e059fde102bab..68aa4f1a7a543 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -190,7 +190,8 @@ end return h end -function sizehint!(d::Dict{T}, newsz; shrink::Bool=true) where T +function sizehint!(d::Dict{T}, newsz::Integer; shrink::Bool=true) where T + newsz = Int(newsz)::Int oldsz = length(d.slots) # limit new element count to max_values of the key type newsz = min(max(newsz, length(d)), max_values(T)::Int) diff --git a/test/dict.jl b/test/dict.jl index 83d35ae18bb85..b3d69a76420ae 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -294,6 +294,14 @@ end @test eq(Dict{Int,Int}(), Dict{AbstractString,AbstractString}()) end +@testset "sizehint!" begin + d = Dict() + sizehint!(d, UInt(3)) + @test d == Dict() + sizehint!(d, 5) + @test isempty(d) +end + @testset "equality special cases" begin @test Dict(1=>0.0) == Dict(1=>-0.0) @test !isequal(Dict(1=>0.0), Dict(1=>-0.0)) From 3a5e1f15ec2480ad74f59c21d85784d08b542ace Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 29 May 2025 00:37:04 -0400 Subject: [PATCH 325/662] test: Automatically install Revise for `revise-` targets (#58559) This fixes #50256, by automatically installing a private copy of `Revise` into test/deps, similar to the mechanism for Documenter in make.jl. As in #58529, I made sure that this worked with both in-tree and out-of-tree builds. --- doc/Makefile | 2 +- test/Makefile | 19 ++++++++++++------- test/choosetests.jl | 6 +++++- test/runtests.jl | 14 +++++++++++++- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 2b95cc9f96e9b..140b2282e3a3f 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -42,7 +42,7 @@ deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt cp "$<" UnicodeData.txt alldeps: deps - $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps + $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps $(DOCUMENTER_OPTIONS) checksum-unicodedata: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" diff --git a/test/Makefile b/test/Makefile index 69b7ad1451d0f..e4c036b20bafd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -25,21 +25,26 @@ EMBEDDING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/embedding" "CC=$(CC GCEXT_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/gcext" "CC=$(CC)" TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/trimming" "CC=$(CC)" +TEST_JULIA_OPTIONS := --check-bounds=yes --startup-file=no --depwarn=error +TEST_SCRIPT_OPTIONS := --buildroot=$(call cygpath_w,$(BUILDROOT)) default: $(TESTS): @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) + +install-revise-deps: + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise) $(addprefix revise-, $(TESTS)): revise-% : @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) relocatedepot: @rm -rf $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) @mkdir $(SRCDIR)/relocatedepot @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @@ -47,12 +52,12 @@ relocatedepot: @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg4 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) -revise-relocatedepot: revise-% : +revise-relocatedepot: revise-% : dep_revise @rm -rf $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) @mkdir $(SRCDIR)/relocatedepot @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @@ -60,7 +65,7 @@ revise-relocatedepot: revise-% : @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg4 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) embedding: @$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS) diff --git a/test/choosetests.jl b/test/choosetests.jl index c68fee2e09f32..17970313da01f 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -100,6 +100,7 @@ function choosetests(choices = []) seed = rand(RandomDevice(), UInt128) ci_option_passed = false dryrun = false + buildroot = joinpath(@__DIR__, "..") for (i, t) in enumerate(choices) if t == "--skip" @@ -109,6 +110,8 @@ function choosetests(choices = []) exit_on_error = true elseif t == "--revise" use_revise = true + elseif startswith(t, "--buildroot=") + buildroot = t[(length("--buildroot=") + 1):end] elseif startswith(t, "--seed=") seed = parse(UInt128, t[(length("--seed=") + 1):end]) elseif t == "--ci" @@ -124,6 +127,7 @@ function choosetests(choices = []) --help-list : prints the options computed without running them --revise : load Revise --seed= : set the initial seed for all testgroups (parsed as a UInt128) + --buildroot= : set the build root directory (default: in-tree) --skip ... : skip test or collection tagged with TESTS: Can be special tokens, such as "all", "unicode", "stdlib", the names of stdlib \ @@ -261,5 +265,5 @@ function choosetests(choices = []) empty!(tests) end - return (; tests, net_on, exit_on_error, use_revise, seed) + return (; tests, net_on, exit_on_error, use_revise, buildroot, seed) end diff --git a/test/runtests.jl b/test/runtests.jl index dec69cf4f306d..931c1ddb07930 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,7 +13,7 @@ include("choosetests.jl") include("testenv.jl") include("buildkitetestjson.jl") -(; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS) +(; tests, net_on, exit_on_error, use_revise, buildroot, seed) = choosetests(ARGS) tests = unique(tests) if Sys.islinux() @@ -26,8 +26,15 @@ else end if use_revise + # First put this at the top of the DEPOT PATH to install revise if necessary. + # Once it's loaded, we swizzle it to the end, to avoid confusing any tests. + pushfirst!(DEPOT_PATH, joinpath(buildroot, "test", "deps")) + using Pkg + Pkg.activate(joinpath(@__DIR__, "deps")) + Pkg.instantiate() using Revise union!(Revise.stdlib_names, Symbol.(STDLIBS)) + push!(DEPOT_PATH, popfirst!(DEPOT_PATH)) # Remote-eval the following to initialize Revise in workers const revise_init_expr = quote using Revise @@ -37,6 +44,11 @@ if use_revise end end +if isempty(tests) + println("No tests selected. Exiting.") + exit() +end + const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB") parse(Int, ENV["JULIA_TEST_MAXRSS_MB"]) * 2^20 else From b03ef6b680703267e48d56b1e7b2dbbe81e52be2 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Thu, 29 May 2025 08:08:40 -0700 Subject: [PATCH 326/662] Make more types jl_static_show unambiguously (#58512) Makes more types survive `jl_static_show` unambiguously: - Symbols - Symbols printed in the `:var"foo"` form use raw string escaping, fixing `:var"a\b"`, `:var"a\\"`, `:var"$a"`, etc. - Symbols that require parens use parens (`:(=)`, ...) - Signed integers: Except for `Int`, signed integers print like `Int8(1)`. - Floats: floats are printed in a naive but reversible (TODO: double check) way. `Inf(16|32|)` and `NaN(16|32|)` are printed, and `Float16`/`Float32` print the type (`Float32(1.5)`). `Float64`s are printed with a trailing `.0` if it is necessary to disambiguate from `Int`. Fixes #52677, https://github.com/JuliaLang/julia/issues/58484#issuecomment-2902468354, https://github.com/JuliaLang/julia/issues/58484#issuecomment-2898733637, and the specific case mentioned in #58484. Improves the situation for #38902 but does not close it, because a few cases still do not round-trip (inexhaustive list): - Non-canonical NaNs - BFloat16 - User-defined primitive types. This one is tricky, because they can have a size different from any type we have literals for. --- src/rtutils.c | 123 ++++++++++++++++++++++++++++++++++++++++++-------- test/show.jl | 54 +++++++++++++++++++++- 2 files changed, 157 insertions(+), 20 deletions(-) diff --git a/src/rtutils.c b/src/rtutils.c index 2f3bd4e8a0074..4e82caa23af31 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -5,6 +5,8 @@ */ #include "platform.h" +#include +#include #include #include #include @@ -691,12 +693,12 @@ static int is_globfunction(jl_value_t *v, jl_datatype_t *dv, jl_sym_t **globname return 0; } -static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int wrap) JL_NOTSAFEPOINT +static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int wrap, int raw) JL_NOTSAFEPOINT { size_t n = 0; if (wrap) n += jl_printf(out, "\""); - if (!u8_isvalid(str, len)) { + if (!raw && !u8_isvalid(str, len)) { // alternate print algorithm that preserves data if it's not UTF-8 static const char hexdig[] = "0123456789abcdef"; for (size_t i = 0; i < len; i++) { @@ -713,7 +715,11 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int special = 0; for (size_t i = 0; i < len; i++) { uint8_t c = str[i]; - if (c < 32 || c == 0x7f || c == '\\' || c == '"' || c == '$') { + if (raw && ((c == '\\' && i == len-1) || c == '"')) { + special = 1; + break; + } + else if (!raw && (c < 32 || c == 0x7f || c == '\\' || c == '"' || c == '$')) { special = 1; break; } @@ -722,6 +728,25 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, jl_uv_puts(out, str, len); n += len; } + else if (raw) { + // REF: Base.escape_raw_string + int escapes = 0; + for (size_t i = 0; i < len; i++) { + uint8_t c = str[i]; + if (c == '\\') { + escapes++; + } + else { + if (c == '"') + for (escapes++; escapes > 0; escapes--) + n += jl_printf(out, "\\"); + escapes = 0; + } + n += jl_printf(out, "%c", str[i]); + } + for (; escapes > 0; escapes--) + n += jl_printf(out, "\\"); + } else { char buf[512]; size_t i = 0; @@ -737,18 +762,28 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, return n; } +static int jl_is_quoted_sym(const char *sn) +{ + static const char *const quoted_syms[] = {":", "::", ":=", "=", "==", "===", "=>", "`"}; + for (int i = 0; i < sizeof quoted_syms / sizeof *quoted_syms; i++) + if (!strcmp(sn, quoted_syms[i])) + return 1; + return 0; +} + +// TODO: in theory, we need a separate function for showing symbols in an +// expression context (where `Symbol("foo\x01bar")` is ok) and a syntactic +// context (where var"" must be used). static size_t jl_static_show_symbol(JL_STREAM *out, jl_sym_t *name) JL_NOTSAFEPOINT { size_t n = 0; const char *sn = jl_symbol_name(name); - int quoted = !jl_is_identifier(sn) && !jl_is_operator(sn); - if (quoted) { - n += jl_printf(out, "var"); - // TODO: this is not quite right, since repr uses String escaping rules, and Symbol uses raw string rules - n += jl_static_show_string(out, sn, strlen(sn), 1); + if (jl_is_identifier(sn) || (jl_is_operator(sn) && !jl_is_quoted_sym(sn))) { + n += jl_printf(out, "%s", sn); } else { - n += jl_printf(out, "%s", sn); + n += jl_printf(out, "var"); + n += jl_static_show_string(out, sn, strlen(sn), 1, 1); } return n; } @@ -777,6 +812,51 @@ static int jl_static_is_function_(jl_datatype_t *vt) JL_NOTSAFEPOINT { return 0; } +static size_t jl_static_show_float(JL_STREAM *out, double v, + jl_datatype_t *vt) JL_NOTSAFEPOINT +{ + size_t n = 0; + // TODO: non-canonical NaNs do not round-trip + // TOOD: BFloat16 + const char *size_suffix = vt == jl_float16_type ? "16" : + vt == jl_float32_type ? "32" : + ""; + // Requires minimum 1 (sign) + 17 (sig) + 1 (dot) + 5 ("e-123") + 1 (null) + char buf[32]; + // Base B significand digits required to print n base-b significand bits + // (including leading 1): N = 2 + floor(n/log(b, B)) + // Float16 5 + // Float32 9 + // Float64 17 + // REF: https://dl.acm.org/doi/pdf/10.1145/93542.93559 + if (isnan(v)) { + n += jl_printf(out, "NaN%s", size_suffix); + } + else if (isinf(v)) { + n += jl_printf(out, "%sInf%s", v < 0 ? "-" : "", size_suffix); + } + else if (vt == jl_float64_type) { + n += jl_printf(out, "%#.17g", v); + } + else if (vt == jl_float32_type) { + size_t m = snprintf(buf, sizeof buf, "%.9g", v); + // If the exponent was printed, replace it with 'f' + char *p = (char *)memchr(buf, 'e', m); + if (p) + *p = 'f'; + jl_uv_puts(out, buf, m); + n += m; + // If no exponent was printed, we must add one + if (!p) + n += jl_printf(out, "f0"); + } + else { + assert(vt == jl_float16_type); + n += jl_printf(out, "Float16(%#.5g)", v); + } + return n; +} + // `v` might be pointing to a field inlined in a structure therefore // `jl_typeof(v)` may not be the same with `vt` and only `vt` should be // used to determine the type of the value. @@ -957,17 +1037,21 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt int f = *(uint32_t*)jl_data_ptr(v); n += jl_printf(out, "#", f, jl_intrinsic_name(f)); } + else if (vt == jl_long_type) { + // Avoid unnecessary Int64(x)/Int32(x) + n += jl_printf(out, "%" PRIdPTR, *(intptr_t*)v); + } else if (vt == jl_int64_type) { - n += jl_printf(out, "%" PRId64, *(int64_t*)v); + n += jl_printf(out, "Int64(%" PRId64 ")", *(int64_t*)v); } else if (vt == jl_int32_type) { - n += jl_printf(out, "%" PRId32, *(int32_t*)v); + n += jl_printf(out, "Int32(%" PRId32 ")", *(int32_t*)v); } else if (vt == jl_int16_type) { - n += jl_printf(out, "%" PRId16, *(int16_t*)v); + n += jl_printf(out, "Int16(%" PRId16 ")", *(int16_t*)v); } else if (vt == jl_int8_type) { - n += jl_printf(out, "%" PRId8, *(int8_t*)v); + n += jl_printf(out, "Int8(%" PRId8 ")", *(int8_t*)v); } else if (vt == jl_uint64_type) { n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); @@ -988,11 +1072,14 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, "0x%08" PRIx32, *(uint32_t*)v); #endif } + else if (vt == jl_float16_type) { + n += jl_static_show_float(out, julia_half_to_float(*(uint16_t *)v), vt); + } else if (vt == jl_float32_type) { - n += jl_printf(out, "%gf", *(float*)v); + n += jl_static_show_float(out, *(float *)v, vt); } else if (vt == jl_float64_type) { - n += jl_printf(out, "%g", *(double*)v); + n += jl_static_show_float(out, *(double *)v, vt); } else if (vt == jl_bool_type) { n += jl_printf(out, "%s", *(uint8_t*)v ? "true" : "false"); @@ -1004,7 +1091,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, "Core.GlobalMethods"); } else if (vt == jl_string_type) { - n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1); + n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1, 0); } else if (v == jl_bottom_type) { n += jl_printf(out, "Union{}"); @@ -1532,10 +1619,10 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, } jl_printf(str, "\n@ "); if (jl_is_string(file)) { - jl_static_show_string(str, jl_string_data(file), jl_string_len(file), 0); + jl_static_show_string(str, jl_string_data(file), jl_string_len(file), 0, 0); } else if (jl_is_symbol(file)) { - jl_static_show_string(str, jl_symbol_name((jl_sym_t*)file), strlen(jl_symbol_name((jl_sym_t*)file)), 0); + jl_static_show_string(str, jl_symbol_name((jl_sym_t*)file), strlen(jl_symbol_name((jl_sym_t*)file)), 0, 0); } jl_printf(str, ":"); jl_static_show(str, line); diff --git a/test/show.jl b/test/show.jl index 89560d0e908d1..8c9a1b655c047 100644 --- a/test/show.jl +++ b/test/show.jl @@ -703,7 +703,7 @@ let oldout = stdout, olderr = stderr redirect_stderr(olderr) close(wrout) close(wrerr) - @test fetch(out) == "primitive type Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" + @test fetch(out) == "primitive type Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123.0000000000000000\"C\"\n" @test fetch(err) == "TESTA\nTESTB\nΑ1Β2\"A\"\n" finally redirect_stdout(oldout) @@ -1570,8 +1570,58 @@ struct var"%X%" end # Invalid name without '#' typeof(+), var"#f#", typeof(var"#f#"), + + # Integers should round-trip (#52677) + 1, UInt(1), + Int8(1), Int16(1), Int32(1), Int64(1), + UInt8(1), UInt16(1), UInt32(1), UInt64(1), + + # Float round-trip + Float16(1), Float32(1), Float64(1), + Float16(1.5), Float32(1.5), Float64(1.5), + Float16(0.4893243538921085), Float32(0.4893243538921085), Float64(0.4893243538921085), + # Examples that require the full 5, 9, and 17 digits of precision + Float16(0.00010014), Float32(1.00000075f-36), Float64(-1.561051336605761e-182), + floatmax(Float16), floatmax(Float32), floatmax(Float64), + floatmin(Float16), floatmin(Float32), floatmin(Float64), + Float16(0.0), 0.0f0, 0.0, + Float16(-0.0), -0.0f0, -0.0, + Inf16, Inf32, Inf, + -Inf16, -Inf32, -Inf, + nextfloat(Float16(0)), nextfloat(Float32(0)), nextfloat(Float64(0)), + NaN16, NaN32, NaN, + Float16(1e3), 1f7, 1e16, + Float16(-1e3), -1f7, -1e16, + Float16(1e4), 1f8, 1e17, + Float16(-1e4), -1f8, -1e17, + + # :var"" escaping rules differ from strings (#58484) + :foo, + :var"bar baz", + :var"a $b", # No escaping for $ in raw string + :var"a\b", # No escaping for backslashes in middle + :var"a\\", # Backslashes must be escaped at the end + :var"a\\\\", + :var"a\"b", + :var"a\"", + :var"\\\"", + :+, :var"+-", + :(=), :(:), :(::), # Requires quoting + Symbol("a\nb"), + + Val(Float16(1.0)), Val(1f0), Val(1.0), + Val(:abc), Val(:(=)), Val(:var"a\b"), + + Val(1), Val(Int8(1)), Val(Int16(1)), Val(Int32(1)), Val(Int64(1)), Val(Int128(1)), + Val(UInt(1)), Val(UInt8(1)), Val(UInt16(1)), Val(UInt32(1)), Val(UInt64(1)), Val(UInt128(1)), + + # BROKEN + # Symbol("a\xffb"), + # User-defined primitive types + # Non-canonical NaNs + # BFloat16 ) - @test v == eval(Meta.parse(static_shown(v))) + @test v === eval(Meta.parse(static_shown(v))) end end From 8ce50a01e0306279b37c116520f02aaef8d54285 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 May 2025 14:20:36 -0400 Subject: [PATCH 327/662] MethodTable/Cache PR updates (#58565) Missed updates from early designs in #58131. Fix #58557 --- base/methodshow.jl | 4 +- doc/src/devdocs/functions.md | 62 +++++++---------------- doc/src/devdocs/types.md | 27 ++++++---- stdlib/Serialization/src/Serialization.jl | 17 ++----- 4 files changed, 40 insertions(+), 70 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index dc3f564d70db7..dd391a46d170a 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -365,7 +365,7 @@ end show(io::IO, ms::MethodList) = show_method_table(io, ms) show(io::IO, ::MIME"text/plain", ms::MethodList) = show_method_table(io, ms) -show(io::IO, mt::Core.MethodTable) = show_method_table(io, MethodList(mt)) +show(io::IO, mt::Core.MethodTable) = print(io, mt.module, ".", mt.name, " is a Core.MethodTable with ", length(mt), " methods.") function inbase(m::Module) if m == Base @@ -470,8 +470,6 @@ function show(io::IO, mime::MIME"text/html", ms::MethodList) print(io, "
") end -show(io::IO, mime::MIME"text/html", mt::Core.MethodTable) = show(io, mime, MethodList(mt)) - # pretty-printing of AbstractVector{Method} function show(io::IO, mime::MIME"text/plain", mt::AbstractVector{Method}) last_shown_line_infos = get(io, :last_shown_line_infos, nothing) diff --git a/doc/src/devdocs/functions.md b/doc/src/devdocs/functions.md index 51df95ba0bc42..0f414d5df2058 100644 --- a/doc/src/devdocs/functions.md +++ b/doc/src/devdocs/functions.md @@ -6,21 +6,17 @@ This document will explain how functions, method definitions, and method tables ## Method Tables Every function in Julia is a generic function. A generic function is conceptually a single function, -but consists of many definitions, or methods. The methods of a generic function are stored in -a method table. Method tables (type `MethodTable`) are associated with `TypeName`s. A `TypeName` -describes a family of parameterized types. For example `Complex{Float32}` and `Complex{Float64}` -share the same `Complex` type name object. - -All objects in Julia are potentially callable, because every object has a type, which in turn -has a `TypeName`. +but consists of many definitions, or methods. The methods of a generic function are stored in a +method table. There is one global method table (type `MethodTable`) named `Core.GlobalMethods`. Any +default operation on methods (such as calls) uses that table. ## [Function calls](@id Function-calls) -Given the call `f(x, y)`, the following steps are performed: first, the method cache to use is -accessed as `typeof(f).name.mt`. Second, an argument tuple type is formed, `Tuple{typeof(f), typeof(x), typeof(y)}`. -Note that the type of the function itself is the first element. This is because the type might -have parameters, and so needs to take part in dispatch. This tuple type is looked up in the method -table. +Given the call `f(x, y)`, the following steps are performed: First, a tuple type is formed, +`Tuple{typeof(f), typeof(x), typeof(y)}`. Note that the type of the function itself is the first +element. This is because the function itself participates symmetrically in method lookup with the +other arguments. This tuple type is looked up in the global method table. However, the system can +then cache the results, so these steps can be skipped later for similar lookups. This dispatch process is performed by `jl_apply_generic`, which takes two arguments: a pointer to an array of the values `f`, `x`, and `y`, and the number of values (in this case 3). @@ -49,15 +45,6 @@ jl_value_t *jl_call(jl_function_t *f, jl_value_t **args, int32_t nargs); Given the above dispatch process, conceptually all that is needed to add a new method is (1) a tuple type, and (2) code for the body of the method. `jl_method_def` implements this operation. -`jl_method_table_for` is called to extract the relevant method table from what would be -the type of the first argument. This is much more complicated than the corresponding procedure -during dispatch, since the argument tuple type might be abstract. For example, we can define: - -```julia -(::Union{Foo{Int},Foo{Int8}})(x) = 0 -``` - -which works since all possible matching methods would belong to the same method table. ## Creating generic functions @@ -94,9 +81,7 @@ end ## Constructors -A constructor call is just a call to a type. The method table for `Type` contains all -constructor definitions. All subtypes of `Type` (`Type`, `UnionAll`, `Union`, and `DataType`) -currently share a method table via special arrangement. +A constructor call is just a call to a type, to a method defined on `Type{T}`. ## Builtins @@ -128,18 +113,14 @@ import Markdown Markdown.parse ``` -These are all singleton objects whose types are subtypes of `Builtin`, which is a subtype of -`Function`. Their purpose is to expose entry points in the run time that use the "jlcall" calling -convention: +These are mostly singleton objects all of whose types are subtypes of `Builtin`, which is a +subtype of `Function`. Their purpose is to expose entry points in the run time that use the +"jlcall" calling convention: ```c jl_value_t *(jl_value_t*, jl_value_t**, uint32_t) ``` -The method tables of builtins are empty. Instead, they have a single catch-all method cache entry -(`Tuple{Vararg{Any}}`) whose jlcall fptr points to the correct function. This is kind of a hack -but works reasonably well. - ## Keyword arguments Keyword arguments work by adding methods to the kwcall function. This function @@ -228,18 +209,13 @@ sees an argument in the `Function` type hierarchy passed to a slot declared as ` it behaves as if the `@nospecialize` annotation were applied. This heuristic seems to be extremely effective in practice. -The next issue concerns the structure of method cache hash tables. Empirical studies show that -the vast majority of dynamically-dispatched calls involve one or two arguments. In turn, many -of these cases can be resolved by considering only the first argument. (Aside: proponents of single -dispatch would not be surprised by this at all. However, this argument means "multiple dispatch -is easy to optimize in practice", and that we should therefore use it, *not* "we should use single -dispatch"!) So the method cache uses the type of the first argument as its primary key. Note, -however, that this corresponds to the *second* element of the tuple type for a function call (the -first element being the type of the function itself). Typically, type variation in head position -is extremely low -- indeed, the majority of functions belong to singleton types with no parameters. -However, this is not the case for constructors, where a single method table holds constructors -for every type. Therefore the `Type` method table is special-cased to use the *first* tuple type -element instead of the second. +The next issue concerns the structure of method tables. Empirical studies show that the vast +majority of dynamically-dispatched calls involve one or two arguments. In turn, many of these cases +can be resolved by considering only the first argument. (Aside: proponents of single dispatch would +not be surprised by this at all. However, this argument means "multiple dispatch is easy to optimize +in practice", and that we should therefore use it, *not* "we should use single dispatch"!). So the +method table and cache splits up on the structure based on a left-to-right decision tree so allow +efficient nearest-neighbor searches. The front end generates type declarations for all closures. Initially, this was implemented by generating normal type declarations. However, this produced an extremely large number of constructors, diff --git a/doc/src/devdocs/types.md b/doc/src/devdocs/types.md index b63f1c315f457..fc4a93b94ca3c 100644 --- a/doc/src/devdocs/types.md +++ b/doc/src/devdocs/types.md @@ -176,7 +176,12 @@ julia> dump(Array{Int,1}.name) TypeName name: Symbol Array module: Module Core - names: empty SimpleVector + singletonname: Symbol Array + names: SimpleVector + 1: Symbol ref + 2: Symbol size + atomicfields: Ptr{Nothing}(0x0000000000000000) + constfields: Ptr{Nothing}(0x0000000000000000) wrapper: UnionAll var: TypeVar name: Symbol T @@ -188,20 +193,20 @@ TypeName lb: Union{} ub: abstract type Any body: mutable struct Array{T, N} <: DenseArray{T, N} + Typeofwrapper: abstract type Type{Array} <: Any cache: SimpleVector ... - linearcache: SimpleVector ... - - hash: Int64 -7900426068641098781 - mt: MethodTable - name: Symbol Array - defs: Nothing nothing - cache: Nothing nothing - module: Module Core - : Int64 0 - : Int64 0 + hash: Int64 2594190783455944385 + backedges: #undef + partial: #undef + max_args: Int32 0 + n_uninitialized: Int32 0 + flags: UInt8 0x02 + cache_entry_count: UInt8 0x00 + max_methods: UInt8 0x00 + constprop_heuristic: UInt8 0x00 ``` In this case, the relevant field is `wrapper`, which holds a reference to the top-level type used diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 9437fedf649ec..3c4152bf10598 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -469,15 +469,13 @@ end function serialize(s::AbstractSerializer, mt::Core.MethodTable) serialize_type(s, typeof(mt)) - serialize(s, mt.cache) + serialize(s, mt.name) + serialize(s, mt.module) nothing end function serialize(s::AbstractSerializer, mc::Core.MethodCache) - serialize_type(s, typeof(mc)) - serialize(s, mc.name) - serialize(s, mc.module) - nothing + error("cannot serialize MethodCache objects") end @@ -1134,16 +1132,9 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) end function deserialize(s::AbstractSerializer, ::Type{Core.MethodTable}) - mc = deserialize(s)::Core.MethodCache - mc === Core.GlobalMethods.cache && return Core.GlobalMethods - return getglobal(mc.mod, mc.name)::Core.MethodTable -end - -function deserialize(s::AbstractSerializer, ::Type{Core.MethodCache}) name = deserialize(s)::Symbol mod = deserialize(s)::Module - f = Base.unwrap_unionall(getglobal(mod, name)) - return (f::Core.MethodTable).cache + return getglobal(mod, name)::Core.MethodTable end function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance}) From 0cbe4669079249e3d8ff48688b45ed3b39db20d5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 May 2025 14:35:34 -0400 Subject: [PATCH 328/662] remove workaround for controlling terminal behavior in repl_cmd (#58554) This was originally added as a workaround for the behavior of sh to try to become the terminal process group leader, but no longer relevant since #25006 removed that flag again: ``` julia> run(detach(`bash -i -c "sleep 0"`)) bash: cannot set terminal process group (-1): Inappropriate ioctl for device bash: no job control in this shell Process(`bash -i -c 'sleep 0'`, ProcessExited(0)) ``` Long explanation: Julia recieves SIGTTOU when a process like "sh -i -c" exits (at least for bash and zsh, but not dash, ksh, or csh), since "sh -i" sometimes takes over the controlling terminal, causing julia to stop once the bash process exits. This can be quite annoying, but julia goes through some pains (in libuv) to ensure it doesn't steal the controlling terminal from the parent, and apparently bash is not so careful about it. However, since PR #25006, julia hasn't needed this workaround for this issue, so we can now follow the intended behavior when the child is in the same session group and steals the controlling terminal from the parent. Even if such behavior seems odd, this seems to be the intended behavior per posix design implementation that it gets SIGTTOU when julia later tries to change mode (from cooked -> raw after printing the prompt): http://curiousthing.org/sigttin-sigttou-deep-dive-linux. For some background on why this is a useful signal and behavior and should not be just blocked, see nodejs/node#35536. According to glibc, there's a half page of code for how to correctly implement a REPL mode change call: https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html ``` $ ./julia -q shell> bash -i -c "sleep 1" [1]+ Stopped ./julia julia> run(`zsh -i -c "sleep 0"`) Process(`zsh -i -c 'sleep 0'`, ProcessExited(0)) julia> [1]+ Stopped ./julia ``` --- base/client.jl | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/base/client.jl b/base/client.jl index 5bf658bead437..ae99654c233a8 100644 --- a/base/client.jl +++ b/base/client.jl @@ -32,9 +32,6 @@ stackframe_lineinfo_color() = repl_color("JULIA_STACKFRAME_LINEINFO_COLOR", :bol stackframe_function_color() = repl_color("JULIA_STACKFRAME_FUNCTION_COLOR", :bold) function repl_cmd(cmd, out) - shell = shell_split(get(ENV, "JULIA_SHELL", get(ENV, "SHELL", "/bin/sh"))) - shell_name = Base.basename(shell[1]) - # Immediately expand all arguments, so that typing e.g. ~/bin/foo works. cmd.exec .= expanduser.(cmd.exec) @@ -64,19 +61,15 @@ function repl_cmd(cmd, out) cd(dir) println(out, pwd()) else - @static if !Sys.iswindows() - if shell_name == "fish" - shell_escape_cmd = "begin; $(shell_escape_posixly(cmd)); and true; end" - else - shell_escape_cmd = "($(shell_escape_posixly(cmd))) && true" - end + if !Sys.iswindows() + shell = shell_split(get(ENV, "JULIA_SHELL", get(ENV, "SHELL", "/bin/sh"))) + shell_escape_cmd = shell_escape_posixly(cmd) cmd = `$shell -c $shell_escape_cmd` end try run(ignorestatus(cmd)) catch - # Windows doesn't shell out right now (complex issue), so Julia tries to run the program itself - # Julia throws an exception if it can't find the program, but the stack trace isn't useful + # Julia throws an exception if it can't find the cmd (which may be the shell itself), but the stack trace isn't useful lasterr = current_exceptions() lasterr = ExceptionStack([(exception = e[1], backtrace = [] ) for e in lasterr]) invokelatest(display_error, lasterr) From 488be228cd628294b1c3eb25fe54613264b9fb3d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 May 2025 14:53:35 -0400 Subject: [PATCH 329/662] fix Module for Base.jl for Revise.jl (#58549) Just happened to notice in passing that `_included_files[1]` did not get fixed when Base_compiler.jl was split from Base.jl so it causes warnings sometimes in Revise.jl CI --- base/Base.jl | 19 +++++++------------ base/Base_compiler.jl | 4 ++++ base/sysimg.jl | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index afa5a3d93d27c..c0737805de99e 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -23,7 +23,7 @@ include(strcat(BUILDROOT, "version_git.jl")) # include($BUILDROOT/base/version_g # a slightly more verbose fashion than usual, because we're running so early. let os = ccall(:jl_get_UNAME, Any, ()) if os === :Darwin || os === :Apple - if Base.DARWIN_FRAMEWORK + if DARWIN_FRAMEWORK push!(DL_LOAD_PATH, "@loader_path/Frameworks") end push!(DL_LOAD_PATH, "@loader_path") @@ -312,8 +312,8 @@ a_method_to_overwrite_in_test() = inferencebarrier(1) (this::IncludeInto)(mapexpr::Function, fname::AbstractString) = include(mapexpr, this.m, fname) # Compatibility with when Compiler was in Core -@eval Core const Compiler = Main.Base.Compiler -@eval Compiler const fl_parse = Core.Main.Base.fl_parse +@eval Core const Compiler = $Base.Compiler +@eval Compiler const fl_parse = $Base.fl_parse # External libraries vendored into Base Core.println("JuliaSyntax/src/JuliaSyntax.jl") @@ -329,13 +329,13 @@ if is_primary_base_module # Profiling helper # triggers printing the report and (optionally) saving a heap snapshot after a SIGINFO/SIGUSR1 profile request # Needs to be in Base because Profile is no longer loaded on boot -function profile_printing_listener(cond::Base.AsyncCondition) +function profile_printing_listener(cond::AsyncCondition) profile = nothing try while _trywait(cond) profile = @something(profile, require_stdlib(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module invokelatest(profile.peek_report[]) - if Base.get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true + if get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true println(stderr, "Saving heap snapshot...") fname = invokelatest(profile.take_heap_snapshot) println(stderr, "Heap snapshot saved to `$(fname)`") @@ -350,8 +350,8 @@ function profile_printing_listener(cond::Base.AsyncCondition) end function start_profile_listener() - cond = Base.AsyncCondition() - Base.uv_unref(cond.handle) + cond = AsyncCondition() + uv_unref(cond.handle) t = errormonitor(Threads.@spawn(profile_printing_listener(cond))) atexit() do # destroy this callback when exiting @@ -411,7 +411,6 @@ end const _compiler_require_dependencies = Any[] @Core.latestworld for i = 1:length(_included_files) - isassigned(_included_files, i) || continue (mod, file) = _included_files[i] if mod === Compiler || parentmodule(mod) === Compiler || endswith(file, "/Compiler.jl") _include_dependency!(_compiler_require_dependencies, true, mod, file, true, false) @@ -427,7 +426,3 @@ end @assert length(_compiler_require_dependencies) >= 15 end - -# Ensure this file is also tracked -@assert !isassigned(_included_files, 1) -_included_files[1] = (parentmodule(Base), abspath(@__FILE__)) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 91e765bac8a13..ef448a02a15e9 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -389,5 +389,9 @@ Core._setlowerer!(fl_lower) # Further definition of Base will happen in Base.jl if loaded. +# Ensure this file is also tracked +@assert !isassigned(_included_files, 1) +_included_files[1] = (@__MODULE__, ccall(:jl_prepend_cwd, Any, (Any,), "Base_compiler.jl")) + end # module Base using .Base diff --git a/base/sysimg.jl b/base/sysimg.jl index f5354c6ebea1f..7e205ca955409 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -Base.Core.include(Base, "Base.jl") # finish populating Base (currently just has the Compiler) +Base.include("Base.jl") # finish populating Base (currently just has the Compiler) # Set up Main module by importing from Base using .Base From 965d007479986e1c5f8c7f0e0b0fe17466ee4643 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 May 2025 14:53:54 -0400 Subject: [PATCH 330/662] fix breakage with `jl_get_global` (#58540) Introduce a new `jl_get_global_value` to do the new world-aware behavior, while preserving the old behavior for `jl_get_global`. Choose between `jl_get_global`, `jl_get_global_value`, and `jl_eval_global_var`, depending on what behavior is required. Also take this opportunity to fix some data race mistakes introduced by bindings (relaxed loads of jl_world_counter outside of assert) and lacking type asserts / unnecessary globals in precompile code. Fix #58097 Addresses post-review comment https://github.com/JuliaLang/julia/pull/57213#discussion_r1939814835, so this is already tested against by existing logic --- base/loading.jl | 8 ++-- src/ast.c | 5 +-- src/gf.c | 7 ++-- src/init.c | 14 +++---- src/interpreter.c | 4 +- src/jl_uv.c | 4 +- src/jlapi.c | 4 +- src/julia.h | 8 +--- src/module.c | 41 ++++++++++++-------- src/precompile.c | 35 +++++++++-------- src/rtutils.c | 10 +++-- src/scheduler.c | 9 +++-- src/staticdata_utils.c | 85 +++++++++++++++++++++--------------------- src/task.c | 2 +- src/threading.c | 6 ++- src/toplevel.c | 19 ++++------ sysimage.mk | 10 ++--- 17 files changed, 142 insertions(+), 129 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index d8b7d2156cf30..cbe69411fe9d1 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2218,7 +2218,7 @@ const include_callbacks = Any[] # used to optionally track dependencies when requiring a module: const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them -const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled +const _require_dependencies = Any[] # a list of (mod::Module, abspath::String, fsize::UInt64, hash::UInt32, mtime::Float64) tuples that are the file dependencies of the module currently being precompiled const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies function _include_dependency(mod::Module, _path::AbstractString; track_content::Bool=true, @@ -2244,9 +2244,9 @@ function _include_dependency!(dep_list::Vector{Any}, track_dependencies::Bool, else @lock require_lock begin if track_content - hash = isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r") + hash = (isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r"))::UInt32 # use mtime=-1.0 here so that fsize==0 && mtime==0.0 corresponds to a missing include_dependency - push!(dep_list, (mod, path, filesize(path), hash, -1.0)) + push!(dep_list, (mod, path, UInt64(filesize(path)), hash, -1.0)) else push!(dep_list, (mod, path, UInt64(0), UInt32(0), mtime(path))) end @@ -3346,7 +3346,7 @@ mutable struct CacheHeaderIncludes const modpath::Vector{String} # seemingly not needed in Base, but used by Revise end -function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, Int64, UInt32, Float64}) +function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, UInt64, UInt32, Float64}) return CacheHeaderIncludes(PkgId(dep_tuple[1]), dep_tuple[2:end]..., String[]) end diff --git a/src/ast.c b/src/ast.c index bb6faff942b3f..8f36cd06decad 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1327,9 +1327,8 @@ JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *filename, int line, size_t world, bool_t warn) { jl_value_t *core_lower = NULL; - if (jl_core_module) { - core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); - } + if (jl_core_module) + core_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower")); if (!core_lower || core_lower == jl_nothing) { return jl_fl_lower(expr, inmodule, filename, line, world, warn); } diff --git a/src/gf.c b/src/gf.c index 4bfe910ac6cae..2fdb10b4f67ff 100644 --- a/src/gf.c +++ b/src/gf.c @@ -512,12 +512,13 @@ JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t w ct->world_age = jl_typeinf_world; jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 4); - jl_module_t *CC = (jl_module_t*)jl_get_global(jl_core_module, jl_symbol("Compiler")); + jl_module_t *CC = (jl_module_t*)jl_get_global_value(jl_core_module, jl_symbol("Compiler")); if (CC != NULL && jl_is_module(CC)) { - fargs[0] = jl_get_global(CC, jl_symbol("NativeInterpreter"));; + JL_GC_PROMISE_ROOTED(CC); + fargs[0] = jl_get_global_value(CC, jl_symbol("NativeInterpreter"));; fargs[1] = jl_box_ulong(world); fargs[1] = jl_apply(fargs, 2); - fargs[0] = jl_get_global(CC, jl_symbol("typeinf_code")); + fargs[0] = jl_get_global_value(CC, jl_symbol("typeinf_code")); fargs[2] = (jl_value_t*)mi; fargs[3] = jl_true; ci = (jl_code_info_t*)jl_apply(fargs, 4); diff --git a/src/init.c b/src/init.c index c3a20443b61f3..f811cf76748fb 100644 --- a/src/init.c +++ b/src/init.c @@ -249,7 +249,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) JL_NOTSAFEPOINT_ENTER if (jl_base_module) { size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_atexit")); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_atexit")); if (f != NULL) { jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 2); @@ -355,13 +355,14 @@ JL_DLLEXPORT void jl_postoutput_hook(void) if (jl_base_module) { jl_task_t *ct = jl_get_current_task(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_postoutput")); + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_postoutput")); if (f != NULL) { JL_TRY { - size_t last_age = ct->world_age; - ct->world_age = jl_get_world_counter(); + JL_GC_PUSH1(&f); jl_apply(&f, 1); - ct->world_age = last_age; + JL_GC_POP(); } JL_CATCH { jl_printf((JL_STREAM*)STDERR_FILENO, "\npostoutput hook threw an error: "); @@ -370,6 +371,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) jlbacktrace(); // written to STDERR_FILENO } } + ct->world_age = last_age; } return; } @@ -599,7 +601,6 @@ static NOINLINE void _finish_jl_init_(jl_image_buf_t sysimage, jl_ptls_t ptls, j jl_init_primitives(); jl_init_main_module(); jl_load(jl_core_module, "boot.jl"); - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); post_boot_hooks(); } @@ -612,7 +613,6 @@ static NOINLINE void _finish_jl_init_(jl_image_buf_t sysimage, jl_ptls_t ptls, j jl_n_threads_per_pool[JL_THREADPOOL_ID_INTERACTIVE] = 0; jl_n_threads_per_pool[JL_THREADPOOL_ID_DEFAULT] = 1; } else { - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); post_image_load_hooks(); } jl_start_threads(); diff --git a/src/interpreter.c b/src/interpreter.c index 97c37cb99b9c1..aa89f7385532f 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -161,14 +161,16 @@ static jl_value_t *do_invoke(jl_value_t **args, size_t nargs, interpreter_state return result; } +// get the global (throwing if null) in the current world jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e) { - jl_value_t *v = jl_get_global(m, e); + jl_value_t *v = jl_get_global_value(m, e); if (v == NULL) jl_undefined_var_error(e, (jl_value_t*)m); return v; } +// get the global (throwing if null) in the current world, optimized jl_value_t *jl_eval_globalref(jl_globalref_t *g) { jl_value_t *v = jl_get_globalref_value(g); diff --git a/src/jl_uv.c b/src/jl_uv.c index 3498952622dce..005d1ea727655 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -160,10 +160,10 @@ static void jl_uv_call_close_callback(jl_value_t *val) { jl_value_t **args; JL_GC_PUSHARGS(args, 2); // val is "rooted" in the finalizer list only right now - args[0] = jl_get_global(jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), + args[0] = jl_eval_global_var( + jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), jl_symbol("_uv_hook_close")); // topmod(typeof(val))._uv_hook_close args[1] = val; - assert(args[0]); jl_apply(args, 2); // TODO: wrap in try-catch? JL_GC_POP(); } diff --git a/src/jlapi.c b/src/jlapi.c index b27c01d9e0f1a..e127fd1367816 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -955,12 +955,14 @@ static NOINLINE int true_main(int argc, char *argv[]) ct->world_age = jl_get_world_counter(); jl_function_t *start_client = jl_base_module ? - (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("_start")) : NULL; + (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("_start")) : NULL; if (start_client) { int ret = 1; JL_TRY { + JL_GC_PUSH1(&start_client); jl_value_t *r = jl_apply(&start_client, 1); + JL_GC_POP(); if (jl_typeof(r) != (jl_value_t*)jl_int32_type) jl_type_error("typeassert", (jl_value_t*)jl_int32_type, r); ret = jl_unbox_int32(r); diff --git a/src/julia.h b/src/julia.h index 70eb654d4b1f3..4a4a4eff81232 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2096,6 +2096,7 @@ JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); +JL_DLLEXPORT jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported); @@ -2689,12 +2690,7 @@ JL_DLLEXPORT jl_task_t *jl_get_current_task(void) JL_GLOBALLY_ROOTED JL_NOTSAFEP STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) { - jl_task_t *ct = jl_get_current_task(); - size_t last_world = ct->world_age; - ct->world_age = jl_get_world_counter(); - jl_value_t *r = jl_get_global(m, jl_symbol(name)); - ct->world_age = last_world; - return (jl_function_t*)r; + return (jl_function_t*)jl_get_global(m, jl_symbol(name)); } // TODO: we need to pin the task while using this (set pure bit) diff --git a/src/module.c b/src/module.c index 9c1a93d08373a..f154b7d29db8e 100644 --- a/src/module.c +++ b/src/module.c @@ -366,7 +366,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart) { - struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_relaxed(&jl_world_counter), 0); + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_acquire(&jl_world_counter), 0); jl_atomic_store_relaxed(&bpart->min_world, resolution.min_world); jl_atomic_store_relaxed(&bpart->max_world, resolution.max_world); bpart->restriction = resolution.binding_or_const; @@ -831,11 +831,12 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, // return module of binding JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var) { + size_t world = jl_current_task->world_age; jl_binding_t *b = jl_get_module_binding(m, var, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); if (jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT_CONST) { - struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_current_task->world_age, 0); + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, world, 0); if (!resolution.debug_only_ultimate_binding) jl_error("Constant binding was imported from multiple modules"); b = resolution.debug_only_ultimate_binding; @@ -885,16 +886,17 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b, size_t w return jl_atomic_load_relaxed(&b->value); } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b) +static jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b, size_t world) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); if (jl_options.depwarn) { int needs_depwarn = 0; - jl_walk_binding_inplace_depwarn(&b, &bpart, jl_current_task->world_age, &needs_depwarn); + jl_walk_binding_inplace_depwarn(&b, &bpart, world, &needs_depwarn); if (needs_depwarn) jl_binding_deprecation_warning(b); - } else { - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + } + else { + jl_walk_binding_inplace(&b, &bpart, world); } enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -907,11 +909,11 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b) return jl_atomic_load_relaxed(&b->value); } - JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + size_t world = jl_current_task->world_age; + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; @@ -926,7 +928,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_const(jl_binding_t *b) { // See note below. Note that this is for some deprecated uses, and should not be added to new code. - size_t world = jl_atomic_load_relaxed(&jl_world_counter); + size_t world = jl_atomic_load_acquire(&jl_world_counter); jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); @@ -1538,18 +1540,27 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, } +// get the value (or null) in the current world JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr) { jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - return jl_get_binding_value_depwarn(b); + return jl_get_binding_value_depwarn(b, jl_current_task->world_age); +} + +// get the value (or null) in the current world +JL_DLLEXPORT jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var) +{ + jl_binding_t *b = jl_get_module_binding(m, var, 1); + return jl_get_binding_value_depwarn(b, jl_current_task->world_age); } +// get the global (or null) in the latest world JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 1); - return jl_get_binding_value_depwarn(b); + return jl_get_binding_value_depwarn(b, jl_atomic_load_acquire(&jl_world_counter)); } JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT) diff --git a/src/precompile.c b/src/precompile.c index c21cf5367fba6..33b71d4605b30 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -36,27 +36,30 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { // char*: src text // At the end we write int32(0) as a terminal sentinel. size_t len = jl_array_nrows(udeps); - static jl_value_t *replace_depot_func = NULL; - if (!replace_depot_func) - replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); - static jl_value_t *normalize_depots_func = NULL; - if (!normalize_depots_func) - normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); ios_t srctext; - jl_value_t *deptuple = NULL, *depots = NULL; - JL_GC_PUSH3(&deptuple, &udeps, &depots); + jl_value_t *replace_depot_func = NULL; + jl_value_t *normalize_depots_func = NULL; + jl_value_t *deptuple = NULL; + jl_value_t *depots = NULL; jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + JL_GC_PUSH4(&deptuple, &depots, &replace_depot_func, &normalize_depots_func); + replace_depot_func = jl_eval_global_var(jl_base_module, jl_symbol("replace_depot_path")); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation")); depots = jl_apply(&normalize_depots_func, 1); - ct->world_age = last_age; + jl_datatype_t *deptuple_p[5] = {jl_module_type, jl_string_type, jl_uint64_type, jl_uint32_type, jl_float64_type}; + jl_value_t *jl_deptuple_type = jl_apply_tuple_type_v((jl_value_t**)deptuple_p, 5); + JL_GC_PROMISE_ROOTED(jl_deptuple_type); +#define jl_is_deptuple(v) (jl_typeis((v), jl_deptuple_type)) for (size_t i = 0; i < len; i++) { deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *depmod = jl_fieldref(deptuple, 0); // module + jl_value_t *depmod = jl_fieldref_noalloc(deptuple, 0); // module // Dependencies declared with `include_dependency` are excluded // because these may not be Julia code (and could be huge) + JL_TYPECHK(write_srctext, deptuple, deptuple); if (depmod != (jl_value_t*)jl_main_module) { - jl_value_t *abspath = jl_fieldref(deptuple, 1); // file abspath + jl_value_t *abspath = jl_fieldref_noalloc(deptuple, 1); // file abspath const char *abspathstr = jl_string_data(abspath); if (!abspathstr[0]) continue; @@ -67,17 +70,11 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { continue; } - jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 3); + jl_value_t *replace_depot_args[3]; replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = abspath; replace_depot_args[2] = depots; - jl_task_t *ct = jl_current_task; - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 3); - ct->world_age = last_age; - JL_GC_POP(); size_t slen = jl_string_len(depalias); write_int32(f, slen); @@ -91,6 +88,8 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { ios_seek_end(f); } } + ct->world_age = last_age; +#undef jl_is_deptuple JL_GC_POP(); } write_int32(f, 0); // mark the end of the source text diff --git a/src/rtutils.c b/src/rtutils.c index 4e82caa23af31..1f341ee60c972 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -1596,15 +1596,17 @@ JL_DLLEXPORT void jl_test_failure_breakpoint(jl_value_t *v) // logging tools -------------------------------------------------------------- +// DO NOT USE THIS FUNCTION FOR NEW CODE +// The internal should not be doing anything that requires logging, which means most functions would trigger UB if calling this void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, jl_value_t *file, jl_value_t *line, jl_value_t *kwargs, jl_value_t *msg) { - static jl_value_t *logmsg_func = NULL; - if (!logmsg_func && jl_base_module) { - jl_value_t *corelogging = jl_get_global(jl_base_module, jl_symbol("CoreLogging")); + jl_value_t *logmsg_func = NULL; + if (jl_base_module) { + jl_value_t *corelogging = jl_get_global_value(jl_base_module, jl_symbol("CoreLogging")); if (corelogging && jl_is_module(corelogging)) { - logmsg_func = jl_get_global((jl_module_t*)corelogging, jl_symbol("logmsg_shim")); + logmsg_func = jl_get_global_value((jl_module_t*)corelogging, jl_symbol("logmsg_shim")); } } if (!logmsg_func) { diff --git a/src/scheduler.c b/src/scheduler.c index 731a0c5146605..8e0202cc7a980 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -329,12 +329,15 @@ void jl_task_wait_empty(void) jl_task_t *ct = jl_current_task; if (jl_atomic_load_relaxed(&ct->tid) == 0 && jl_base_module) { jl_wait_empty_begin(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("wait")); - wait_empty = ct; size_t lastage = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - if (f) + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("wait")); + wait_empty = ct; + if (f) { + JL_GC_PUSH1(&f); jl_apply_generic(f, NULL, 0); + JL_GC_POP(); + } // we are back from jl_task_get_next now ct->world_age = lastage; wait_empty = NULL; diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index d699e0c262d26..3f05de189c3ef 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -532,35 +532,40 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t { int64_t initial_pos = 0; int64_t pos = 0; - static jl_array_t *deps = NULL; - if (!deps) - deps = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("_require_dependencies")); - - // unique(deps) to eliminate duplicates while preserving order: - // we preserve order so that the topmost included .jl file comes first - static jl_value_t *unique_func = NULL; - if (!unique_func) - unique_func = jl_get_global(jl_base_module, jl_symbol("unique")); - jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)deps}; jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_array_t *udeps = (*udepsp = deps && unique_func ? (jl_array_t*)jl_apply(uniqargs, 2) : NULL); - ct->world_age = last_age; + jl_value_t *depots = NULL, *prefs_hash = NULL, *prefs_list = NULL; + jl_value_t *unique_func = NULL; + jl_value_t *replace_depot_func = NULL; + jl_value_t *normalize_depots_func = NULL; + jl_value_t *toplevel = NULL; + jl_value_t *prefs_hash_func = NULL; + jl_value_t *get_compiletime_prefs_func = NULL; + JL_GC_PUSH8(&depots, &prefs_list, &unique_func, &replace_depot_func, &normalize_depots_func, &toplevel, &prefs_hash_func, &get_compiletime_prefs_func); + + jl_array_t *udeps = (jl_array_t*)jl_get_global_value(jl_base_module, jl_symbol("_require_dependencies")); + *udepsp = udeps; + + // unique(udeps) to eliminate duplicates while preserving order: + // we preserve order so that the topmost included .jl file comes first + if (udeps) { + unique_func = jl_eval_global_var(jl_base_module, jl_symbol("unique")); + jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)udeps}; + udeps = (jl_array_t*)jl_apply(uniqargs, 2); + *udepsp = udeps; + JL_TYPECHK(write_dependency_list, array_any, (jl_value_t*)udeps); + } - static jl_value_t *replace_depot_func = NULL; - if (!replace_depot_func) - replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); - static jl_value_t *normalize_depots_func = NULL; - if (!normalize_depots_func) - normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); + replace_depot_func = jl_get_global_value(jl_base_module, jl_symbol("replace_depot_path")); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation")); - jl_value_t *depots = NULL, *prefs_hash = NULL, *prefs_list = NULL; - JL_GC_PUSH2(&depots, &prefs_list); - last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); depots = jl_apply(&normalize_depots_func, 1); - ct->world_age = last_age; + + jl_datatype_t *deptuple_p[5] = {jl_module_type, jl_string_type, jl_uint64_type, jl_uint32_type, jl_float64_type}; + jl_value_t *jl_deptuple_type = jl_apply_tuple_type_v((jl_value_t**)deptuple_p, 5); + JL_GC_PROMISE_ROOTED(jl_deptuple_type); +#define jl_is_deptuple(v) (jl_typeis((v), jl_deptuple_type)) // write a placeholder for total size so that we can quickly seek past all of the // dependencies if we don't need them @@ -569,20 +574,16 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t size_t i, l = udeps ? jl_array_nrows(udeps) : 0; for (i = 0; i < l; i++) { jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *deppath = jl_fieldref(deptuple, 1); + JL_TYPECHK(write_dependency_list, deptuple, deptuple); + jl_value_t *deppath = jl_fieldref_noalloc(deptuple, 1); if (replace_depot_func) { - jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 3); + jl_value_t *replace_depot_args[3]; replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = deppath; replace_depot_args[2] = depots; - ct = jl_current_task; - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); deppath = (jl_value_t*)jl_apply(replace_depot_args, 3); - ct->world_age = last_age; - JL_GC_POP(); + JL_TYPECHK(write_dependency_list, string, deppath); } size_t slen = jl_string_len(deppath); @@ -591,7 +592,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_uint64(s, jl_unbox_uint64(jl_fieldref(deptuple, 2))); // fsize write_uint32(s, jl_unbox_uint32(jl_fieldref(deptuple, 3))); // hash write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 4))); // mtime - jl_module_t *depmod = (jl_module_t*)jl_fieldref(deptuple, 0); // evaluating module + jl_module_t *depmod = (jl_module_t*)jl_fieldref_noalloc(deptuple, 0); // evaluating module jl_module_t *depmod_top = depmod; while (!is_serialization_root_module(depmod_top)) depmod_top = depmod_top->parent; @@ -615,34 +616,31 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // Calculate Preferences hash for current package. if (jl_base_module) { // Toplevel module is the module we're currently compiling, use it to get our preferences hash - jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__")); - jl_value_t * prefs_hash_func = jl_get_global(jl_base_module, jl_symbol("get_preferences_hash")); - jl_value_t * get_compiletime_prefs_func = jl_get_global(jl_base_module, jl_symbol("get_compiletime_preferences")); - - if (toplevel && prefs_hash_func && get_compiletime_prefs_func) { - // Temporary invoke in newest world age - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + toplevel = jl_get_global_value(jl_base_module, jl_symbol("__toplevel__")); + prefs_hash_func = jl_eval_global_var(jl_base_module, jl_symbol("get_preferences_hash")); + get_compiletime_prefs_func = jl_eval_global_var(jl_base_module, jl_symbol("get_compiletime_preferences")); + if (toplevel) { // call get_compiletime_prefs(__toplevel__) jl_value_t *args[3] = {get_compiletime_prefs_func, (jl_value_t*)toplevel, NULL}; prefs_list = (jl_value_t*)jl_apply(args, 2); + JL_TYPECHK(write_dependency_list, array, prefs_list); // Call get_preferences_hash(__toplevel__, prefs_list) args[0] = prefs_hash_func; args[2] = prefs_list; prefs_hash = (jl_value_t*)jl_apply(args, 3); - - // Reset world age to normal - ct->world_age = last_age; + JL_TYPECHK(write_dependency_list, uint64, prefs_hash); } } + ct->world_age = last_age; // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file. if (prefs_hash != NULL && prefs_list != NULL) { size_t i, l = jl_array_nrows(prefs_list); for (i = 0; i < l; i++) { jl_value_t *pref_name = jl_array_ptr_ref(prefs_list, i); + JL_TYPECHK(write_dependency_list, string, pref_name); size_t slen = jl_string_len(pref_name); write_int32(s, slen); ios_write(s, jl_string_data(pref_name), slen); @@ -660,6 +658,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_uint64(s, 0); } JL_GC_POP(); // for depots, prefs_list +#undef jl_is_deptuple // write a dummy file position to indicate the beginning of the source-text pos = ios_pos(s); diff --git a/src/task.c b/src/task.c index ae36c98066d01..5804be8bff1a9 100644 --- a/src/task.c +++ b/src/task.c @@ -335,7 +335,7 @@ void JL_NORETURN jl_finish_task(jl_task_t *ct) // let the runtime know this task is dead and find a new task to run jl_function_t *done = jl_atomic_load_relaxed(&task_done_hook_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("task_done_hook")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("task_done_hook")); if (done != NULL) jl_atomic_store_release(&task_done_hook_func, done); } diff --git a/src/threading.c b/src/threading.c index 97b66d2e2068f..37e5ac3b856a3 100644 --- a/src/threading.c +++ b/src/threading.c @@ -409,9 +409,11 @@ static _Atomic(jl_function_t*) init_task_lock_func JL_GLOBALLY_ROOTED = NULL; static void jl_init_task_lock(jl_task_t *ct) { + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); jl_function_t *done = jl_atomic_load_relaxed(&init_task_lock_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("init_task_lock")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("init_task_lock")); if (done != NULL) jl_atomic_store_release(&init_task_lock_func, done); } @@ -424,6 +426,7 @@ static void jl_init_task_lock(jl_task_t *ct) jl_no_exc_handler(jl_current_exception(ct), ct); } } + ct->world_age = last_age; } JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void) @@ -446,7 +449,6 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void) JL_GC_PROMISE_ROOTED(ct); uv_random(NULL, NULL, &ct->rngState, sizeof(ct->rngState), 0, NULL); jl_atomic_fetch_add(&jl_gc_disable_counter, -1); - ct->world_age = jl_get_world_counter(); // root_task sets world_age to 1 jl_init_task_lock(ct); return &ct->gcstack; } diff --git a/src/toplevel.c b/src/toplevel.c index 739eb8024ece0..7b20c5df19b12 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -54,12 +54,6 @@ void jl_init_main_module(void) jl_set_initial_const(jl_main_module, jl_symbol("Core"), (jl_value_t*)jl_core_module, 0); // const Core.Main = Main } -static jl_function_t *jl_module_get_initializer(jl_module_t *m JL_PROPAGATES_ROOT) -{ - return (jl_function_t*)jl_get_global(m, jl_symbol("__init__")); -} - - void jl_module_run_initializer(jl_module_t *m) { JL_TIMING(INIT_MODULE, INIT_MODULE); @@ -68,9 +62,12 @@ void jl_module_run_initializer(jl_module_t *m) size_t last_age = ct->world_age; JL_TRY { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_function_t *f = jl_module_get_initializer(m); - if (f != NULL) + jl_value_t *f = jl_get_global_value(m, jl_symbol("__init__")); + if (f != NULL) { + JL_GC_PUSH1(&f); jl_apply(&f, 1); + JL_GC_POP(); + } ct->world_age = last_age; } JL_CATCH { @@ -110,7 +107,7 @@ jl_array_t *jl_get_loaded_modules(void) static int jl_is__toplevel__mod(jl_module_t *mod) { return jl_base_module && - (jl_value_t*)mod == jl_get_global(jl_base_module, jl_symbol("__toplevel__")); + (jl_value_t*)mod == jl_get_global_value(jl_base_module, jl_symbol("__toplevel__")); } // TODO: add locks around global state mutation operations @@ -829,7 +826,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex) jl_filename = "none"; size_t last_age = ct->world_age; JL_TRY { - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); v = jl_toplevel_eval(m, ex); } JL_CATCH { @@ -889,7 +886,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, continue; } expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0); - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } ct->world_age = last_age; diff --git a/sysimage.mk b/sysimage.mk index 987ca4035870e..6dbc1f2abf3e5 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -71,7 +71,7 @@ RELDATADIR := $(call rel_path,$(JULIAHOME)/base,$(build_datarootdir))/ # <-- mak $(build_private_libdir)/basecompiler.ji: $(COMPILER_SRCS) @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ JULIA_NUM_THREADS=1 $(call spawn,$(JULIA_EXECUTABLE)) $(HEAPLIM) --output-ji $(call cygpath_w,$@).tmp \ - --startup-file=no --warn-overwrite=yes -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) + --startup-file=no --warn-overwrite=yes --depwarn=error -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) @mv $@.tmp $@ define base_builder @@ -80,7 +80,7 @@ $$(build_private_libdir)/basecompiler$1-o.a $$(build_private_libdir)/basecompile WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp \ - --startup-file=no --warn-overwrite=yes -g$$(BOOTSTRAP_DEBUG_LEVEL) Base_compiler.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR)) + --startup-file=no --warn-overwrite=yes --depwarn=error -g$$(BOOTSTRAP_DEBUG_LEVEL) Base_compiler.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR)) @mv $$@.tmp $$@ $$(build_private_libdir)/sysbase$1.ji: $$(build_private_libdir)/basecompiler$1.$$(SHLIB_EXT) $$(JULIAHOME)/VERSION $$(BASE_SRCS) $$(STDLIB_SRCS) @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ @@ -88,7 +88,7 @@ $$(build_private_libdir)/sysbase$1.ji: $$(build_private_libdir)/basecompiler$1.$ WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $$(JULIA_EXECUTABLE)) -g1 $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-ji $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ echo '*** This error might be fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) @@ -103,7 +103,7 @@ $$(build_private_libdir)/sysbase$1-o.a $$(build_private_libdir)/sysbase$1-bc.a : WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $$(JULIA_EXECUTABLE)) -g1 $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ echo '*** This error might be fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) @@ -117,7 +117,7 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil JULIA_DEPOT_PATH=':' \ JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) From 5a0eb96a9e51e2b67e98fda27edb2b768e6135c2 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Thu, 29 May 2025 20:35:54 -0500 Subject: [PATCH 331/662] CI: `PrAssignee.yml`: Don't retry on 403 (#58555) As noted in https://github.com/JuliaLang/julia/pull/58550#discussion_r2112748035 --------- Co-authored-by: Dilum Aluthge --- .github/workflows/PrAssignee.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 728dcbb902cfa..62d8e192a59bb 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -38,11 +38,11 @@ jobs: uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: retries: 5 # retry GitHub API requests up to 5 times, with exponential backoff - retry-exempt-status-codes: "403,404" - # In the above list: - # - # - 404 is in the list because we will always hit a 404 when the PR author is not - # a committer. This 404 is normal and expected. + retry-exempt-status-codes: 404 + # Don't retry 404 because we will hit a 404 when the PR author is a committer. + # This 404 is normal and expected. + # Do retry 400 and other 4xx errors because github sometimes (erroneously) + # returns a 4xx error code due to server errors. script: | const oldPrAssignees = context.payload.pull_request.assignees .map(obj => obj.login) From e302ed3558316700c74936cb8efd65cdce442383 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 29 May 2025 21:38:31 -0400 Subject: [PATCH 332/662] Check in Manifest for test-revise (#58562) This was supposed to be in #58559, but I accidentally forgot to check it in. --- test/deps/Manifest.toml | 144 ++++++++++++++++++++++++++++++++++++++++ test/deps/Project.toml | 5 ++ 2 files changed, 149 insertions(+) create mode 100644 test/deps/Manifest.toml create mode 100644 test/deps/Project.toml diff --git a/test/deps/Manifest.toml b/test/deps/Manifest.toml new file mode 100644 index 0000000000000..bfc8483f487e3 --- /dev/null +++ b/test/deps/Manifest.toml @@ -0,0 +1,144 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.13.0-DEV" +manifest_format = "2.0" +project_hash = "6de5e8b1c4d9b467a5c126490dbc755dc0575a9c" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "062c5e1a5bf6ada13db96a4ae4749a4c2234f521" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "1.3.9" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "6ac9e4acc417a5b534ace12690bc6973c25b862f" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.10.3" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LoweredCodeUtils]] +deps = ["JuliaInterpreter"] +git-tree-sha1 = "4ef1c538614e3ec30cb6383b9eb0326a5c3a9763" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.3.0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.0+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.REPL]] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Revise]] +deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] +git-tree-sha1 = "1d03585ab1bb9a6604094d0e521034df95f92cf2" +repo-rev = "master" +repo-url = "https://github.com/timholy/Revise.jl.git" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.8.0" + + [deps.Revise.extensions] + DistributedExt = "Distributed" + + [deps.Revise.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" diff --git a/test/deps/Project.toml b/test/deps/Project.toml new file mode 100644 index 0000000000000..7f7ec8b06162c --- /dev/null +++ b/test/deps/Project.toml @@ -0,0 +1,5 @@ +[deps] +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" + +[sources] +Revise = {rev = "master", url = "https://github.com/timholy/Revise.jl.git"} From ebc1c2c50de6972896b7f54acbf0ddec806106e5 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 29 May 2025 21:39:02 -0400 Subject: [PATCH 333/662] doc: Add link to Documenter.jl doctest page (#58558) As requested in https://github.com/JuliaLang/julia/pull/58506#discussion_r2111810725 --- doc/src/devdocs/contributing/jldoctests.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/src/devdocs/contributing/jldoctests.md b/doc/src/devdocs/contributing/jldoctests.md index 0542e461c265f..bf989062b1305 100644 --- a/doc/src/devdocs/contributing/jldoctests.md +++ b/doc/src/devdocs/contributing/jldoctests.md @@ -83,3 +83,6 @@ created in the first block remain available in the following ones. When a snippet needs to preserve its result for later examples, give it a label and reuse that label. This avoids repeating setup code and mirrors a REPL session more closely. + +## Further reading +For a complete reference of doctest syntax, see the [corresponding Documenter.jl docs](https://documenter.juliadocs.org/stable/man/doctests/). From b6c81ed7f211cc97e32c2ee8dcadef1be61517f8 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 29 May 2025 21:40:39 -0400 Subject: [PATCH 334/662] Add AGENTS.md and docs on how to use AI agents (#58561) AGENTS.md has instructions to various AI agents on how to handle various tasks. The current contents are preliminary, I haven't really verified that they work well yet (there's still various other issues I'm working through), but I wanted to get the basic structure in place. In addition, add documentation to the Contributor's Guide on how to set up and configure various AI agents to work well with Julia. --- AGENTS.md | 60 ++++++++++++++++++++++++ CLAUDE.md | 1 + doc/make.jl | 1 + doc/src/devdocs/contributing/aiagents.md | 49 +++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md create mode 100644 doc/src/devdocs/contributing/aiagents.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000..6b2d6e4c080ec --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,60 @@ +# Information for AI agents + +## Module Organization +- `base/` - Core standard library (loaded at startup) +- `stdlib/` - Standard library packages (can be loaded independently) +- `Compiler/` - Julia compiler as a separate module (can be swapped) +- `src/` - C/C++ runtime and LLVM codegen +- `cli/` - Command-line interface and loader +- `doc/` - Documentation and User Manual + +## Running Julia + +You should have a recent binary copy of julia in your `$HOME/.juliaup/bin` directory. +You may use this julia executable for validation. +If a built version of Julia exists in the current source tree (at `usr/bin/julia`), +prefer that version. +Note that any changes you make to the source code after the binary is built +will not be reflected, unless you use `Revise`. + +## For all changes + +1. Run `make test-whitespace` before creating the PR to make sure you're not committing any whitespace errors. + +## Building Julia + +If you made changes to the runtime (any files in `src/`), you will need to rebuild +julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes +depending on your changes. + +## Using Revise + +If you have made changes to files included in the system image (base/ or stdlib/), +and need to run code with these changes included, you can use `Revise`. +To do so, run `using Revise; Revise.track(Base)` (or Revise.track with the stdlib you modified). +The test system supports doing this automatically (see below). + +## Specific instructions for particular changes + +### Doctests + +If you have changed doctests (i.e. any `jldoctest` code block), you should run the doctests before +preparing a PR. See `doc/README.md` for how to do this. For instructions on writing doctests, see +`doc/src/devdocs/contributing/jldoctests.md`. Note that the doctests may take up to 15 minutes. +Do not terminate the doctests before completion. If you are ChatGPT, you may have to increase yield_timeout_ms. + +### Test changes + +If you have changed a test (e.g. `foo`), you should run `make test-revise-foo` for the +corresponding test to ensure that the test is still passing with your changes. +If you are adding a new test, add it to an existing test file. Do not +create a new test file unless explicitly instructed. + +## Commit message formatting + +When writing commit messages, follow the format "component: Brief summary" for +the title. In the body of the commit message, provide a brief prose summary +of the purpose of the changes made. Do not specifically mention added tests, comments, +documentation, etc., unless this is the main purpose of the change. Do not mention +the test plan, unless it differs from what you were instructed to do in AGENTS.md. +If your change fixes one or more issues, use the syntax "Fixes #" at the end of the commit message, but do not include it in the title. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000000000..47dc3e3d863cf --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/doc/make.jl b/doc/make.jl index e2009725ae0d9..32c6f11cda879 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -288,6 +288,7 @@ DevDocs = [ "devdocs/contributing/patch-releases.md", "devdocs/contributing/formatting.md", "devdocs/contributing/git-workflow.md", + "devdocs/contributing/aiagents.md" ] ] diff --git a/doc/src/devdocs/contributing/aiagents.md b/doc/src/devdocs/contributing/aiagents.md new file mode 100644 index 0000000000000..6fc7b9a662071 --- /dev/null +++ b/doc/src/devdocs/contributing/aiagents.md @@ -0,0 +1,49 @@ +# Using AI agents to work on Julia + +> ![WARNING] +> You are responsible for the code you submit in PRs. Do not submit PRs +> containing AI-generated code that you do not understand or that does not +> meet the ordinary quality bar for PRs to julia. + +This page documents best practices for setting up AI agents to work with Julia. +If you find additional prompt instructions that work well for common tasks, +consider submitting a PR to add these to AGENTS.md. + +## Google Jules + +Use the following for your `Initial Setup` configuration. + +``` +curl -fsSL https://install.julialang.org | sh -s -- -y --default-channel nightly +. /home/swebot/.profile +``` + +Jules has access to the internet, so you can give it links to issues or additional +documentation in your prompting. + +## OpenAI Codex + +Configure the following: + +Setup Script +``` +apt update +apt install less +curl -fsSL https://install.julialang.org | sh -s -- -y --default-channel nightly +source /root/.bashrc +make -C /workspace/julia/doc alldeps JULIA_EXECUTABLE="/root/.juliaup/bin/julia" +make -C /workspace/julia/test install-revise-deps JULIA_EXECUTABLE="/root/.juliaup/bin/julia" +``` + +Environment Variables +``` +JULIA_PKG_OFFLINE=true +``` + +Codex does not have internet access after initial setup, so you cannot give it +additional information as links - you will need to copy any relevant text into +the prompt. + +Note that Codex rebuilds the environment after every invocation. This can +add significant latency. Codex work best for well-defined tasks that can +be solved in a single shot. From b236438a6b4e497c44ae24d5f877ffc29bdd1ae8 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 29 May 2025 22:56:30 -0400 Subject: [PATCH 335/662] note that --trim is experimental and other history file fixes (#58569) fixes #58094 --- HISTORY.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 8f4168dd7fcfb..6c2ca03662233 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,15 +1,14 @@ Julia v1.12 Release Notes -======================== +========================= New language features --------------------- -* New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from - entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). To support - Core.finalizer, inference will now opportunistically discover future invokelatest calls and compile - the required code for them. -* Redefinition of constants is now well defined and follows world age semantics. Additional redefinitions - (e.g. of structs) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). +* New experimental option `--trim` that creates smaller binaries by removing code not proven to be reachable from + entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). Not all + code is expected to work with this option, and since it is experimental you may encounter problems. +* Redefinition of constants is now well defined and follows world age semantics ([#57253]). Additional redefinitions + (e.g. of types) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). * A new keyword argument `usings::Bool` has been added to `names`, returning all names visible via `using` ([#54609]). * The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, @@ -59,7 +58,7 @@ Language changes * Calling `using` on a package name inside of that package of that name (especially relevant for a submodule) now explicitly uses that package without examining the Manifest and environment, which is identical to the behavior of `..Name`. This appears to better match - how users expect this to behave in the wild. ([#57727]) + how users expect this to behave in the wild ([#57727]). Compiler/Runtime improvements ----------------------------- @@ -283,9 +282,10 @@ Tooling Improvements [#57081]: https://github.com/JuliaLang/julia/issues/57081 [#57087]: https://github.com/JuliaLang/julia/issues/57087 [#57109]: https://github.com/JuliaLang/julia/issues/57109 +[#57253]: https://github.com/JuliaLang/julia/issues/57253 Julia v1.11 Release Notes -======================== +========================= New language features --------------------- From bc30cf23de79eee8e5cbe68813890cf145df1e47 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 30 May 2025 03:47:44 -0400 Subject: [PATCH 336/662] test: Fix accidentally left over make dependency (#58571) --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index e4c036b20bafd..243e15db3565a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -54,7 +54,7 @@ relocatedepot: @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) -revise-relocatedepot: revise-% : dep_revise +revise-relocatedepot: revise-% : @rm -rf $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) From 75946ce31ccc4c32979e6a0fec5daabe20f8a58a Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 30 May 2025 05:53:36 -0500 Subject: [PATCH 337/662] Add binding invalidations to log (#58226) Currently we log the invalidated backedges of a binding invalidation, but the actual trigger of the invalidation is not logged. This is needed to allow SnoopCompile to attribute a cause to those invalidations. --- base/invalidation.jl | 23 ++++++++++++++++++----- src/gf.c | 13 +++++++++++++ test/precompile.jl | 26 ++++++++++++++++++++++++++ test/worlds.jl | 28 ++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 34f260f7379fd..8057c16bd9fa9 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -67,20 +67,25 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid binding = convert(Core.Binding, gr) if isdefined(method, :source) src = _uncompressed_ir(method) - old_stmts = src.code invalidate_all = should_invalidate_code_for_globalref(gr, src) end + invalidated_any = false for mi in specializations(method) isdefined(mi, :cache) || continue ci = mi.cache + invalidated = false while true if ci.max_world > new_max_world && (invalidate_all || scan_edge_list(ci, binding)) ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), ci, new_max_world) + invalidated = true end isdefined(ci, :next) || break ci = ci.next end + invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), mi) + invalidated_any |= invalidated end + return invalidated_any end export_affecting_partition_flags(bpart::Core.BindingPartition) = @@ -104,18 +109,21 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== export_affecting_partition_flags(new_bpart) + invalidated_any = false + queued_bindings = Tuple{Core.Binding, Core.BindingPartition, Core.BindingPartition}[] # defer handling these to keep the logging coherent if need_to_invalidate_code if (b.flags & BINDING_FLAG_ANY_IMPLICIT_EDGES) != 0 nmethods = ccall(:jl_module_scanned_methods_length, Csize_t, (Any,), gr.mod) for i = 1:nmethods method = ccall(:jl_module_scanned_methods_getindex, Any, (Any, Csize_t), gr.mod, i)::Method - invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) + invalidated_any |= invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) end end if isdefined(b, :backedges) for edge in b.backedges if isa(edge, CodeInstance) ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) + invalidated_any = true elseif isa(edge, Core.Binding) isdefined(edge, :partitions) || continue latest_bpart = edge.partitions @@ -124,9 +132,9 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core if is_some_binding_imported(binding_kind(latest_bpart)) partition_restriction(latest_bpart) === b || continue end - invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) + push!(queued_bindings, (edge, latest_bpart, latest_bpart)) else - invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) + invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end @@ -148,11 +156,16 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : latest_bpart if need_to_invalidate_code || new_bpart !== latest_bpart - invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) + push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart)) end end end end + invalidated_any && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) + for (edge, invalidated_bpart, new_bpart) in queued_bindings + invalidated_any |= invalidate_code_for_globalref!(edge, invalidated_bpart, new_bpart, new_max_world) + end + return invalidated_any end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) diff --git a/src/gf.c b/src/gf.c index 2fdb10b4f67ff..6e2403826b7f1 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1891,6 +1891,19 @@ JL_DLLEXPORT void jl_invalidate_code_instance(jl_code_instance_t *replaced, size invalidate_code_instance(replaced, max_world, 1); } +JL_DLLEXPORT void jl_maybe_log_binding_invalidation(jl_value_t *replaced) +{ + if (_jl_debug_method_invalidation) { + if (replaced) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, replaced); + } + jl_value_t *loctag = jl_cstr_to_string("jl_maybe_log_binding_invalidation"); + JL_GC_PUSH1(&loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + JL_GC_POP(); + } +} + static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth) { uint8_t recursion_flags = 0; jl_array_t *backedges = jl_mi_get_backedges_mutate(replaced_mi, &recursion_flags); diff --git a/test/precompile.jl b/test/precompile.jl index 2555086214c77..e95df0d50ae60 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -967,6 +967,12 @@ precompile_test_harness("code caching") do dir const gib = makewib(1) fib() = gib.ib.x + struct LogBindingInvalidation + x::Int + end + const glbi = LogBindingInvalidation(1) + flbi() = @__MODULE__().glbi.x + # force precompilation build_stale(37) stale('c') @@ -991,10 +997,13 @@ precompile_test_harness("code caching") do dir useA() = $StaleA.stale("hello") useA2() = useA() + useflbi() = $StaleA.flbi() + # force precompilation begin Base.Experimental.@force_compile useA2() + useflbi() end precompile($StaleA.fib, ()) @@ -1035,6 +1044,13 @@ precompile_test_harness("code caching") do dir end const gib = makewib(2.0) end) + # TODO: test a "method_globalref" invalidation also + Base.eval(MA, quote + struct LogBindingInvalidation # binding invalidations can't be done during precompilation + x::Float64 + end + const glbi = LogBindingInvalidation(2.0) + end) @eval using $StaleC invalidations = Base.StaticData.debug_method_invalidation(true) @eval using $StaleB @@ -1096,6 +1112,16 @@ precompile_test_harness("code caching") do dir @test !hasvalid(mi, world) @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) + idxb = findfirst(x -> x isa Core.Binding, invalidations) + @test invalidations[idxb+1] == "insert_backedges_callee" + idxv = findnext(==("verify_methods"), invalidations, idxb) + if invalidations[idxv-1].def.def.name === :getproperty + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + end + @test invalidations[idxv-1].def.def.name === :flbi + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + @test invalidations[idxv-1].def.def.name === :useflbi + m = only(methods(MB.map_nbits)) @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges end diff --git a/test/worlds.jl b/test/worlds.jl index 4b0ef208f9c7e..5c9510e67d2a1 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -436,6 +436,34 @@ idxi = findfirst(==(m58080i), logmeths) @test logmeths[end-1] == m58080s @test logmeths[end] == "jl_method_table_insert" +# logging binding invalidations +struct LogBindingInvalidation + x::Int +end +makelbi(x) = LogBindingInvalidation(x) +const glbi = makelbi(1) +oLBI, oglbi = LogBindingInvalidation, glbi +flbi() = @__MODULE__().glbi.x +flbi() +milbi1 = only(Base.specializations(only(methods(makelbi)))) +milbi2 = only(Base.specializations(only(methods(flbi)))) +logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) +struct LogBindingInvalidation + x::Float64 +end +const glbi = makelbi(2.0) +@test flbi() === 2.0 +ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) +@test milbi1.cache.def ∈ logmeths +@test milbi2.cache.next.def ∈ logmeths +i = findfirst(x -> isa(x, Core.BindingPartition), logmeths) +T = logmeths[i].restriction +@test T === oLBI +@test logmeths[i+1] == "jl_maybe_log_binding_invalidation" +T = logmeths[end-1].restriction +@test T === oglbi +@test logmeths[end] == "jl_maybe_log_binding_invalidation" + # issue #50091 -- missing invoke edge affecting nospecialized dispatch module ExceptionUnwrapping @nospecialize From b61664e8432554356f6f49fe2c69e8d06c173a5c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 30 May 2025 13:23:36 -0400 Subject: [PATCH 338/662] Rollup misc doc and infrastructure tweaks (#58573) This rolls up a couple of bug fixes with some tweaks to documentation and AGENTS.md that came in useful getting the AI to do #58572. --- .gitignore | 1 + AGENTS.md | 21 ++++++++++++++++++--- base/runtime_internals.jl | 8 +++++++- doc/src/manual/worldage.md | 2 +- test/Makefile | 2 +- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7975fa91a69d7..1953c3ae06b30 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ /source-dist.tmp1 /test/results_*.json /test/results_*.dat +/test/deps *.expmap *.exe diff --git a/AGENTS.md b/AGENTS.md index 6b2d6e4c080ec..e5d9875856df4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ will not be reflected, unless you use `Revise`. ## For all changes -1. Run `make test-whitespace` before creating the PR to make sure you're not committing any whitespace errors. +1. Run `make check-whitespace` before creating the PR to make sure you're not committing any whitespace errors. ## Building Julia @@ -47,8 +47,18 @@ Do not terminate the doctests before completion. If you are ChatGPT, you may hav If you have changed a test (e.g. `foo`), you should run `make test-revise-foo` for the corresponding test to ensure that the test is still passing with your changes. -If you are adding a new test, add it to an existing test file. Do not -create a new test file unless explicitly instructed. +- If you are adding a new test, add it to an existing test file. Do not create a new test file unless explicitly instructed. +- Write one comment at the top of the test to explain what is being tested. + Otherwise keep comments minimal. + +### Writing code +After writing code, look up the docstring for each function you used. If there +are recommendations or additional considerations that apply to these functions, +make sure to take them into account. + +#### Specific instructions +- Do not `ccall` runtime C functions directly if there are existing wrappers for the function. +- Do not explicitly add a module prefix if the code you're adding is in the same module. E.g. do not use `Base.` for code in Base unless required. ## Commit message formatting @@ -58,3 +68,8 @@ of the purpose of the changes made. Do not specifically mention added tests, com documentation, etc., unless this is the main purpose of the change. Do not mention the test plan, unless it differs from what you were instructed to do in AGENTS.md. If your change fixes one or more issues, use the syntax "Fixes #" at the end of the commit message, but do not include it in the title. + +When creating pull requests, if the pull request consists of one commit only, +use the body of the commit for the body of the pull request. If there are multiple +commits in the pull request, follow the same guidelines for the pull request +as for the commit body. diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index b50d0c7b23881..902ac516f1775 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1306,8 +1306,14 @@ max_world(m::Core.CodeInfo) = m.max_world """ get_world_counter() -Returns the current maximum world-age counter. This counter is global and monotonically +Returns the current maximum world-age counter. This counter is monotonically increasing. + +!!! warning + This counter is global and may change at any time between invocations. + In general, most reflection functions operate on the current task's world + age, rather than the global maximum world age. See [`tls_world_age`](@ref) + as well as the [manual chapter of world age](@ref man-world-age). """ get_world_counter() = ccall(:jl_get_world_counter, UInt, ()) diff --git a/doc/src/manual/worldage.md b/doc/src/manual/worldage.md index 26853b84b3031..41cceec3b87c6 100644 --- a/doc/src/manual/worldage.md +++ b/doc/src/manual/worldage.md @@ -1,4 +1,4 @@ -# The World Age mechanism +# [The World Age mechanism](@id man-world-age) !!! note World age is an advanced concept. For the vast majority of Julia users, the world age diff --git a/test/Makefile b/test/Makefile index 243e15db3565a..8b9fc139488f4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -35,7 +35,7 @@ $(TESTS): $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) install-revise-deps: - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise --help-list install_revise_deps) $(addprefix revise-, $(TESTS)): revise-% : @cd $(SRCDIR) && \ From aa06976c29fb0a9659d9dc194b58ef4ec61d8d2c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 30 May 2025 14:07:41 -0400 Subject: [PATCH 339/662] make typejoin nothrow (#58578) Fixes #50985 --- base/promotion.jl | 5 +++++ test/core.jl | 19 +++---------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/base/promotion.jl b/base/promotion.jl index 719cd2dc32b61..f935c546915be 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -23,11 +23,16 @@ typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u)) = (@_foldable_met typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u), ts...) = (@_foldable_meta; @_nospecializeinfer_meta; afoldl(typejoin, typejoin(t, s, u), ts...)) function typejoin(@nospecialize(a), @nospecialize(b)) @_foldable_meta + @_nothrow_meta @_nospecializeinfer_meta if isa(a, TypeVar) return typejoin(a.ub, b) elseif isa(b, TypeVar) return typejoin(a, b.ub) + elseif a === b + return a + elseif !isa(a, Type) || !isa(b, Type) + return Any elseif a <: b return b elseif b <: a diff --git a/test/core.jl b/test/core.jl index 0c4133d949346..c84d68bafece9 100644 --- a/test/core.jl +++ b/test/core.jl @@ -309,22 +309,9 @@ end |> only == Type{typejoin(Int, UInt)} typejoin(Int, UInt, Float64) end |> only == Type{typejoin(Int, UInt, Float64)} -let res = @test_throws TypeError let - Base.Experimental.@force_compile - typejoin(1, 2) - nothing - end - err = res.value - @test err.func === :<: -end -let res = @test_throws TypeError let - Base.Experimental.@force_compile - typejoin(1, 2, 3) - nothing - end - err = res.value - @test err.func === :<: -end +@test typejoin(1, 2) === Any +@test typejoin(1, 2, 3) === Any +@test typejoin(Int, Int, 3) === Any # promote_typejoin returns a Union only with Nothing/Missing combined with concrete types for T in (Nothing, Missing) From f7d8a58060957456e7e10333c8519265e619ae89 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 30 May 2025 15:17:05 -0400 Subject: [PATCH 340/662] doc/test: Consolidate julia package utility dependencies (#58574) Our docsystem has the ability to install a local working copy of documenter, complete with manifest/project and local depot to keep things isolated from user packages. I recently made all of this work out of tree, and in #58559 added the same mechanism for the installation of revise in test-revise-*. However, then I realized that the docsystem also has the ability to install a copy of Reivse. Clearly a third copy of all of this is excessive. Instead, unify all the manifests into `deps/jlutilities`, so that the same Revise project/manifest is used for both doc and test. Similarly, only use one depot at `buildroot/deps/jlutilities/depot` that both systems share. --- .gitignore | 1 + .../jlutilities/documenter}/Manifest.toml | 4 ++-- {doc => deps/jlutilities/documenter}/Project.toml | 0 .../jlutilities/revise}/Manifest.toml | 0 .../deps => deps/jlutilities/revise}/Project.toml | 0 doc/Makefile | 2 +- doc/make.jl | 15 +++++---------- test/runtests.jl | 4 ++-- 8 files changed, 11 insertions(+), 15 deletions(-) rename {doc => deps/jlutilities/documenter}/Manifest.toml (99%) rename {doc => deps/jlutilities/documenter}/Project.toml (100%) rename {test/deps => deps/jlutilities/revise}/Manifest.toml (100%) rename {test/deps => deps/jlutilities/revise}/Project.toml (100%) diff --git a/.gitignore b/.gitignore index 1953c3ae06b30..c4df2542005d4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /usr-staging /Make.user /julia-* +/deps/jlutilities/depot /source-dist.tmp /source-dist.tmp1 /test/results_*.json diff --git a/doc/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml similarity index 99% rename from doc/Manifest.toml rename to deps/jlutilities/documenter/Manifest.toml index 86a9a130d49f1..e531512abef78 100644 --- a/doc/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -129,7 +129,7 @@ uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" version = "1.9.0+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -164,7 +164,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.2.25" +version = "2025.5.20" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" diff --git a/doc/Project.toml b/deps/jlutilities/documenter/Project.toml similarity index 100% rename from doc/Project.toml rename to deps/jlutilities/documenter/Project.toml diff --git a/test/deps/Manifest.toml b/deps/jlutilities/revise/Manifest.toml similarity index 100% rename from test/deps/Manifest.toml rename to deps/jlutilities/revise/Manifest.toml diff --git a/test/deps/Project.toml b/deps/jlutilities/revise/Project.toml similarity index 100% rename from test/deps/Project.toml rename to deps/jlutilities/revise/Project.toml diff --git a/doc/Makefile b/doc/Makefile index 140b2282e3a3f..d076d9c01a31e 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -70,4 +70,4 @@ deploy: deps update-documenter: @echo "Updating Documenter." - JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project --color=yes -e 'using Pkg; Pkg.update("Documenter")' + JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project=$(call cygpath_w,$(SRCDIR)/../deps/jlutilities/documenter/) --color=yes -e 'using Pkg; Pkg.update("Documenter")' diff --git a/doc/make.jl b/doc/make.jl index 32c6f11cda879..da78a8420fa2c 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -22,13 +22,12 @@ let r = r"stdlibdir=(.+)", i = findfirst(x -> occursin(r, x), ARGS) end # Install dependencies needed to build the documentation. -Base.ACTIVE_PROJECT[] = nothing -empty!(LOAD_PATH) -push!(LOAD_PATH, @__DIR__, "@stdlib") +documenter_project_dir = joinpath(@__DIR__, "..", "deps", "jlutilities", "documenter") empty!(DEPOT_PATH) -push!(DEPOT_PATH, joinpath(buildrootdoc, "deps")) +push!(DEPOT_PATH, joinpath(buildroot, "deps", "jlutilities", "depot")) push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg +Pkg.activate(documenter_project_dir) Pkg.instantiate() if "deps" in ARGS @@ -314,12 +313,8 @@ end const use_revise = "revise=true" in ARGS if use_revise - let revise_env = joinpath(buildrootdoc, "deps", "revise") - Pkg.activate(revise_env) - Pkg.add("Revise"; preserve=Pkg.PRESERVE_NONE) - Base.ACTIVE_PROJECT[] = nothing - pushfirst!(LOAD_PATH, revise_env) - end + Pkg.activate(joinpath(@__DIR__, "..", "deps", "jlutilities", "revise")) + Pkg.instantiate() end function maybe_revise(ex) use_revise || return ex diff --git a/test/runtests.jl b/test/runtests.jl index 931c1ddb07930..bd56552377421 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,9 +28,9 @@ end if use_revise # First put this at the top of the DEPOT PATH to install revise if necessary. # Once it's loaded, we swizzle it to the end, to avoid confusing any tests. - pushfirst!(DEPOT_PATH, joinpath(buildroot, "test", "deps")) + pushfirst!(DEPOT_PATH, joinpath(buildroot, "deps", "jlutilities", "depot")) using Pkg - Pkg.activate(joinpath(@__DIR__, "deps")) + Pkg.activate(joinpath(@__DIR__, "..", "deps", "jlutilities", "revise")) Pkg.instantiate() using Revise union!(Revise.stdlib_names, Symbol.(STDLIBS)) From 36bd3ad86b2962e2ffe497e756d68e0aa432d867 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 30 May 2025 17:32:18 -0400 Subject: [PATCH 341/662] Make `Ptr` values static-show w/ type-information (#58584) Small follow-up to https://github.com/JuliaLang/julia/pull/58512/ --- src/rtutils.c | 7 ------- test/show.jl | 3 +++ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/rtutils.c b/src/rtutils.c index 1f341ee60c972..d5ec8a78b39ce 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -1065,13 +1065,6 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (vt == jl_uint8_type) { n += jl_printf(out, "0x%02" PRIx8, *(uint8_t*)v); } - else if (jl_pointer_type && jl_is_cpointer_type((jl_value_t*)vt)) { -#ifdef _P64 - n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); -#else - n += jl_printf(out, "0x%08" PRIx32, *(uint32_t*)v); -#endif - } else if (vt == jl_float16_type) { n += jl_static_show_float(out, julia_half_to_float(*(uint16_t *)v), vt); } diff --git a/test/show.jl b/test/show.jl index 8c9a1b655c047..8abd99cf95455 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1595,6 +1595,9 @@ struct var"%X%" end # Invalid name without '#' Float16(1e4), 1f8, 1e17, Float16(-1e4), -1f8, -1e17, + # Pointers should round-trip + Ptr{Cvoid}(0), Ptr{Cvoid}(typemax(UInt)), Ptr{Any}(0), Ptr{Any}(typemax(UInt)), + # :var"" escaping rules differ from strings (#58484) :foo, :var"bar baz", From 8275173edbb2aa269376e8dabbced37841cc4776 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Fri, 30 May 2025 19:37:47 -0300 Subject: [PATCH 342/662] make _jl_gc_collect jl_notsafepoint (#58590) Seems sketchy to trigger a GC inside of a GC. Catching this bug statically would have saved a bunch of time in https://github.com/JuliaLang/julia/pull/58487. --- src/gc-stock.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gc-stock.c b/src/gc-stock.c index ce0502058baa7..0132bbfeb1931 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -2766,7 +2766,7 @@ void gc_mark_clean_reclaim_sets(void) } } -static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { jl_task_t *task; task = ptls2->root_task; @@ -2795,7 +2795,7 @@ static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) } } -static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { jl_bt_element_t *bt_data = ptls2->bt_data; size_t bt_size = ptls2->bt_size; @@ -2809,7 +2809,7 @@ static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) } } -static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { void **items = ptls2->gc_tls.heap.remset.items; size_t len = ptls2->gc_tls.heap.remset.len; @@ -2824,7 +2824,7 @@ static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) ptls2->gc_tls.heap.remset_nptr = 0; } -static void gc_check_all_remsets_are_empty(void) +static void gc_check_all_remsets_are_empty(void) JL_NOTSAFEPOINT { for (int i = 0; i < gc_n_threads; i++) { jl_ptls_t ptls2 = gc_all_tls_states[i]; @@ -2839,7 +2839,7 @@ extern jl_value_t *cmpswap_names JL_GLOBALLY_ROOTED; extern jl_task_t *wait_empty JL_GLOBALLY_ROOTED; // mark the initial root set -static void gc_mark_roots(jl_gc_markqueue_t *mq) +static void gc_mark_roots(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT { // modules gc_try_claim_and_push(mq, jl_main_module, NULL); @@ -3023,7 +3023,7 @@ static uint64_t overallocation(uint64_t old_val, uint64_t val, uint64_t max_val) size_t jl_maxrss(void); // Only one thread should be running in this function -static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) +static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTSAFEPOINT { combine_thread_gc_counts(&gc_num, 1); From 2e39f646241ea0510b6286d88873a1d656168fee Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 31 May 2025 00:51:28 -0400 Subject: [PATCH 343/662] Add world age hint for UndefVarError (#58572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similar to the existing world age hint for MethodError, this adds a helpful message when an UndefVarError occurs because a binding was defined in a newer world age than the code trying to access it. The implementation checks if a binding that was undefined at the error's world age is now defined in the current world, and displays: "The binding may be too new: running in world age X, while current world is Y." Additionally, for all binding kinds, the error hint now notes when the binding state has changed between the error's world and the current world. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-authored-by: Claude --- base/errorshow.jl | 38 +++++++++++++++++++++++++++++--------- test/errorshow.jl | 18 ++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 8d90aae79be82..8f8a0afbe58ed 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1154,22 +1154,42 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) if isdefined(ex, :scope) scope = ex.scope if scope isa Module - bpart = Base.lookup_binding_partition(ex.world, GlobalRef(scope, var)) - kind = Base.binding_kind(bpart) - if kind === Base.PARTITION_KIND_GLOBAL || kind === Base.PARTITION_KIND_UNDEF_CONST || kind == Base.PARTITION_KIND_DECLARED + bpart = lookup_binding_partition(ex.world, GlobalRef(scope, var)) + kind = binding_kind(bpart) + + # Get the current world's binding partition for comparison + curworld = tls_world_age() + cur_bpart = lookup_binding_partition(curworld, GlobalRef(scope, var)) + cur_kind = binding_kind(cur_bpart) + + # Track if we printed the "too new" message + printed_too_new = false + + # Check if the binding exists in the current world but was undefined in the error's world + if kind === PARTITION_KIND_GUARD + if isdefinedglobal(scope, var) + print(io, "\nThe binding may be too new: running in world age $(ex.world), while current world is $(curworld).") + printed_too_new = true + else + print(io, "\nSuggestion: check for spelling errors or missing imports.") + end + elseif kind === PARTITION_KIND_GLOBAL || kind === PARTITION_KIND_UNDEF_CONST || kind == PARTITION_KIND_DECLARED print(io, "\nSuggestion: add an appropriate import or assignment. This global was declared but not assigned.") - elseif kind === Base.PARTITION_KIND_FAILED + elseif kind === PARTITION_KIND_FAILED print(io, "\nHint: It looks like two or more modules export different ", "bindings with this name, resulting in ambiguity. Try explicitly ", "importing it from a particular module, or qualifying the name ", "with the module it should come from.") - elseif kind === Base.PARTITION_KIND_GUARD - print(io, "\nSuggestion: check for spelling errors or missing imports.") - elseif Base.is_some_explicit_imported(kind) - print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") - elseif kind === Base.PARTITION_KIND_BACKDATED_CONST + elseif is_some_explicit_imported(kind) + print(io, "\nSuggestion: this global was defined as `$(partition_restriction(bpart).globalref)` but not assigned a value.") + elseif kind === PARTITION_KIND_BACKDATED_CONST print(io, "\nSuggestion: define the const at top-level before running function that uses it (stricter Julia v1.12+ rule).") end + + # Check if binding kind changed between the error's world and current world + if !printed_too_new && kind !== cur_kind + print(io, "\nNote: the binding state changed since the error occurred (was: $(kind), now: $(cur_kind)).") + end elseif scope === :static_parameter print(io, "\nSuggestion: run Test.detect_unbound_args to detect method arguments that do not fully constrain a type parameter.") elseif scope === :local diff --git a/test/errorshow.jl b/test/errorshow.jl index 246a7f16bfc90..8b225e6907ebc 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -943,6 +943,24 @@ end @test_throws expected_message X.x end +# Module for UndefVarError world age testing +module TestWorldAgeUndef end + +@testset "UndefVarError world age hint" begin + ex = try + TestWorldAgeUndef.newvar + catch e + e + end + @test ex isa UndefVarError + + Core.eval(TestWorldAgeUndef, :(newvar = 42)) + + err_str = sprint(Base.showerror, ex) + @test occursin("The binding may be too new: running in world age", err_str) + @test occursin("while current world is", err_str) +end + # test showing MethodError with type argument struct NoMethodsDefinedHere; end let buf = IOBuffer() From d0fc24267c4ac7474e68a2ea35a71218010cc162 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 31 May 2025 14:28:40 +0900 Subject: [PATCH 344/662] REPLCompletions: remove `builtin_tfunction(::REPLInterpreter)` overload (#58585) This overload was necessary in the past, but in the current compiler implementation, all `getglobal` calls that were aggressively concretized by this overload are already concretized by `concrete_eval_call`. So this overload is no longer necessary. - closes JuliaLang/julia#58537 --- stdlib/REPL/src/REPLCompletions.jl | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 8b8fb64b07dae..1a9556e74c00f 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -563,18 +563,18 @@ CC.bail_out_toplevel_call(::REPLInterpreter, ::CC.InferenceLoopState, ::CC.Infer # be employed, for instance, by `typeinf_ext_toplevel`. is_repl_frame(sv::CC.InferenceState) = sv.linfo.def isa Module && sv.cache_mode === CC.CACHE_MODE_NULL -function is_call_graph_uncached(sv::CC.InferenceState) +function is_call_stack_uncached(sv::CC.InferenceState) CC.is_cached(sv) && return false parent = CC.frame_parent(sv) parent === nothing && return true - return is_call_graph_uncached(parent::CC.InferenceState) + return is_call_stack_uncached(parent::CC.InferenceState) end # aggressive global binding resolution within `repl_frame` function CC.abstract_eval_globalref(interp::REPLInterpreter, g::GlobalRef, bailed::Bool, sv::CC.InferenceState) # Ignore saw_latestworld - if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) + if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_stack_uncached(sv)) partition = CC.abstract_eval_binding_partition!(interp, g, sv) if CC.is_defined_const_binding(CC.binding_kind(partition)) return CC.RTEffects(Const(CC.partition_restriction(partition)), Union{}, CC.EFFECTS_TOTAL) @@ -598,33 +598,11 @@ function is_repl_frame_getproperty(sv::CC.InferenceState) return is_repl_frame(CC.frame_parent(sv)) end -# aggressive global binding resolution for `getproperty(::Module, ::Symbol)` calls within `repl_frame` -function CC.builtin_tfunction(interp::REPLInterpreter, @nospecialize(f), - argtypes::Vector{Any}, sv::CC.InferenceState) - if f === Core.getglobal && (interp.limit_aggressive_inference ? is_repl_frame_getproperty(sv) : is_call_graph_uncached(sv)) - if length(argtypes) == 2 - a1, a2 = argtypes - if isa(a1, Const) && isa(a2, Const) - a1val, a2val = a1.val, a2.val - if isa(a1val, Module) && isa(a2val, Symbol) - g = GlobalRef(a1val, a2val) - if isdefined_globalref(g) - return Const(ccall(:jl_get_globalref_value, Any, (Any,), g)) - end - return Union{} - end - end - end - end - return @invoke CC.builtin_tfunction(interp::CC.AbstractInterpreter, f::Any, - argtypes::Vector{Any}, sv::CC.InferenceState) -end - # aggressive concrete evaluation for `:inconsistent` frames within `repl_frame` function CC.concrete_eval_eligible(interp::REPLInterpreter, @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) - if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) + if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_stack_uncached(sv)) neweffects = CC.Effects(result.effects; consistent=CC.ALWAYS_TRUE) result = CC.MethodCallResult(result.rt, result.exct, neweffects, result.edge, result.edgecycle, result.edgelimited, result.volatile_inf_result) From 345c6ecfc98fee70a3ffbed6eb2c20ba5dea152f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 31 May 2025 08:08:12 -0400 Subject: [PATCH 345/662] ensure Type{T} gets inserted with the right key when T is a TypeVar (#58577) Fix #58479 --- src/typemap.c | 2 +- test/reflection.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/typemap.c b/src/typemap.c index 5b25389fe1cab..4a77af32afd13 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -31,7 +31,7 @@ static jl_value_t *jl_type_extract_name(jl_value_t *t1 JL_PROPAGATES_ROOT, int i return jl_type_extract_name(jl_unwrap_vararg(t1), invariant); } else if (jl_is_typevar(t1)) { - return jl_type_extract_name(((jl_tvar_t*)t1)->ub, invariant); + return jl_type_extract_name(((jl_tvar_t*)t1)->ub, 0); } else if (t1 == jl_bottom_type || t1 == (jl_value_t*)jl_typeofbottom_type || t1 == (jl_value_t*)jl_typeofbottom_type->super) { return (jl_value_t*)jl_typeofbottom_type->name; // put Union{} and typeof(Union{}) and Type{Union{}} together for convenience diff --git a/test/reflection.jl b/test/reflection.jl index f7c81df32f41e..8cf2385ab8b18 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -577,6 +577,32 @@ fLargeTable(::Union, ::Union) = "b" @test length(methods(fLargeTable)) == 205 @test fLargeTable(Union{Int, Missing}, Union{Int, Missing}) == "b" +# issue #58479 +fLargeTable(::Type) = "Type" +fLargeTable(::Type{<:DataType}) = "DataType" +@test fLargeTable(Type) == "Type" +@test fLargeTable(DataType) == "DataType" +@test fLargeTable(Type{DataType}) == "DataType" +@test fLargeTable(Type{UnionAll}) == "DataType" +@test fLargeTable(Type{Int}) == "DataType" +@test fLargeTable(Type{Vector}) == "Type" +@test fLargeTable(Type{Type{Union{}}}) == "DataType" +@test fLargeTable(Type{Union{}}) == "Type" +@test fLargeTable(Union{}) == "DataType" +@test fLargeTable(Type{<:DataType}) == "Type" +fLargeTable(::Type{<:UnionAll}) = "UnionAll" +@test fLargeTable(UnionAll) == "UnionAll" +@test fLargeTable(Type{Vector}) == "UnionAll" +@test fLargeTable(Type{Int}) == "DataType" +@test fLargeTable(Type{Type{Union{}}}) == "DataType" +@test fLargeTable(Type{Union{}}) == "Type" +@test_throws MethodError fLargeTable(Union{}) +@test fLargeTable(Type{<:DataType}) == "Type" +@test fLargeTable(Type{Vector{T}} where T) == "DataType" +@test fLargeTable(Union{DataType,Type{Vector{T}} where T}) == "DataType" +@test fLargeTable(Union{DataType,UnionAll,Type{Vector{T}} where T}) == "Type" +@test fLargeTable(Union{Type{Vector},Type{Vector{T}} where T}) == "Type" + # issue #15280 function f15280(x) end @test functionloc(f15280)[2] > 0 From 805f85f6e957af6d8ecdfd1ef0f5887ae4c44467 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sat, 31 May 2025 18:58:56 +0200 Subject: [PATCH 346/662] relax dispatch for the `IteratorSize` method for `Generator` (#58110) Fixes #58109 --- base/generator.jl | 2 +- test/iterators.jl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/base/generator.jl b/base/generator.jl index eb1465d7e1e11..26bb7c7d91b5d 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -98,7 +98,7 @@ IteratorSize(::Type{Any}) = SizeUnknown() IteratorSize(::Type{<:Tuple}) = HasLength() IteratorSize(::Type{<:AbstractArray{<:Any,N}}) where {N} = HasShape{N}() -IteratorSize(::Type{Generator{I,F}}) where {I,F} = IteratorSize(I) +IteratorSize(::Type{<:Generator{I}}) where {I} = (@isdefined I) ? IteratorSize(I) : SizeUnknown() haslength(iter) = IteratorSize(iter) isa Union{HasShape, HasLength} diff --git a/test/iterators.jl b/test/iterators.jl index df4fa63b433b8..a6ab4720c0d0c 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -988,6 +988,12 @@ end @test accumulate(+, (x^2 for x in 1:3); init=100) == [101, 105, 114] end +@testset "issue #58109" begin + i = Iterators.map(identity, 3) + j = Iterators.map(sqrt, 7) + @test (@inferred Base.IteratorSize(i)) === @inferred Base.IteratorSize(eltype([i, j])) +end + @testset "IteratorSize trait for zip" begin @test (@inferred Base.IteratorSize(zip())) == Base.IsInfinite() # for zip of empty tuple @test (@inferred Base.IteratorSize(zip((1,2,3), repeated(0)))) == Base.HasLength() # for zip of ::HasLength and ::IsInfinite From a49bb53e0f069da15aea058148a82ef0e9308684 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sat, 31 May 2025 21:01:52 +0200 Subject: [PATCH 347/662] expand the `IteratorSize(::Type{Char})` method to cover `AbstractChar` (#58524) --- base/char.jl | 2 +- test/char.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/char.jl b/base/char.jl index 22ffa977ca6ed..90636a6d9536e 100644 --- a/base/char.jl +++ b/base/char.jl @@ -205,7 +205,7 @@ size(c::AbstractChar, d::Integer) = d < 1 ? throw(BoundsError()) : 1 ndims(c::AbstractChar) = 0 ndims(::Type{<:AbstractChar}) = 0 length(c::AbstractChar) = 1 -IteratorSize(::Type{Char}) = HasShape{0}() +IteratorSize(::Type{<:AbstractChar}) = HasShape{0}() firstindex(c::AbstractChar) = 1 lastindex(c::AbstractChar) = 1 getindex(c::AbstractChar) = c diff --git a/test/char.jl b/test/char.jl index 5523125529031..cd7997d4a37f5 100644 --- a/test/char.jl +++ b/test/char.jl @@ -290,6 +290,7 @@ Base.codepoint(c::ASCIIChar) = reinterpret(UInt8, c) @test !isempty(ASCIIChar('x')) @test ndims(ASCIIChar('x')) == 0 @test ndims(ASCIIChar) == 0 + @test Base.IteratorSize(ASCIIChar) === Base.HasShape{0}() @test firstindex(ASCIIChar('x')) == 1 @test lastindex(ASCIIChar('x')) == 1 @test eltype(ASCIIChar) == ASCIIChar From d3759ba4680789fe69aefc1640eb8f7f1bfb1e04 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Sat, 31 May 2025 17:18:23 -0300 Subject: [PATCH 348/662] Introduce a few GC controls to limit the heap size when running benchmarks (#58487) We will benefit from having more control over Julia's heap size when benchmarking MMTk: This PR introduces two heap-limit flags: - `--hard-heap-limit`: Set a hard limit on the heap size: if we ever go above this limit, we will abort. - `--upper-bound-for-heap-target-increment`: Set an upper bound on how much the heap target can increase between consecutive collections. Note that they are behind a `GC_ENABLE_HIDDEN_CTRLS` build-time flag, so these options won't be available for most Julia users. It may be a bit tricky to test this, given that the flags are only enabled if you define `GC_ENABLE_HIDDEN_CTRLS`. --- base/options.jl | 2 ++ src/gc-mmtk.c | 2 +- src/gc-stock.c | 28 +++++++++++++++-- src/jloptions.c | 49 +++++++++++++++++++++++------ src/jloptions.h | 2 ++ src/julia.h | 2 +- test/cmdlineargs.jl | 76 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 147 insertions(+), 14 deletions(-) diff --git a/base/options.jl b/base/options.jl index 818fd33d6f7fc..e3865e076d5bb 100644 --- a/base/options.jl +++ b/base/options.jl @@ -60,6 +60,8 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 + hard_heap_limit::UInt64 + heap_target_increment::UInt64 trace_compile_timing::Int8 trim::Int8 task_metrics::Int8 diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index a6650dd7cb68c..edf74dcc65443 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -78,7 +78,7 @@ void jl_gc_init(void) { if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); } #ifdef _P64 if (hint == 0) { diff --git a/src/gc-stock.c b/src/gc-stock.c index 0132bbfeb1931..d518df6dfb52a 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3214,7 +3214,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS uint64_t target_heap; const char *reason = ""; (void)reason; // for GC_TIME output stats old_heap_size = heap_size; // TODO: Update these values dynamically instead of just during the GC - if (collection == JL_GC_AUTO) { + if (collection == JL_GC_AUTO && jl_options.hard_heap_limit == 0) { // update any heuristics only when the user does not force the GC // but still update the timings, since GC was run and reset, even if it was too early uint64_t target_allocs = 0.0; @@ -3295,6 +3295,27 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS target_heap = jl_atomic_load_relaxed(&gc_heap_stats.heap_target); } + // Kill the process if we are above the hard heap limit + if (jl_options.hard_heap_limit != 0) { + if (heap_size > jl_options.hard_heap_limit) { + // Can't use `jl_errorf` here, because it will try to allocate memory + // and we are already at the hard limit. + jl_safe_printf("Heap size exceeded hard limit of %" PRIu64 " bytes.\n", + jl_options.hard_heap_limit); + abort(); + } + } + // Ignore heap limit computation from MemBalancer-like heuristics + // if the heap target increment goes above the value specified through + // `--heap-target-increment`. + // Note that if we reach this code, we can guarantee that the heap size + // is less than the hard limit, so there will be some room to grow the heap + // until the next GC without hitting the hard limit. + if (jl_options.heap_target_increment != 0) { + target_heap = heap_size + jl_options.heap_target_increment; + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap); + } + double old_ratio = (double)promoted_bytes/(double)heap_size; if (heap_size > user_max) { next_sweep_full = 1; @@ -3692,6 +3713,9 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); jl_atomic_store_relaxed(&gc_heap_stats.heap_target, default_collect_interval); + if (jl_options.hard_heap_limit != 0) { + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, jl_options.hard_heap_limit); + } gc_num.interval = default_collect_interval; gc_num.allocd = 0; gc_num.max_pause = 0; @@ -3705,7 +3729,7 @@ void jl_gc_init(void) if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); } #ifdef _P64 total_mem = uv_get_total_memory(); diff --git a/src/jloptions.c b/src/jloptions.c index 21d034a666b1d..e31aaba4c532f 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -3,6 +3,7 @@ #include #include +#include "options.h" #include "julia.h" #include "julia_internal.h" @@ -36,7 +37,7 @@ JL_DLLEXPORT const char *jl_get_default_sysimg_path(void) /* This function is also used by gc-stock.c to parse the * JULIA_HEAP_SIZE_HINT environment variable. */ -uint64_t parse_heap_size_hint(const char *optarg, const char *option_name) +uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct) { long double value = 0.0; char unit[4] = {0}; @@ -62,14 +63,16 @@ uint64_t parse_heap_size_hint(const char *optarg, const char *option_name) multiplier <<= 40; break; case '%': - if (value > 100) - jl_errorf("julia: invalid percentage specified in %s", option_name); - uint64_t mem = uv_get_total_memory(); - uint64_t cmem = uv_get_constrained_memory(); - if (cmem > 0 && cmem < mem) - mem = cmem; - multiplier = mem/100; - break; + if (allow_pct) { + if (value > 100) + jl_errorf("julia: invalid percentage specified in %s", option_name); + uint64_t mem = uv_get_total_memory(); + uint64_t cmem = uv_get_constrained_memory(); + if (cmem > 0 && cmem < mem) + mem = cmem; + multiplier = mem/100; + break; + } default: jl_errorf("julia: invalid argument to %s (%s)", option_name, optarg); break; @@ -151,6 +154,8 @@ JL_DLLEXPORT void jl_init_options(void) 0, // strip-ir 0, // permalloc_pkgimg 0, // heap-size-hint + 0, // hard-heap-limit + 0, // heap-target-increment 0, // trace_compile_timing JL_TRIM_NO, // trim 0, // task_metrics @@ -289,6 +294,14 @@ static const char opts[] = " number of bytes, optionally in units of: B, K (kibibytes),\n" " M (mebibytes), G (gibibytes), T (tebibytes), or % (percentage\n" " of physical memory).\n\n" + " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever go above this\n" + " limit, we will abort. The value may be specified as a\n" + " number of bytes, optionally in units of: B, K (kibibytes),\n" + " M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" + " --heap-target-increment=[] Set an upper bound on how much the heap target\n" + " can increase between consecutive collections. The value may be\n" + " specified as a number of bytes, optionally in units of: B,\n" + " K (kibibytes), M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" ; static const char opts_hidden[] = @@ -380,6 +393,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_metadata, opt_strip_ir, opt_heap_size_hint, + opt_hard_heap_limit, + opt_heap_target_increment, opt_gc_threads, opt_permalloc_pkgimg, opt_trim, @@ -451,6 +466,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "hard-heap-limit", required_argument, 0, opt_hard_heap_limit }, + { "heap-target-increment", required_argument, 0, opt_heap_target_increment }, { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -960,11 +977,23 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) break; case opt_heap_size_hint: if (optarg != NULL) - jl_options.heap_size_hint = parse_heap_size_hint(optarg, "--heap-size-hint=[]"); + jl_options.heap_size_hint = parse_heap_size_option(optarg, "--heap-size-hint=[]", 1); if (jl_options.heap_size_hint == 0) jl_errorf("julia: invalid memory size specified in --heap-size-hint=[]"); break; + case opt_hard_heap_limit: + if (optarg != NULL) + jl_options.hard_heap_limit = parse_heap_size_option(optarg, "--hard-heap-limit=[]", 0); + if (jl_options.hard_heap_limit == 0) + jl_errorf("julia: invalid memory size specified in --hard-heap-limit=[]"); + break; + case opt_heap_target_increment: + if (optarg != NULL) + jl_options.heap_target_increment = parse_heap_size_option(optarg, "--heap-target-increment=[]", 0); + if (jl_options.heap_target_increment == 0) + jl_errorf("julia: invalid memory size specified in --heap-target-increment=[]"); + break; case opt_gc_threads: errno = 0; long nmarkthreads = strtol(optarg, &endptr, 10); diff --git a/src/jloptions.h b/src/jloptions.h index 06e00e9309dba..35cc8c3e13375 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -64,6 +64,8 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; + uint64_t hard_heap_limit; + uint64_t heap_target_increment; int8_t trace_compile_timing; int8_t trim; int8_t task_metrics; diff --git a/src/julia.h b/src/julia.h index 4a4a4eff81232..4cce04ac1fe76 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2580,7 +2580,7 @@ JL_DLLEXPORT ssize_t jl_sizeof_jl_options(void); JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp); JL_DLLEXPORT char *jl_format_filename(const char *output_pattern); -uint64_t parse_heap_size_hint(const char *optarg, const char *option_name); +uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct); // Set julia-level ARGS array according to the arguments provided in // argc/argv diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 12b13c1984a5e..c2ccdfce20701 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1214,6 +1214,16 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end + +@testset "hard heap limit" begin + # Set the hard heap limit to 100MB, try to allocate an array of 200MB + # and assert that the process is aborted, by checking the exit code. + cmd = `$(Base.julia_cmd()) --startup-file=no --hard-heap-limit=100M -e "a = Array{UInt8}(undef, 200*1024*1024); GC.gc()"` + p = open(pipeline(cmd, stderr=devnull, stdout=devnull)) + exitcode = wait(p) + # The process should be aborted with an error code + @test exitcode != 0 +end end ## `Main.main` entrypoint @@ -1253,6 +1263,72 @@ end end end +@testset "--hard-heap-limit" begin + exename = `$(Base.julia_cmd())` + @test errors_not_signals(`$exename --hard-heap-limit -e "exit(0)"`) + @testset "--hard-heap-limit=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] + @test errors_not_signals(`$exename --hard-heap-limit=$str -e "exit(0)"`) + end + k = 1024 + m = 1024k + g = 1024m + t = 1024g + # Express one hundred megabytes as 100MB, 100m, 100e6, etc. + one_hundred_mb_strs_and_vals = [ + ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), + ] + @testset "--hard-heap-limit=$str" for (str, val) in one_hundred_mb_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end + # Express two and a half gigabytes as 2.5g, 2.5GB, etc. + two_and_a_half_gigabytes_strs_and_vals = [ + ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), + ] + @testset "--hard-heap-limit=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end + # Express one terabyte as 1TB, 1e12, etc. + one_terabyte_strs_and_vals = [ + ("1000000000000", 1000000000000), ("1e12", 1e12), ("1TB", 1t), ("1e9gB", 1e9g), + ] + @testset "--hard-heap-limit=$str" for (str, val) in one_terabyte_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end +end + +@testset "--heap-target-increment" begin + exename = `$(Base.julia_cmd())` + @test errors_not_signals(`$exename --heap-target-increment -e "exit(0)"`) + @testset "--heap-target-increment=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] + @test errors_not_signals(`$exename --heap-target-increment=$str -e "exit(0)"`) + end + k = 1024 + m = 1024k + g = 1024m + t = 1024g + # Express one hundred megabytes as 100MB, 100m, 100e6, etc. + one_hundred_mb_strs_and_vals = [ + ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), + ] + @testset "--heap-target-increment=$str" for (str, val) in one_hundred_mb_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end + # Express two and a half gigabytes as 2.5g, 2.5GB, etc. + two_and_a_half_gigabytes_strs_and_vals = [ + ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), + ] + @testset "--heap-target-increment=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end + # Express one terabyte as 1TB, 1e12, etc. + one_terabyte_strs_and_vals = [ + ("1000000000000", 1000000000000), ("1e12", 1e12), ("1TB", 1t), ("1e9gB", 1e9g), + ] + @testset "--heap-target-increment=$str" for (str, val) in one_terabyte_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end +end + @testset "--timeout-for-safepoint-straggler" begin exename = `$(Base.julia_cmd())` timeout = 120 From 838a8f7972f28da68049c370c5e4c4f89576361a Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Sun, 1 Jun 2025 00:49:56 -0300 Subject: [PATCH 349/662] Revert "Introduce a few GC controls to limit the heap size when running benchmarks" (#58599) Reverts JuliaLang/julia#58487. X-ref: https://github.com/JuliaLang/julia/issues/58597. --- base/options.jl | 2 -- src/gc-mmtk.c | 2 +- src/gc-stock.c | 28 ++--------------- src/jloptions.c | 49 ++++++----------------------- src/jloptions.h | 2 -- src/julia.h | 2 +- test/cmdlineargs.jl | 76 --------------------------------------------- 7 files changed, 14 insertions(+), 147 deletions(-) diff --git a/base/options.jl b/base/options.jl index e3865e076d5bb..818fd33d6f7fc 100644 --- a/base/options.jl +++ b/base/options.jl @@ -60,8 +60,6 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 - hard_heap_limit::UInt64 - heap_target_increment::UInt64 trace_compile_timing::Int8 trim::Int8 task_metrics::Int8 diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index edf74dcc65443..a6650dd7cb68c 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -78,7 +78,7 @@ void jl_gc_init(void) { if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); + hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); } #ifdef _P64 if (hint == 0) { diff --git a/src/gc-stock.c b/src/gc-stock.c index d518df6dfb52a..0132bbfeb1931 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3214,7 +3214,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS uint64_t target_heap; const char *reason = ""; (void)reason; // for GC_TIME output stats old_heap_size = heap_size; // TODO: Update these values dynamically instead of just during the GC - if (collection == JL_GC_AUTO && jl_options.hard_heap_limit == 0) { + if (collection == JL_GC_AUTO) { // update any heuristics only when the user does not force the GC // but still update the timings, since GC was run and reset, even if it was too early uint64_t target_allocs = 0.0; @@ -3295,27 +3295,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS target_heap = jl_atomic_load_relaxed(&gc_heap_stats.heap_target); } - // Kill the process if we are above the hard heap limit - if (jl_options.hard_heap_limit != 0) { - if (heap_size > jl_options.hard_heap_limit) { - // Can't use `jl_errorf` here, because it will try to allocate memory - // and we are already at the hard limit. - jl_safe_printf("Heap size exceeded hard limit of %" PRIu64 " bytes.\n", - jl_options.hard_heap_limit); - abort(); - } - } - // Ignore heap limit computation from MemBalancer-like heuristics - // if the heap target increment goes above the value specified through - // `--heap-target-increment`. - // Note that if we reach this code, we can guarantee that the heap size - // is less than the hard limit, so there will be some room to grow the heap - // until the next GC without hitting the hard limit. - if (jl_options.heap_target_increment != 0) { - target_heap = heap_size + jl_options.heap_target_increment; - jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap); - } - double old_ratio = (double)promoted_bytes/(double)heap_size; if (heap_size > user_max) { next_sweep_full = 1; @@ -3713,9 +3692,6 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); jl_atomic_store_relaxed(&gc_heap_stats.heap_target, default_collect_interval); - if (jl_options.hard_heap_limit != 0) { - jl_atomic_store_relaxed(&gc_heap_stats.heap_target, jl_options.hard_heap_limit); - } gc_num.interval = default_collect_interval; gc_num.allocd = 0; gc_num.max_pause = 0; @@ -3729,7 +3705,7 @@ void jl_gc_init(void) if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); + hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); } #ifdef _P64 total_mem = uv_get_total_memory(); diff --git a/src/jloptions.c b/src/jloptions.c index e31aaba4c532f..21d034a666b1d 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -3,7 +3,6 @@ #include #include -#include "options.h" #include "julia.h" #include "julia_internal.h" @@ -37,7 +36,7 @@ JL_DLLEXPORT const char *jl_get_default_sysimg_path(void) /* This function is also used by gc-stock.c to parse the * JULIA_HEAP_SIZE_HINT environment variable. */ -uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct) +uint64_t parse_heap_size_hint(const char *optarg, const char *option_name) { long double value = 0.0; char unit[4] = {0}; @@ -63,16 +62,14 @@ uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int multiplier <<= 40; break; case '%': - if (allow_pct) { - if (value > 100) - jl_errorf("julia: invalid percentage specified in %s", option_name); - uint64_t mem = uv_get_total_memory(); - uint64_t cmem = uv_get_constrained_memory(); - if (cmem > 0 && cmem < mem) - mem = cmem; - multiplier = mem/100; - break; - } + if (value > 100) + jl_errorf("julia: invalid percentage specified in %s", option_name); + uint64_t mem = uv_get_total_memory(); + uint64_t cmem = uv_get_constrained_memory(); + if (cmem > 0 && cmem < mem) + mem = cmem; + multiplier = mem/100; + break; default: jl_errorf("julia: invalid argument to %s (%s)", option_name, optarg); break; @@ -154,8 +151,6 @@ JL_DLLEXPORT void jl_init_options(void) 0, // strip-ir 0, // permalloc_pkgimg 0, // heap-size-hint - 0, // hard-heap-limit - 0, // heap-target-increment 0, // trace_compile_timing JL_TRIM_NO, // trim 0, // task_metrics @@ -294,14 +289,6 @@ static const char opts[] = " number of bytes, optionally in units of: B, K (kibibytes),\n" " M (mebibytes), G (gibibytes), T (tebibytes), or % (percentage\n" " of physical memory).\n\n" - " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever go above this\n" - " limit, we will abort. The value may be specified as a\n" - " number of bytes, optionally in units of: B, K (kibibytes),\n" - " M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" - " --heap-target-increment=[] Set an upper bound on how much the heap target\n" - " can increase between consecutive collections. The value may be\n" - " specified as a number of bytes, optionally in units of: B,\n" - " K (kibibytes), M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" ; static const char opts_hidden[] = @@ -393,8 +380,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_metadata, opt_strip_ir, opt_heap_size_hint, - opt_hard_heap_limit, - opt_heap_target_increment, opt_gc_threads, opt_permalloc_pkgimg, opt_trim, @@ -466,8 +451,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, - { "hard-heap-limit", required_argument, 0, opt_hard_heap_limit }, - { "heap-target-increment", required_argument, 0, opt_heap_target_increment }, { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -977,23 +960,11 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) break; case opt_heap_size_hint: if (optarg != NULL) - jl_options.heap_size_hint = parse_heap_size_option(optarg, "--heap-size-hint=[]", 1); + jl_options.heap_size_hint = parse_heap_size_hint(optarg, "--heap-size-hint=[]"); if (jl_options.heap_size_hint == 0) jl_errorf("julia: invalid memory size specified in --heap-size-hint=[]"); break; - case opt_hard_heap_limit: - if (optarg != NULL) - jl_options.hard_heap_limit = parse_heap_size_option(optarg, "--hard-heap-limit=[]", 0); - if (jl_options.hard_heap_limit == 0) - jl_errorf("julia: invalid memory size specified in --hard-heap-limit=[]"); - break; - case opt_heap_target_increment: - if (optarg != NULL) - jl_options.heap_target_increment = parse_heap_size_option(optarg, "--heap-target-increment=[]", 0); - if (jl_options.heap_target_increment == 0) - jl_errorf("julia: invalid memory size specified in --heap-target-increment=[]"); - break; case opt_gc_threads: errno = 0; long nmarkthreads = strtol(optarg, &endptr, 10); diff --git a/src/jloptions.h b/src/jloptions.h index 35cc8c3e13375..06e00e9309dba 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -64,8 +64,6 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; - uint64_t hard_heap_limit; - uint64_t heap_target_increment; int8_t trace_compile_timing; int8_t trim; int8_t task_metrics; diff --git a/src/julia.h b/src/julia.h index 4cce04ac1fe76..4a4a4eff81232 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2580,7 +2580,7 @@ JL_DLLEXPORT ssize_t jl_sizeof_jl_options(void); JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp); JL_DLLEXPORT char *jl_format_filename(const char *output_pattern); -uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct); +uint64_t parse_heap_size_hint(const char *optarg, const char *option_name); // Set julia-level ARGS array according to the arguments provided in // argc/argv diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index c2ccdfce20701..12b13c1984a5e 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1214,16 +1214,6 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end - -@testset "hard heap limit" begin - # Set the hard heap limit to 100MB, try to allocate an array of 200MB - # and assert that the process is aborted, by checking the exit code. - cmd = `$(Base.julia_cmd()) --startup-file=no --hard-heap-limit=100M -e "a = Array{UInt8}(undef, 200*1024*1024); GC.gc()"` - p = open(pipeline(cmd, stderr=devnull, stdout=devnull)) - exitcode = wait(p) - # The process should be aborted with an error code - @test exitcode != 0 -end end ## `Main.main` entrypoint @@ -1263,72 +1253,6 @@ end end end -@testset "--hard-heap-limit" begin - exename = `$(Base.julia_cmd())` - @test errors_not_signals(`$exename --hard-heap-limit -e "exit(0)"`) - @testset "--hard-heap-limit=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] - @test errors_not_signals(`$exename --hard-heap-limit=$str -e "exit(0)"`) - end - k = 1024 - m = 1024k - g = 1024m - t = 1024g - # Express one hundred megabytes as 100MB, 100m, 100e6, etc. - one_hundred_mb_strs_and_vals = [ - ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), - ] - @testset "--hard-heap-limit=$str" for (str, val) in one_hundred_mb_strs_and_vals - @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val - end - # Express two and a half gigabytes as 2.5g, 2.5GB, etc. - two_and_a_half_gigabytes_strs_and_vals = [ - ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), - ] - @testset "--hard-heap-limit=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals - @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val - end - # Express one terabyte as 1TB, 1e12, etc. - one_terabyte_strs_and_vals = [ - ("1000000000000", 1000000000000), ("1e12", 1e12), ("1TB", 1t), ("1e9gB", 1e9g), - ] - @testset "--hard-heap-limit=$str" for (str, val) in one_terabyte_strs_and_vals - @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val - end -end - -@testset "--heap-target-increment" begin - exename = `$(Base.julia_cmd())` - @test errors_not_signals(`$exename --heap-target-increment -e "exit(0)"`) - @testset "--heap-target-increment=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] - @test errors_not_signals(`$exename --heap-target-increment=$str -e "exit(0)"`) - end - k = 1024 - m = 1024k - g = 1024m - t = 1024g - # Express one hundred megabytes as 100MB, 100m, 100e6, etc. - one_hundred_mb_strs_and_vals = [ - ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), - ] - @testset "--heap-target-increment=$str" for (str, val) in one_hundred_mb_strs_and_vals - @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val - end - # Express two and a half gigabytes as 2.5g, 2.5GB, etc. - two_and_a_half_gigabytes_strs_and_vals = [ - ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), - ] - @testset "--heap-target-increment=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals - @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val - end - # Express one terabyte as 1TB, 1e12, etc. - one_terabyte_strs_and_vals = [ - ("1000000000000", 1000000000000), ("1e12", 1e12), ("1TB", 1t), ("1e9gB", 1e9g), - ] - @testset "--heap-target-increment=$str" for (str, val) in one_terabyte_strs_and_vals - @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val - end -end - @testset "--timeout-for-safepoint-straggler" begin exename = `$(Base.julia_cmd())` timeout = 120 From 905a44b8ef49cc7f5e59e7ee65b0a71877aa8f58 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 1 Jun 2025 07:15:44 +0200 Subject: [PATCH 350/662] simplify `wait()` (#58595) Follows up on this PR: * https://github.com/JuliaLang/julia/pull/57544#discussion_r2076253721 --- base/task.jl | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/base/task.jl b/base/task.jl index bc20ff8ac320a..7e7e5dfc3076b 100644 --- a/base/task.jl +++ b/base/task.jl @@ -1201,30 +1201,19 @@ function wait() process_events() # get the next task to run - result = nothing - have_result = false W = workqueue_for(Threads.threadid()) task = trypoptask(W) - if !(task isa Task) + if task === nothing # No tasks to run; switch to the scheduler task to run the # thread sleep logic. sched_task = get_sched_task() if ct !== sched_task - result = yieldto(sched_task) - have_result = true - else - task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), - trypoptask, W, checktaskempty) + return yieldto(sched_task) end + task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), trypoptask, W, checktaskempty) end - # We may have already switched tasks (via the scheduler task), so - # only switch if we haven't. - if !have_result - @assert task isa Task - set_next_task(task) - result = try_yieldto(ensure_rescheduled) - end - return result + set_next_task(task) + return try_yieldto(ensure_rescheduled) end if Sys.iswindows() From 0e5d6aefe27afc14d76647700b6c650ff4657d03 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 1 Jun 2025 14:37:22 +0200 Subject: [PATCH 351/662] add methods for two-arg `copyto!` between `Array` and `Memory` (#58576) Just forward to the five-arg `copyto!`. Also deduplicate code a bit. Improve the performance of `copyto!(::Memory, ::Vector)` and `copyto!(::Vector, ::Memory)`. --- base/array.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 9bf0bbc87aba9..40ecb18c6e828 100644 --- a/base/array.jl +++ b/base/array.jl @@ -316,10 +316,16 @@ end # It is also mitigated by using a constant string. _throw_argerror(s) = (@noinline; throw(ArgumentError(s))) -copyto!(dest::Array, src::Array) = copyto!(dest, 1, src, 1, length(src)) +_copyto2arg!(dest, src) = copyto!(dest, firstindex(dest), src, firstindex(src), length(src)) + +copyto!(dest::Array, src::Array) = _copyto2arg!(dest, src) +copyto!(dest::Array, src::Memory) = _copyto2arg!(dest, src) +copyto!(dest::Memory, src::Array) = _copyto2arg!(dest, src) # also to avoid ambiguities in packages -copyto!(dest::Array{T}, src::Array{T}) where {T} = copyto!(dest, 1, src, 1, length(src)) +copyto!(dest::Array{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) +copyto!(dest::Array{T}, src::Memory{T}) where {T} = _copyto2arg!(dest, src) +copyto!(dest::Memory{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) # N.B: The generic definition in multidimensional.jl covers, this, this is just here # for bootstrapping purposes. From 3e9f486430a254dd766bcd7f185fc21147d44345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Sun, 1 Jun 2025 22:34:23 +0100 Subject: [PATCH 352/662] [LibUnwind_jll] Upgrade to v1.8.2 (#58601) --- deps/checksums/unwind | 58 +++---- .../libunwind-aarch64-inline-asm.patch | 157 ------------------ .../libunwind-configure-static-lzma.patch | 22 +-- deps/unwind.mk | 6 +- deps/unwind.version | 6 +- stdlib/LibUnwind_jll/Project.toml | 2 +- 6 files changed, 47 insertions(+), 204 deletions(-) delete mode 100644 deps/patches/libunwind-aarch64-inline-asm.patch diff --git a/deps/checksums/unwind b/deps/checksums/unwind index 5d4967cb0cf22..65cd2a3ab0897 100644 --- a/deps/checksums/unwind +++ b/deps/checksums/unwind @@ -1,28 +1,30 @@ -LibUnwind.v1.8.1+2.aarch64-linux-gnu.tar.gz/md5/de3690f3a8ecf0aa5d2525813bdab3c8 -LibUnwind.v1.8.1+2.aarch64-linux-gnu.tar.gz/sha512/366090b4291623603e54d3c73437efcbc3c7f52ce0c64a63e8439eff8a3ddeb4efc1ab6b2513e0a60e2714239bf259cd667159a24207f0c9ce3134530e539155 -LibUnwind.v1.8.1+2.aarch64-linux-musl.tar.gz/md5/e8adf4e842e998b6806653964e721a47 -LibUnwind.v1.8.1+2.aarch64-linux-musl.tar.gz/sha512/77411646767f5f13e2f45d32bfa48d6864b712d46d339e3fd4d62d12f4a26b6ffb8293636209ee5645d8e5552bdf70db5a848736ef0df75db74c8c878553cd40 -LibUnwind.v1.8.1+2.aarch64-unknown-freebsd.tar.gz/md5/ee8fc39c934cf1c640ae4ae41addcc30 -LibUnwind.v1.8.1+2.aarch64-unknown-freebsd.tar.gz/sha512/6245fc3003ef24fce0f84007c0fa1390658e71dc64da6a2f5d296d3928351096ed2c0c83808890413332883abe5fcee7615eb40b2baeddfc56d3484315f3dacf -LibUnwind.v1.8.1+2.armv6l-linux-gnueabihf.tar.gz/md5/4c454e174be7b5f220f4cb8f659722d8 -LibUnwind.v1.8.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/f6e3d83576ae963f400972250c8558b0b15bdd9657aac6eacbd0c3f59af6a3574d0cc475c6e606ad8f2e0b178ba33f297aec0aeac8a5970d93b2c36d9ffae59d -LibUnwind.v1.8.1+2.armv6l-linux-musleabihf.tar.gz/md5/dbec8675d2b73807c9d9e3afc2ce2260 -LibUnwind.v1.8.1+2.armv6l-linux-musleabihf.tar.gz/sha512/45d9ac63282c21bdc6488b65fae8f03bbaa55d18b346ac3fc3d40f38ebd05b2a0db539f23dc6c6f88bbbad8f2ec2cdcf677db1acff83a99d9875bee93555ad1e -LibUnwind.v1.8.1+2.armv7l-linux-gnueabihf.tar.gz/md5/98517b7a4ae874099ef0aafb46e740c9 -LibUnwind.v1.8.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/3a00792415a15fe45c3454f9bf480222862217178a61db0738837537c7e2c50f71b53063facd591680b14e7b3bde218c34cee9b2854ad94897b306388749af1b -LibUnwind.v1.8.1+2.armv7l-linux-musleabihf.tar.gz/md5/f276569278383f7711f40e623670620d -LibUnwind.v1.8.1+2.armv7l-linux-musleabihf.tar.gz/sha512/48160616ac1ed4b3e343556517e3cbb4959e80e9be237fc820e33e06f6668e95d9365dd7c86e68dc898fee1141cd825495bbbc27d685913a2f2808d974b54c19 -LibUnwind.v1.8.1+2.i686-linux-gnu.tar.gz/md5/2cd0203f2b70436ac2323077dad1d5d1 -LibUnwind.v1.8.1+2.i686-linux-gnu.tar.gz/sha512/fa42b3306d9b67011468b2c07bdb6cca6847f0f1632ee4aca3212c5944e991f9a1ae8f881fb4ce86e641e977695942d873a39fc212bdcf6acdf3e12c24b31d8e -LibUnwind.v1.8.1+2.i686-linux-musl.tar.gz/md5/3c456a1b3da2f5d785e02e1b6cb4cd74 -LibUnwind.v1.8.1+2.i686-linux-musl.tar.gz/sha512/fce8368ee670109b681c9d442ad89fee8fdf8eac1e115407784d1e8b82cfb98acd9d2edb4dbea29f8c63c83054da2a4d34149fe231655e2535834a4ef7319666 -LibUnwind.v1.8.1+2.powerpc64le-linux-gnu.tar.gz/md5/73b04ae80ca9fdbe06b3eeaae40d5dc5 -LibUnwind.v1.8.1+2.powerpc64le-linux-gnu.tar.gz/sha512/d4083a696a3492ced38b05fb573d44c4cc2b5332a351b65be2c3992d9e932bb6ea71f48260c643fa54219adb800b5da41160e1d56b0d9145061edf2e5dfc0ef6 -LibUnwind.v1.8.1+2.x86_64-linux-gnu.tar.gz/md5/f9d6132f4166c5ede15b2303280a1066 -LibUnwind.v1.8.1+2.x86_64-linux-gnu.tar.gz/sha512/124159e7d13ce1caee5e2527746ec98b10a776f57e5f9c99053b7ab76e7d9447b998cbc044da7671fd39356445a983f16f2c7bbefc076b29e45d2c2bb4d0364e -LibUnwind.v1.8.1+2.x86_64-linux-musl.tar.gz/md5/665d9215ef915269e009f7dde1f827b3 -LibUnwind.v1.8.1+2.x86_64-linux-musl.tar.gz/sha512/2d8754bbfa7a4b576fb58a2d22b08940bb9f615988bfc388e9ea2cc96e3a573e6c31a4023b2509a3424a0ce3d946584c09ac5d18e4bca6f0f47e52597e193944 -LibUnwind.v1.8.1+2.x86_64-unknown-freebsd.tar.gz/md5/cc8149747db86524da0c9749ed538f3d -LibUnwind.v1.8.1+2.x86_64-unknown-freebsd.tar.gz/sha512/4d416999616fbf08103553aa43603ce62109c21e9a97d6a391fb267c04d382834da380f459c96412773f19d93b8e996ddd405831623ce118d239ad1a0d9025fd -libunwind-1.8.1.tar.gz/md5/10c96118ff30b88c9eeb6eac8e75599d -libunwind-1.8.1.tar.gz/sha512/aba7b578c1b8cbe78f05b64e154f3530525f8a34668b2a9f1ee6acb4b22c857befe34ad4e9e8cca99dbb66689d41bc72060a8f191bd8be232725d342809431b3 +LibUnwind.v1.8.2+0.aarch64-linux-gnu.tar.gz/md5/f23ea74855dd2db466170f170a8eea7a +LibUnwind.v1.8.2+0.aarch64-linux-gnu.tar.gz/sha512/27b4c37d225c1632e1d8814989d768b0fda2675552c611cb9fad1eea158e4b3815e0a8127c2eea97bcbfd23685d970cf1c478d24f6b5a8abe8357e12d8762ce7 +LibUnwind.v1.8.2+0.aarch64-linux-musl.tar.gz/md5/5073e00eeacd4f5921164090966f5ae8 +LibUnwind.v1.8.2+0.aarch64-linux-musl.tar.gz/sha512/0499cdd22510745e3e1f7da5c4c937a79b2eb07e2bf97afa5db9ce7e91aa2bab85bef2dc3ad8b859ac147ca8da908047bf12daf6e3d3fcb2c7a865402dd90beb +LibUnwind.v1.8.2+0.aarch64-unknown-freebsd.tar.gz/md5/b4aa1f850db793ffe9c1e0bf0524da65 +LibUnwind.v1.8.2+0.aarch64-unknown-freebsd.tar.gz/sha512/92bbb42a742427d4a4bfb67530191093ee55a210a99abe0145bdf4ccdc3e583a9852a8045dfdd20002a8fd8eaf9feb6245f84a4995fee4a097a33a2b64998dcd +LibUnwind.v1.8.2+0.armv6l-linux-gnueabihf.tar.gz/md5/a6eff40a5bef97c13fed1546dd77a503 +LibUnwind.v1.8.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/b0e687036c080cf5a421a924e070e57d9f52cee1e67e6c9558d586edc58296d15c9e10e2e46c27e6d290c866b84edeee11023077fb2481dfc9f08fd08540326a +LibUnwind.v1.8.2+0.armv6l-linux-musleabihf.tar.gz/md5/19f8f17923dc1b87b3e89480b1eeaedb +LibUnwind.v1.8.2+0.armv6l-linux-musleabihf.tar.gz/sha512/23251a9fe459242589797a364ff69e38711ebf93d4f047acc7d302c8bc669d3b0546a318568787f0f9bf4d1e633670e0941887cbac1c34e34d86e217040533af +LibUnwind.v1.8.2+0.armv7l-linux-gnueabihf.tar.gz/md5/74bd5a25811e4977386ba3aefe98bd8c +LibUnwind.v1.8.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/2d41f653640ffaebb7295d49a533efebf2ffca7314a181e6e669c399e65b49d176915afeb60b4702632d65aa20aad772052373efbd3c6b9fdc0c9fd772986b2a +LibUnwind.v1.8.2+0.armv7l-linux-musleabihf.tar.gz/md5/7d882db1897c88c7e2eb7cf94e3b3f9e +LibUnwind.v1.8.2+0.armv7l-linux-musleabihf.tar.gz/sha512/dad5bdbccd630eb27af69a32f01245865fc9cf494f1e2ab4cf9e06d78985d997accbac7db6c947bb1964f0aa98f7c977cdb9288e4453fa22d47eb99d73144536 +LibUnwind.v1.8.2+0.i686-linux-gnu.tar.gz/md5/f75ff3a516d6849ed583b66a25ae3784 +LibUnwind.v1.8.2+0.i686-linux-gnu.tar.gz/sha512/8a9ab5729cfb52d3f8e0a1d7f6ef40566fd03fd1aa7f94269a3f6acd7c3cc969c008be97a91d7fa982ff5deb2db64d39bac304a948bc3bd07795a5811a41d0b1 +LibUnwind.v1.8.2+0.i686-linux-musl.tar.gz/md5/e9e479c47ea9f1ec297d040ef058dfc4 +LibUnwind.v1.8.2+0.i686-linux-musl.tar.gz/sha512/ac945a97288179d34f96b228ca631b78218b83e69320fd7c4516a5770b32078b184798e070ab87dabe89370fd7bd1657c43a20c543bc73a146e8003806f65852 +LibUnwind.v1.8.2+0.powerpc64le-linux-gnu.tar.gz/md5/268f72d03fd4cad65fe9913074d05226 +LibUnwind.v1.8.2+0.powerpc64le-linux-gnu.tar.gz/sha512/533d701bc78adc725626822fbe75002560198381a695373bd61805e6773de2a6ec4314f56d3d414cb9c1564da1ccb519d50eb3fa3aa99a9fd03306859ad8effe +LibUnwind.v1.8.2+0.riscv64-linux-gnu.tar.gz/md5/bee9019f3d9cc90ff2dfda358f7ffd92 +LibUnwind.v1.8.2+0.riscv64-linux-gnu.tar.gz/sha512/4b4b3a52af26e069184b3a273427da3e5625a2ea77a9baa6ea0efe17c0b32d6c4a49610ef6554ecc594c0c76572ed2da80860c7c2cafe2c0a2b9341d6455f409 +LibUnwind.v1.8.2+0.x86_64-linux-gnu.tar.gz/md5/2688969e445093554ff7545708f94b9d +LibUnwind.v1.8.2+0.x86_64-linux-gnu.tar.gz/sha512/64461eb5c0f00ba391933b16a0c5cbca9b59056982a04c26bb81fb60c63d728a909ab11190a1720de17bc8587e1a5c79edb100d6c5ff077aff1e35b53383970a +LibUnwind.v1.8.2+0.x86_64-linux-musl.tar.gz/md5/3c1d3abe07a178b0463c8f83c09ee190 +LibUnwind.v1.8.2+0.x86_64-linux-musl.tar.gz/sha512/32919df2e57562c9c85c22586c7d6f35227c360ef7cf0ec0535f876a36fcba862f9874d7fd627b1e6e25a85d04279676aa5192600aa92ec78c00d0b07304897f +LibUnwind.v1.8.2+0.x86_64-unknown-freebsd.tar.gz/md5/69612f0c88883aa9713fa8b90022654c +LibUnwind.v1.8.2+0.x86_64-unknown-freebsd.tar.gz/sha512/3a04384cfc3c38ebc3603c512228d5e50bb0313a2712721a2b60707b4b477c8959aa07aadb54f1b25c9002e09a29407c8bf8370aa4c5dbd7c9e71110c3bea14b +libunwind-1.8.2.tar.gz/md5/0124a38fb752aa5492635f35d089f6b7 +libunwind-1.8.2.tar.gz/sha512/f1ff26763c1b2e68948413c4aec22303b6c886425a8264eb65fbd58fc202f79c7b04bd4784bd8499850d08933f0e363cfa3a7d177efdadc223ed0254bc381345 diff --git a/deps/patches/libunwind-aarch64-inline-asm.patch b/deps/patches/libunwind-aarch64-inline-asm.patch deleted file mode 100644 index 123643e30cdeb..0000000000000 --- a/deps/patches/libunwind-aarch64-inline-asm.patch +++ /dev/null @@ -1,157 +0,0 @@ -From 6ae71b3ea71bff0f38c7a6a05beda30b7dce1ef6 Mon Sep 17 00:00:00 2001 -From: Stephen Webb -Date: Mon, 22 Apr 2024 15:56:54 -0400 -Subject: [PATCH] Rework inline aarch64 as for setcontext - -Modern GC and clang were barfing on the inline asm constraints for the -aarch64-linux setcontext() replacement. Reformulated the asm code to -reduce the required constraints. ---- - src/aarch64/Gos-linux.c | 115 +++++++++++++++++++++------------------- - 1 file changed, 61 insertions(+), 54 deletions(-) - -diff --git a/src/aarch64/Gos-linux.c b/src/aarch64/Gos-linux.c -index 7cd8c879f..1e4949623 100644 ---- a/src/aarch64/Gos-linux.c -+++ b/src/aarch64/Gos-linux.c -@@ -2,6 +2,7 @@ - Copyright (C) 2008 CodeSourcery - Copyright (C) 2011-2013 Linaro Limited - Copyright (C) 2012 Tommi Rantala -+ Copyright 2024 Stephen M. Webb - - This file is part of libunwind. - -@@ -28,6 +29,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - - #ifndef UNW_REMOTE_ONLY - -+/* Magic constants generated from gen-offsets.c */ -+#define SC_R0_OFF "8" -+#define SC_R2_OFF "24" -+#define SC_R18_OFF "152" -+#define SC_R20_OFF "168" -+#define SC_R22_OFF "184" -+#define SC_R24_OFF "200" -+#define SC_R26_OFF "216" -+#define SC_R28_OFF "232" -+#define SC_R30_OFF "248" -+ -+#define FP_R08_OFF "80" -+#define FP_R09_OFF "88" -+#define FP_R10_OFF "96" -+#define FP_R11_OFF "104" -+#define FP_R12_OFF "112" -+#define FP_R13_OFF "120" -+#define FP_R14_OFF "128" -+#define FP_R15_OFF "136" -+ -+#define SC_SP_OFF "0x100" -+ - HIDDEN int - aarch64_local_resume (unw_addr_space_t as, unw_cursor_t *cursor, void *arg) - { -@@ -36,65 +59,49 @@ aarch64_local_resume (unw_addr_space_t as, unw_cursor_t *cursor, void *arg) - - if (c->sigcontext_format == AARCH64_SCF_NONE) - { -+ -+ /* -+ * This is effectively the old POSIX setcontext(). -+ * -+ * This inline asm is broken up to use local scratch registers for the -+ * uc_mcontext.regs and FPCTX base addresses because newer versions of GCC -+ * and clang barf on too many constraints (gh-702) when the C array -+ * elements are used directly. -+ * -+ * Clobbers aren't required for the inline asm because they just convince -+ * the compiler to save those registers and they never get restored -+ * becauise the asm ends with a plain ol' ret. -+ */ -+ register void* uc_mcontext __asm__ ("x5") = (void*) &uc->uc_mcontext; -+ register void* fpctx __asm__ ("x4") = (void*) GET_FPCTX(uc); -+ - /* Since there are no signals involved here we restore EH and non scratch - registers only. */ - __asm__ __volatile__ ( -- "ldr x0, %[x0]\n\t" -- "ldr x1, %[x1]\n\t" -- "ldr x2, %[x2]\n\t" -- "ldr x3, %[x3]\n\t" -- "ldr x19, %[x19]\n\t" -- "ldr x20, %[x20]\n\t" -- "ldr x21, %[x21]\n\t" -- "ldr x22, %[x22]\n\t" -- "ldr x23, %[x23]\n\t" -- "ldr x24, %[x24]\n\t" -- "ldr x25, %[x25]\n\t" -- "ldr x26, %[x26]\n\t" -- "ldr x27, %[x27]\n\t" -- "ldr x28, %[x28]\n\t" -- "ldr x29, %[x29]\n\t" -- "ldr x30, %[x30]\n\t" -- "ldr d8, %[d8]\n\t" -- "ldr d9, %[d9]\n\t" -- "ldr d10, %[d10]\n\t" -- "ldr d11, %[d11]\n\t" -- "ldr d12, %[d12]\n\t" -- "ldr d13, %[d13]\n\t" -- "ldr d14, %[d14]\n\t" -- "ldr d15, %[d15]\n\t" -- "ldr x5, %[sp]\n\t" -+ "ldp x0, x1, [x5, " SC_R0_OFF "]\n\t" -+ "ldp x2, x3, [x5, " SC_R2_OFF "]\n\t" -+ "ldp x18, x19, [x5, " SC_R18_OFF "]\n\t" -+ "ldp x20, x21, [x5, " SC_R20_OFF "]\n\t" -+ "ldp x22, x23, [x5, " SC_R22_OFF "]\n\t" -+ "ldp x24, x25, [x5, " SC_R24_OFF "]\n\t" -+ "ldp x26, x27, [x5, " SC_R26_OFF "]\n\t" -+ "ldp x28, x29, [x5, " SC_R28_OFF "]\n\t" -+ "ldr x30, [x5, " SC_R30_OFF "]\n\t" -+ "ldr d8, [x4, " FP_R08_OFF "]\n\t" -+ "ldr d9, [x4, " FP_R09_OFF "]\n\t" -+ "ldr d10, [x4, " FP_R10_OFF "]\n\t" -+ "ldr d11, [x4, " FP_R11_OFF "]\n\t" -+ "ldr d12, [x4, " FP_R12_OFF "]\n\t" -+ "ldr d13, [x4, " FP_R13_OFF "]\n\t" -+ "ldr d14, [x4, " FP_R14_OFF "]\n\t" -+ "ldr d15, [x4, " FP_R15_OFF "]\n\t" -+ "ldr x5, [x5, " SC_SP_OFF "]\n\t" - "mov sp, x5\n\t" - "ret\n" -- : -- : [x0] "m"(uc->uc_mcontext.regs[0]), -- [x1] "m"(uc->uc_mcontext.regs[1]), -- [x2] "m"(uc->uc_mcontext.regs[2]), -- [x3] "m"(uc->uc_mcontext.regs[3]), -- [x19] "m"(uc->uc_mcontext.regs[19]), -- [x20] "m"(uc->uc_mcontext.regs[20]), -- [x21] "m"(uc->uc_mcontext.regs[21]), -- [x22] "m"(uc->uc_mcontext.regs[22]), -- [x23] "m"(uc->uc_mcontext.regs[23]), -- [x24] "m"(uc->uc_mcontext.regs[24]), -- [x25] "m"(uc->uc_mcontext.regs[25]), -- [x26] "m"(uc->uc_mcontext.regs[26]), -- [x27] "m"(uc->uc_mcontext.regs[27]), -- [x28] "m"(uc->uc_mcontext.regs[28]), -- [x29] "m"(uc->uc_mcontext.regs[29]), /* FP */ -- [x30] "m"(uc->uc_mcontext.regs[30]), /* LR */ -- [d8] "m"(GET_FPCTX(uc)->vregs[8]), -- [d9] "m"(GET_FPCTX(uc)->vregs[9]), -- [d10] "m"(GET_FPCTX(uc)->vregs[10]), -- [d11] "m"(GET_FPCTX(uc)->vregs[11]), -- [d12] "m"(GET_FPCTX(uc)->vregs[12]), -- [d13] "m"(GET_FPCTX(uc)->vregs[13]), -- [d14] "m"(GET_FPCTX(uc)->vregs[14]), -- [d15] "m"(GET_FPCTX(uc)->vregs[15]), -- [sp] "m"(uc->uc_mcontext.sp) -- : "x0", "x1", "x2", "x3", "x19", "x20", "x21", "x22", "x23", "x24", -- "x25", "x26", "x27", "x28", "x29", "x30" -- ); -+ : -+ : [uc_mcontext] "r"(uc_mcontext), -+ [fpctx] "r"(fpctx) -+ ); - } - else - { diff --git a/deps/patches/libunwind-configure-static-lzma.patch b/deps/patches/libunwind-configure-static-lzma.patch index f8b428f60550b..16991d5cd2656 100644 --- a/deps/patches/libunwind-configure-static-lzma.patch +++ b/deps/patches/libunwind-configure-static-lzma.patch @@ -1,20 +1,20 @@ ---- configure.orig 2023-06-04 05:19:04 -+++ configure 2023-06-07 08:35:11 -@@ -18117,7 +18117,7 @@ - $as_echo_n "(cached) " >&6 - else +--- configure.orig 2025-05-26 15:25:01 ++++ configure 2025-05-26 15:25:41 +@@ -20878,7 +20878,7 @@ + printf %s "(cached) " >&6 + else $as_nop ac_check_lib_save_LIBS=$LIBS -LIBS="-llzma $LIBS" -+LIBS="-L${libdir} -l:liblzma.a $LIBS" ++LIBS="-L${libdir} -l:liblzma.a $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -@@ -18148,7 +18148,7 @@ - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_mf_is_supported" >&5 - $as_echo "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; } - if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes; then : +@@ -20908,7 +20908,7 @@ + printf "%s\n" "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; } + if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes + then : - LIBLZMA=-llzma + LIBLZMA="-L${libdir} -l:liblzma.a" - $as_echo "#define HAVE_LZMA 1" >>confdefs.h + printf "%s\n" "#define HAVE_LZMA 1" >>confdefs.h diff --git a/deps/unwind.mk b/deps/unwind.mk index d144f4d0a509b..08d0059c89731 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -35,11 +35,7 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applie cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-revert_prelink_unwind.patch echo 1 > $@ -$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-aarch64-inline-asm.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applied - cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-aarch64-inline-asm.patch - echo 1 > $@ - -$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-aarch64-inline-asm.patch-applied +$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applied cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-disable-initial-exec-tls.patch echo 1 > $@ diff --git a/deps/unwind.version b/deps/unwind.version index e3ed63675fd8c..d2709230d18fc 100644 --- a/deps/unwind.version +++ b/deps/unwind.version @@ -1,6 +1,8 @@ +# -*- makefile -*- + ## jll artifact UNWIND_JLL_NAME := LibUnwind ## source build -UNWIND_VER_TAG := 1.8.1 -UNWIND_VER := 1.8.1 +UNWIND_VER_TAG := 1.8.2 +UNWIND_VER := 1.8.2 diff --git a/stdlib/LibUnwind_jll/Project.toml b/stdlib/LibUnwind_jll/Project.toml index 8e6ba70a3deb3..5643f0989f6da 100644 --- a/stdlib/LibUnwind_jll/Project.toml +++ b/stdlib/LibUnwind_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibUnwind_jll" uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" -version = "1.8.1+2" +version = "1.8.2+0" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" From 9da64559e3046cf9215c1cc752d301e735996473 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 2 Jun 2025 03:14:22 +0200 Subject: [PATCH 353/662] fix Markdown in the `Lockable` doc strings (#58603) --- base/lock.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/lock.jl b/base/lock.jl index 993b771d73f1e..5fca3a982f22d 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -398,7 +398,7 @@ macro lock_nofail(l, expr) end """ - Lockable(value, lock = ReentrantLock()) + Lockable(value, lock = ReentrantLock()) Creates a `Lockable` object that wraps `value` and associates it with the provided `lock`. This object @@ -431,7 +431,7 @@ Lockable(value) = Lockable(value, ReentrantLock()) getindex(l::Lockable) = (assert_havelock(l.lock); l.value) """ - lock(f::Function, l::Lockable) + lock(f::Function, l::Lockable) Acquire the lock associated with `l`, execute `f` with the lock held, and release the lock when `f` returns. `f` will receive one positional From a4c58c80c97b776f6f2e827610daced9656eecd6 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 2 Jun 2025 04:05:18 +0200 Subject: [PATCH 354/662] Clarify documentation around `close`, `flush`, and `isopen` (#58020) Co-authored-by: Max Horn --- base/asyncevent.jl | 15 ++++++++++++--- base/io.jl | 28 +++++++++++++++++++--------- base/iostream.jl | 10 ++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/base/asyncevent.jl b/base/asyncevent.jl index 8297626ee0f97..13b3a94580fec 100644 --- a/base/asyncevent.jl +++ b/base/asyncevent.jl @@ -9,7 +9,8 @@ Create a async condition that wakes up tasks waiting for it (by calling [`wait`](@ref) on the object) when notified from C by a call to `uv_async_send`. Waiting tasks are woken with an error when the object is closed (by [`close`](@ref)). -Use [`isopen`](@ref) to check whether it is still active. +Use [`isopen`](@ref) to check whether it is still active. A closed condition is inactive and will +not wake up tasks. This provides an implicit acquire & release memory ordering between the sending and waiting threads. """ @@ -74,8 +75,8 @@ Create a timer that wakes up tasks waiting for it (by calling [`wait`](@ref) on Waiting tasks are woken after an initial delay of at least `delay` seconds, and then repeating after at least `interval` seconds again elapse. If `interval` is equal to `0`, the timer is only triggered once. When the timer is closed (by [`close`](@ref)) waiting tasks are woken with an error. Use -[`isopen`](@ref) to check whether a timer is still active. Use `t.timeout` and `t.interval` to read -the setup conditions of a `Timer` `t`. +[`isopen`](@ref) to check whether a timer is still active. An inactive timer will not fire. +Use `t.timeout` and `t.interval` to read the setup conditions of a `Timer` `t`. ```julia-repl julia> t = Timer(1.0; interval=0.5) @@ -206,6 +207,14 @@ end isopen(t::Union{Timer, AsyncCondition}) = @atomic :acquire t.isopen +""" + close(t::Union{Timer, AsyncCondition}) + +Close an object `t` and thus mark it as inactive. Once a timer or condition is inactive, it will not produce +a new event. + +See also: [`isopen`](@ref) +""" function close(t::Union{Timer, AsyncCondition}) t.handle == C_NULL && !t.isopen && return # short-circuit path, :monotonic iolock_begin() diff --git a/base/io.jl b/base/io.jl index c0e1740c5bd11..71e4c404dfeac 100644 --- a/base/io.jl +++ b/base/io.jl @@ -41,11 +41,9 @@ buffer_writes(x::IO, bufsize=SZ_UNBUFFERED_IO) = x """ isopen(object)::Bool -Determine whether an object - such as a stream or timer --- is not yet closed. Once an object is closed, it will never produce a new event. -However, since a closed stream may still have data to read in its buffer, -use [`eof`](@ref) to check for the ability to read data. -Use the `FileWatching` package to be notified when a stream might be writable or readable. +Determine whether an object, such as an IO or timer, is still open and hence active. + +See also: [`close`](@ref) # Examples ```jldoctest @@ -63,9 +61,19 @@ false function isopen end """ - close(stream) + close(io::IO) + +Close `io`. Performs a [`flush`](@ref) first. + +Closing an IO signals that its underlying resources (OS handle, network +connections, etc) should be destroyed. +A closed IO is in an undefined state and should not be written to or read from. +When attempting to do so, the IO may throw an exception, continue to behave +normally, or read/write zero bytes, depending on the implementation. +However, implementations should make sure that reading to or writing from a +closed IO does not cause undefined behaviour. -Close an I/O stream. Performs a [`flush`](@ref) first. +See also: [`isopen`](@ref) """ function close end @@ -97,9 +105,11 @@ julia> read(io, String) function closewrite end """ - flush(stream) + flush(io::IO) -Commit all currently buffered writes to the given stream. +Commit all currently buffered writes to the given io. +This has a default implementation `flush(::IO) = nothing`, so may be called +in generic IO code. """ function flush end diff --git a/base/iostream.jl b/base/iostream.jl index 8847c1b2d8ccd..c58818968c0b1 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -75,6 +75,16 @@ fd(s::IOStream) = RawFD(ccall(:jl_ios_fd, Clong, (Ptr{Cvoid},), s.ios)) stat(s::IOStream) = stat(fd(s)) +""" + isopen(s::IOStream) + +Check if the stream is not yet closed. + +A closed `IOStream` may still have data to read in its buffer, +use [`eof`](@ref) to check for the ability to read data. + +Use the `FileWatching` package to be notified when a file might be writable or readable. +""" isopen(s::IOStream) = ccall(:ios_isopen, Cint, (Ptr{Cvoid},), s.ios) != 0 function close(s::IOStream) From 4f2f6bef1db89f33f9e78612dfab091af12917d4 Mon Sep 17 00:00:00 2001 From: SundaraRaman R Date: Mon, 2 Jun 2025 20:22:18 +0530 Subject: [PATCH 355/662] tiny PR to fix writeshortest for 0.0 when hash==false (#58502) Currently, ```julia-repl julia> Base.Ryu.writeshortest(0.0, false, false, false) "00" ``` The second 0 is intended to be printed when the last argument `hash` is `true` and a decimal point is printed, to follow that `.` (i.e. `"0.0"`). But it's currently printed also when `hash` is `false`, resulting in this function not printing the actual "shortest" numeric string for this case. (issue first noted by Dave MacMahon on Slack) --- base/ryu/shortest.jl | 6 ++++-- test/ryu.jl | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index c1ec648bfacdd..37024f18b4df1 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -250,8 +250,10 @@ function writeshortest(buf::AbstractVector{UInt8}, pos, x::T, pos += 1 end if precision == -1 - @inbounds buf[pos] = UInt8('0') - pos += 1 + if hash + @inbounds buf[pos] = UInt8('0') + pos += 1 + end if typed && x isa Float32 @inbounds buf[pos] = UInt8('f') @inbounds buf[pos + 1] = UInt8('0') diff --git a/test/ryu.jl b/test/ryu.jl index 05eedef9a0da2..e885d6c10838f 100644 --- a/test/ryu.jl +++ b/test/ryu.jl @@ -19,6 +19,34 @@ todouble(sign, exp, mant) = Core.bitcast(Float64, (UInt64(sign) << 63) | (UInt64 @test Ryu.writeshortest(-Inf) == "-Inf" end +@testset "OutputOptions" begin + # plus + @test "+1" == Base.Ryu.writeshortest(1.0, true, false, false) + @test "-1" == Base.Ryu.writeshortest(-1.0, true, false, false) + + # space + @test " 1" == Ryu.writeshortest(1.0, false, true, false) + + # hash + @test "0" == Ryu.writeshortest(0.0, false, false, false) + + # precision + @test "9.9900" == Ryu.writeshortest(9.99, false, false, true, 5) + @test "1." == Ryu.writeshortest(1.0, false, false, true, 1) + + # expchar + @test "1.0d6" == Ryu.writeshortest(1e6, false, false, true, -1, UInt8('d')) + + # padexp + @test "3.0e+08" == Ryu.writeshortest(3e8, false, false, true, -1, UInt8('e'), true) + + # decchar + @test "3,14" == Ryu.writeshortest(3.14, false, false, true, -1, UInt8('e'), false, UInt8(',')) + + # compact + @test "0.333333" == Ryu.writeshortest(1/3, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, true) +end + @testset "SwitchToSubnormal" begin @test "2.2250738585072014e-308" == Ryu.writeshortest(2.2250738585072014e-308) end @@ -241,6 +269,17 @@ end # Float64 @test "-Inf" == Ryu.writeshortest(Float32(-Inf)) end +@testset "OutputOptions" begin + # typed + @test "1.0f0" == Ryu.writeshortest(Float32(1.0), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Inf32" == Ryu.writeshortest(Float32(Inf), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "NaN32" == Ryu.writeshortest(Float32(NaN), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "3.14f0" == Ryu.writeshortest(Float32(3.14), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + + # typed and no-hash + @test "1f0" == Ryu.writeshortest(1.0f0, false, false, false, -1, UInt8('e'), false, UInt8('.'), true) +end + @testset "SwitchToSubnormal" begin @test "1.1754944e-38" == Ryu.writeshortest(1.1754944f-38) end @@ -341,6 +380,17 @@ end # Float32 @test "-Inf" == Ryu.writeshortest(Float16(-Inf)) end +@testset "OutputOptions" begin + # typed + @test "Float16(1.0)" == Ryu.writeshortest(Float16(1.0), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Inf16" == Ryu.writeshortest(Float16(Inf), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "NaN16" == Ryu.writeshortest(Float16(NaN), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Float16(3.14)" == Ryu.writeshortest(Float16(3.14), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + + # typed and no-hash + @test "Float16(1)" == Ryu.writeshortest(Float16(1.0), false, false, false, -1, UInt8('e'), false, UInt8('.'), true) +end + let x=floatmin(Float16) while x <= floatmax(Float16) @test parse(Float16, Ryu.writeshortest(x)) == x From ce0d920acf5e2f3884755f2ba03b793d10686363 Mon Sep 17 00:00:00 2001 From: ubertakter Date: Mon, 2 Jun 2025 10:53:57 -0400 Subject: [PATCH 356/662] Add reminder about docstrings, objects, and intervening lines (#58541) Added a specific call out (as a reminder note) that no lines are allowed between docstrings and the object being documented. This is strictly an addition, with none of the current text being changed. --------- Co-authored-by: Neven Sajko <4944410+nsajko@users.noreply.github.com> --- doc/src/manual/documentation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/src/manual/documentation.md b/doc/src/manual/documentation.md index 2c32b57fad38d..58f2de1fe2680 100644 --- a/doc/src/manual/documentation.md +++ b/doc/src/manual/documentation.md @@ -37,6 +37,8 @@ the documented object. Here is a basic example: "Tell whether there are too foo items in the array." foo(xs::Array) = ... ``` +!!! note "Reminder" + Any empty lines between the docstring and the object being documented detach the former from the latter, making the docstring ineffective. Documentation is interpreted as [Markdown](https://en.wikipedia.org/wiki/Markdown), so you can use indentation and code fences to delimit code examples from text. Technically, any object can From 6f129571004beac33a5cbc4aa7cbb6fcca4f5769 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Mon, 2 Jun 2025 10:55:07 -0400 Subject: [PATCH 357/662] Re-enable tab completion of kwargs for large method tables (#58012) while testing to ensure that ~~absurdly large method tables~~ tab completing over an abstract function call doesn't tank the performance of the REPL Fixes #57836 --- stdlib/REPL/src/REPLCompletions.jl | 8 +++++++- stdlib/REPL/test/replcompletions.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 1a9556e74c00f..d7b0b9c6d0b89 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -888,7 +888,13 @@ function complete_keyword_argument!(suggestions::Vector{Completion}, kwargs_flag == 2 && return false # one of the previous kwargs is invalid methods = Completion[] - complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, shift ? -1 : MAX_METHOD_COMPLETIONS, arg_pos == :kwargs) + # Limit kwarg completions to cases when function is concretely known; looking up + # matching methods for abstract functions — particularly `Any` or `Function` — can + # take many seconds to run over the thousands of possible methods. Note that + # isabstracttype would return naively return true for common constructor calls + # like Array, but the REPL's introspection here may know their Type{T}. + isconcretetype(funct) || return false + complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, -1, arg_pos == :kwargs) # TODO: use args_ex instead of Any[Vararg{Any}] and only provide kwarg completion for # method calls compatible with the current arguments. diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 047a53edac1e5..5f2990e970d97 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -2632,6 +2632,35 @@ const issue57780_orig = copy(issue57780) test_complete_context("empty!(issue57780).", Main) @test issue57780 == issue57780_orig +function g54131 end +for i in 1:498 + @eval g54131(::Val{$i}) = i +end +g54131(::Val{499}; kwarg=true) = 499*kwarg +struct F54131; end +Base.getproperty(::F54131, ::Symbol) = Any[cos, sin, g54131][rand(1:3)] +f54131 = F54131() +@testset "performance of kwarg completion with large method tables" begin + # The goal here is to simply ensure we aren't hitting catestrophically bad + # behaviors when shift isn't pressed. The difference between good and bad + # is on the order of tens of milliseconds vs tens of seconds; using 1 sec as + # a very rough canary that is hopefully robust even in the noisy CI coalmines + s = "g54131(kwa" + a, b, c = completions(s, lastindex(s), @__MODULE__, #= shift =# false) + @test REPLCompletions.KeywordArgumentCompletion("kwarg") in a + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 + + s = "f54131.x(" + a, b, c = completions(s, lastindex(s), @__MODULE__, false) + @test only(a) isa REPLCompletions.TextCompletion + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 + + s = "f54131.x(kwa" + a, b, c = completions(s, lastindex(s), @__MODULE__, false) + @test_broken REPLCompletions.KeywordArgumentCompletion("kwarg") in a + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 +end + # Completion inside string interpolation let s = "\"example: \$varflo" c, r = test_complete_foo(s) From f12256bb545a4e7bc81bb1235b3ecf1115b7497d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 2 Jun 2025 14:45:27 -0400 Subject: [PATCH 358/662] fix `@invokelatest` performance regression (#58582) A bit of hacking to get back near to the same performance as before by using the GlobalRef to optimize the getglobal lookup for now and avoiding the extra Vararg function indirection which forced some extra boxing and lookups. julia> @btime foo(1.5) 22.892 ns (1 allocation: 16 bytes) # v1.11 141.543 ns (3 allocations: 48 bytes) # master 38.759 ns (2 allocations: 32 bytes) # PR The remaining difference is split about equally between the need now to box the world counter value for invoke_in_world and the extra cost of scanning the partition table for `Base.sin` to find the current entry. Fix #58334 --- base/reflection.jl | 11 ++++++----- src/ast.c | 2 +- src/builtins.c | 2 +- src/gf.c | 6 +++--- src/init.c | 4 ++-- src/interpreter.c | 12 ++++++------ src/jl_uv.c | 3 ++- src/jlapi.c | 2 +- src/jltypes.c | 1 + src/julia.h | 2 -- src/julia_internal.h | 5 ++++- src/module.c | 12 ++++++------ src/precompile.c | 4 ++-- src/rtutils.c | 5 +++-- src/scheduler.c | 2 +- src/staticdata_utils.c | 14 +++++++------- src/task.c | 2 +- src/threading.c | 2 +- src/toplevel.c | 37 ++++++++++++++++++++----------------- 19 files changed, 68 insertions(+), 60 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index 304683639b6d3..edc1afc8a6aaf 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1269,16 +1269,17 @@ macro invoke(ex) return esc(out) end -apply_gr(gr::GlobalRef, @nospecialize args...) = getglobal(gr.mod, gr.name)(args...) -apply_gr_kw(@nospecialize(kwargs::NamedTuple), gr::GlobalRef, @nospecialize args...) = Core.kwcall(kwargs, getglobal(gr.mod, gr.name), args...) +getglobalref(gr::GlobalRef, world::UInt) = ccall(:jl_eval_globalref, Any, (Any, UInt), gr, world) -function invokelatest_gr(gr::GlobalRef, @nospecialize args...; kwargs...) +function invokelatest_gr(gr::GlobalRef, args...; kwargs...) @inline kwargs = merge(NamedTuple(), kwargs) + world = get_world_counter() + f = getglobalref(gr, world) if isempty(kwargs) - return invokelatest(apply_gr, gr, args...) + return invoke_in_world(world, f, args...) end - return invokelatest(apply_gr_kw, kwargs, gr, args...) + return invoke_in_world(world, Core.kwcall, kwargs, f, args...) end """ diff --git a/src/ast.c b/src/ast.c index 8f36cd06decad..fdb29349a0db6 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1328,7 +1328,7 @@ JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, { jl_value_t *core_lower = NULL; if (jl_core_module) - core_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower")); + core_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower"), jl_current_task->world_age); if (!core_lower || core_lower == jl_nothing) { return jl_fl_lower(expr, inmodule, filename, line, world, warn); } diff --git a/src/builtins.c b/src/builtins.c index 0e309ab912f9c..46fceb91f86e7 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1340,7 +1340,7 @@ JL_CALLABLE(jl_f_getglobal) jl_atomic_error("getglobal: module binding cannot be read non-atomically"); else if (order >= jl_memory_order_seq_cst) jl_fence(); - jl_value_t *v = jl_eval_global_var(mod, sym); // relaxed load + jl_value_t *v = jl_eval_global_var(mod, sym, jl_current_task->world_age); // relaxed load if (order >= jl_memory_order_acquire) jl_fence(); return v; diff --git a/src/gf.c b/src/gf.c index 6e2403826b7f1..8205cf70b99c3 100644 --- a/src/gf.c +++ b/src/gf.c @@ -512,13 +512,13 @@ JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t w ct->world_age = jl_typeinf_world; jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 4); - jl_module_t *CC = (jl_module_t*)jl_get_global_value(jl_core_module, jl_symbol("Compiler")); + jl_module_t *CC = (jl_module_t*)jl_get_global_value(jl_core_module, jl_symbol("Compiler"), ct->world_age); if (CC != NULL && jl_is_module(CC)) { JL_GC_PROMISE_ROOTED(CC); - fargs[0] = jl_get_global_value(CC, jl_symbol("NativeInterpreter"));; + fargs[0] = jl_get_global_value(CC, jl_symbol("NativeInterpreter"), ct->world_age); fargs[1] = jl_box_ulong(world); fargs[1] = jl_apply(fargs, 2); - fargs[0] = jl_get_global_value(CC, jl_symbol("typeinf_code")); + fargs[0] = jl_get_global_value(CC, jl_symbol("typeinf_code"), ct->world_age); fargs[2] = (jl_value_t*)mi; fargs[3] = jl_true; ci = (jl_code_info_t*)jl_apply(fargs, 4); diff --git a/src/init.c b/src/init.c index f811cf76748fb..a64f43c62f776 100644 --- a/src/init.c +++ b/src/init.c @@ -249,7 +249,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) JL_NOTSAFEPOINT_ENTER if (jl_base_module) { size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); - jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_atexit")); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_atexit"), ct->world_age); if (f != NULL) { jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 2); @@ -357,7 +357,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) jl_task_t *ct = jl_get_current_task(); size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); - jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_postoutput")); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_postoutput"), ct->world_age); if (f != NULL) { JL_TRY { JL_GC_PUSH1(&f); diff --git a/src/interpreter.c b/src/interpreter.c index aa89f7385532f..c4de34da37ce9 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -162,18 +162,18 @@ static jl_value_t *do_invoke(jl_value_t **args, size_t nargs, interpreter_state } // get the global (throwing if null) in the current world -jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e) +jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e, size_t world) { - jl_value_t *v = jl_get_global_value(m, e); + jl_value_t *v = jl_get_global_value(m, e, world); if (v == NULL) jl_undefined_var_error(e, (jl_value_t*)m); return v; } // get the global (throwing if null) in the current world, optimized -jl_value_t *jl_eval_globalref(jl_globalref_t *g) +jl_value_t *jl_eval_globalref(jl_globalref_t *g, size_t world) { - jl_value_t *v = jl_get_globalref_value(g); + jl_value_t *v = jl_get_globalref_value(g, world); if (v == NULL) jl_undefined_var_error(g->name, (jl_value_t*)g->mod); return v; @@ -218,10 +218,10 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) return jl_quotenode_value(e); } if (jl_is_globalref(e)) { - return jl_eval_globalref((jl_globalref_t*)e); + return jl_eval_globalref((jl_globalref_t*)e, jl_current_task->world_age); } if (jl_is_symbol(e)) { // bare symbols appear in toplevel exprs not wrapped in `thunk` - return jl_eval_global_var(s->module, (jl_sym_t*)e); + return jl_eval_global_var(s->module, (jl_sym_t*)e, jl_current_task->world_age); } if (jl_is_pinode(e)) { jl_value_t *val = eval_value(jl_fieldref_noalloc(e, 0), s); diff --git a/src/jl_uv.c b/src/jl_uv.c index 005d1ea727655..a21b05433b8c6 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -162,7 +162,8 @@ static void jl_uv_call_close_callback(jl_value_t *val) JL_GC_PUSHARGS(args, 2); // val is "rooted" in the finalizer list only right now args[0] = jl_eval_global_var( jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), - jl_symbol("_uv_hook_close")); // topmod(typeof(val))._uv_hook_close + jl_symbol("_uv_hook_close"), + jl_current_task->world_age); // topmod(typeof(val))._uv_hook_close args[1] = val; jl_apply(args, 2); // TODO: wrap in try-catch? JL_GC_POP(); diff --git a/src/jlapi.c b/src/jlapi.c index e127fd1367816..0d8ddd10d9ea1 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -955,7 +955,7 @@ static NOINLINE int true_main(int argc, char *argv[]) ct->world_age = jl_get_world_counter(); jl_function_t *start_client = jl_base_module ? - (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("_start")) : NULL; + (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("_start"), ct->world_age) : NULL; if (start_client) { int ret = 1; diff --git a/src/jltypes.c b/src/jltypes.c index fcb15a1a9a69d..7d731082bd786 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3287,6 +3287,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_perm_symsvec(3, "mod", "name", "binding"), jl_svec(3, jl_module_type, jl_symbol_type, jl_binding_type), jl_emptysvec, 0, 0, 3); + jl_globalref_type->name->mayinlinealloc = 0; // not at all worthwhile, since the only constructor returns a boxed object jl_core_module = jl_new_module(jl_symbol("Core"), NULL); diff --git a/src/julia.h b/src/julia.h index 4a4a4eff81232..03a0185800d12 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2094,9 +2094,7 @@ JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b JL_PROPAGATES JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr); -JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); -JL_DLLEXPORT jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported); diff --git a/src/julia_internal.h b/src/julia_internal.h index b94011e17a3e3..24fb7fdb05f90 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -901,7 +901,10 @@ STATIC_INLINE size_t module_usings_max(jl_module_t *m) JL_NOTSAFEPOINT { JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) JL_NOTSAFEPOINT; void jl_add_scanned_method(jl_module_t *m, jl_method_t *meth); -jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e); +jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e, size_t world); +JL_DLLEXPORT jl_value_t *jl_eval_globalref(jl_globalref_t *g, size_t world); +jl_value_t *jl_get_globalref_value(jl_globalref_t *gr, size_t world); +jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var, size_t world); jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *clos, jl_value_t **args, size_t nargs); jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src); jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e, diff --git a/src/module.c b/src/module.c index f154b7d29db8e..8c6c86b144f38 100644 --- a/src/module.c +++ b/src/module.c @@ -1540,20 +1540,20 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, } -// get the value (or null) in the current world -JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr) +// get the value (or null) in the world +jl_value_t *jl_get_globalref_value(jl_globalref_t *gr, size_t world) { jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - return jl_get_binding_value_depwarn(b, jl_current_task->world_age); + return jl_get_binding_value_depwarn(b, world); } -// get the value (or null) in the current world -JL_DLLEXPORT jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var) +// get the value (or null) in the world +jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var, size_t world) { jl_binding_t *b = jl_get_module_binding(m, var, 1); - return jl_get_binding_value_depwarn(b, jl_current_task->world_age); + return jl_get_binding_value_depwarn(b, world); } // get the global (or null) in the latest world diff --git a/src/precompile.c b/src/precompile.c index 33b71d4605b30..a6ec4d550cfba 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -45,8 +45,8 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); JL_GC_PUSH4(&deptuple, &depots, &replace_depot_func, &normalize_depots_func); - replace_depot_func = jl_eval_global_var(jl_base_module, jl_symbol("replace_depot_path")); - normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation")); + replace_depot_func = jl_eval_global_var(jl_base_module, jl_symbol("replace_depot_path"), jl_current_task->world_age); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation"), jl_current_task->world_age); depots = jl_apply(&normalize_depots_func, 1); jl_datatype_t *deptuple_p[5] = {jl_module_type, jl_string_type, jl_uint64_type, jl_uint32_type, jl_float64_type}; jl_value_t *jl_deptuple_type = jl_apply_tuple_type_v((jl_value_t**)deptuple_p, 5); diff --git a/src/rtutils.c b/src/rtutils.c index d5ec8a78b39ce..e3672ab5d887e 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -1596,10 +1596,11 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, jl_value_t *msg) { jl_value_t *logmsg_func = NULL; + jl_task_t *ct = jl_current_task; if (jl_base_module) { - jl_value_t *corelogging = jl_get_global_value(jl_base_module, jl_symbol("CoreLogging")); + jl_value_t *corelogging = jl_get_global_value(jl_base_module, jl_symbol("CoreLogging"), ct->world_age); if (corelogging && jl_is_module(corelogging)) { - logmsg_func = jl_get_global_value((jl_module_t*)corelogging, jl_symbol("logmsg_shim")); + logmsg_func = jl_get_global_value((jl_module_t*)corelogging, jl_symbol("logmsg_shim"), ct->world_age); } } if (!logmsg_func) { diff --git a/src/scheduler.c b/src/scheduler.c index 8e0202cc7a980..b13e4072c7d73 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -331,7 +331,7 @@ void jl_task_wait_empty(void) jl_wait_empty_begin(); size_t lastage = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("wait")); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("wait"), ct->world_age); wait_empty = ct; if (f) { JL_GC_PUSH1(&f); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 3f05de189c3ef..0b8cfc1cf4ebd 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -544,21 +544,21 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t jl_value_t *get_compiletime_prefs_func = NULL; JL_GC_PUSH8(&depots, &prefs_list, &unique_func, &replace_depot_func, &normalize_depots_func, &toplevel, &prefs_hash_func, &get_compiletime_prefs_func); - jl_array_t *udeps = (jl_array_t*)jl_get_global_value(jl_base_module, jl_symbol("_require_dependencies")); + jl_array_t *udeps = (jl_array_t*)jl_get_global_value(jl_base_module, jl_symbol("_require_dependencies"), ct->world_age); *udepsp = udeps; // unique(udeps) to eliminate duplicates while preserving order: // we preserve order so that the topmost included .jl file comes first if (udeps) { - unique_func = jl_eval_global_var(jl_base_module, jl_symbol("unique")); + unique_func = jl_eval_global_var(jl_base_module, jl_symbol("unique"), ct->world_age); jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)udeps}; udeps = (jl_array_t*)jl_apply(uniqargs, 2); *udepsp = udeps; JL_TYPECHK(write_dependency_list, array_any, (jl_value_t*)udeps); } - replace_depot_func = jl_get_global_value(jl_base_module, jl_symbol("replace_depot_path")); - normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation")); + replace_depot_func = jl_get_global_value(jl_base_module, jl_symbol("replace_depot_path"), ct->world_age); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation"), ct->world_age); depots = jl_apply(&normalize_depots_func, 1); @@ -616,9 +616,9 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // Calculate Preferences hash for current package. if (jl_base_module) { // Toplevel module is the module we're currently compiling, use it to get our preferences hash - toplevel = jl_get_global_value(jl_base_module, jl_symbol("__toplevel__")); - prefs_hash_func = jl_eval_global_var(jl_base_module, jl_symbol("get_preferences_hash")); - get_compiletime_prefs_func = jl_eval_global_var(jl_base_module, jl_symbol("get_compiletime_preferences")); + toplevel = jl_get_global_value(jl_base_module, jl_symbol("__toplevel__"), ct->world_age); + prefs_hash_func = jl_eval_global_var(jl_base_module, jl_symbol("get_preferences_hash"), ct->world_age); + get_compiletime_prefs_func = jl_eval_global_var(jl_base_module, jl_symbol("get_compiletime_preferences"), ct->world_age); if (toplevel) { // call get_compiletime_prefs(__toplevel__) diff --git a/src/task.c b/src/task.c index 5804be8bff1a9..f8367a514572c 100644 --- a/src/task.c +++ b/src/task.c @@ -335,7 +335,7 @@ void JL_NORETURN jl_finish_task(jl_task_t *ct) // let the runtime know this task is dead and find a new task to run jl_function_t *done = jl_atomic_load_relaxed(&task_done_hook_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("task_done_hook")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("task_done_hook"), ct->world_age); if (done != NULL) jl_atomic_store_release(&task_done_hook_func, done); } diff --git a/src/threading.c b/src/threading.c index 37e5ac3b856a3..11726f5452cf0 100644 --- a/src/threading.c +++ b/src/threading.c @@ -413,7 +413,7 @@ static void jl_init_task_lock(jl_task_t *ct) ct->world_age = jl_get_world_counter(); jl_function_t *done = jl_atomic_load_relaxed(&init_task_lock_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("init_task_lock")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("init_task_lock"), ct->world_age); if (done != NULL) jl_atomic_store_release(&init_task_lock_func, done); } diff --git a/src/toplevel.c b/src/toplevel.c index 7b20c5df19b12..8540d8e5f3c56 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -62,7 +62,7 @@ void jl_module_run_initializer(jl_module_t *m) size_t last_age = ct->world_age; JL_TRY { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *f = jl_get_global_value(m, jl_symbol("__init__")); + jl_value_t *f = jl_get_global_value(m, jl_symbol("__init__"), ct->world_age); if (f != NULL) { JL_GC_PUSH1(&f); jl_apply(&f, 1); @@ -104,10 +104,10 @@ jl_array_t *jl_get_loaded_modules(void) return NULL; } -static int jl_is__toplevel__mod(jl_module_t *mod) +static int jl_is__toplevel__mod(jl_module_t *mod, jl_task_t *ct) { return jl_base_module && - (jl_value_t*)mod == jl_get_global_value(jl_base_module, jl_symbol("__toplevel__")); + (jl_value_t*)mod == jl_get_global_value(jl_base_module, jl_symbol("__toplevel__"), ct->world_age); } // TODO: add locks around global state mutation operations @@ -129,7 +129,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex jl_type_error("module", (jl_value_t*)jl_symbol_type, (jl_value_t*)name); } - int is_parent__toplevel__ = jl_is__toplevel__mod(parent_module); + int is_parent__toplevel__ = jl_is__toplevel__mod(parent_module, ct); // If we have `Base`, don't also try to import `Core` - the `Base` exports are a superset. // While we allow multiple imports of the same binding from different modules, various error printing // performs reflection on which module a binding came from and we'd prefer users see "Base" here. @@ -247,19 +247,18 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex return (jl_value_t*)newm; } -static jl_value_t *jl_eval_dot_expr(jl_module_t *m, jl_value_t *x, jl_value_t *f, int fast, const char **toplevel_filename, int *toplevel_lineno) +static jl_value_t *jl_eval_dot_expr(jl_task_t *ct, jl_module_t *m, jl_value_t *x, jl_value_t *f, int fast, const char **toplevel_filename, int *toplevel_lineno) { - jl_task_t *ct = jl_current_task; jl_value_t **args; JL_GC_PUSHARGS(args, 3); args[1] = jl_toplevel_eval_flex(m, x, fast, 0, toplevel_filename, toplevel_lineno); args[2] = jl_toplevel_eval_flex(m, f, fast, 0, toplevel_filename, toplevel_lineno); if (jl_is_module(args[1])) { JL_TYPECHK(getglobal, symbol, args[2]); - args[0] = jl_eval_global_var((jl_module_t*)args[1], (jl_sym_t*)args[2]); + args[0] = jl_eval_global_var((jl_module_t*)args[1], (jl_sym_t*)args[2], ct->world_age); } else { - args[0] = jl_eval_global_var(jl_base_relative_to(m), jl_symbol("getproperty")); + args[0] = jl_eval_global_var(jl_base_relative_to(m), jl_symbol("getproperty"), ct->world_age); size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); args[0] = jl_apply(args, 3); @@ -604,6 +603,16 @@ JL_DLLEXPORT void jl_eval_const_decl(jl_module_t *m, jl_value_t *arg, jl_value_t JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno) { jl_task_t *ct = jl_current_task; + if (jl_is_globalref(e)) { + return jl_eval_globalref((jl_globalref_t*)e, ct->world_age); + } + if (jl_is_symbol(e)) { + char *n = jl_symbol_name((jl_sym_t*)e), *n0 = n; + while (*n == '_') ++n; + if (*n == 0 && n > n0) + jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, "all-underscore identifiers are write-only and their values cannot be used in expressions"); + return jl_eval_global_var(m, (jl_sym_t*)e, ct->world_age); + } if (!jl_is_expr(e)) { if (jl_is_linenode(e)) { *toplevel_lineno = jl_linenode_line(e); @@ -617,12 +626,6 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val jl_lineno = *toplevel_lineno; return jl_nothing; } - if (jl_is_symbol(e)) { - char *n = jl_symbol_name((jl_sym_t*)e), *n0 = n; - while (*n == '_') ++n; - if (*n == 0 && n > n0) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, "all-underscore identifiers are write-only and their values cannot be used in expressions"); - } return jl_interpret_toplevel_expr_in(m, e, NULL, NULL); } @@ -635,7 +638,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val jl_value_t *rhs = jl_exprarg(ex, 1); // only handle `a.b` syntax here, so qualified names can be eval'd in pure contexts if (jl_is_quotenode(rhs) && jl_is_symbol(jl_fieldref(rhs, 0))) { - return jl_eval_dot_expr(m, lhs, rhs, fast, toplevel_filename, toplevel_lineno); + return jl_eval_dot_expr(ct, m, lhs, rhs, fast, toplevel_filename, toplevel_lineno); } } @@ -727,7 +730,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val } else if (jl_is_symbol(ex)) { JL_GC_POP(); - return jl_eval_global_var(m, (jl_sym_t*)ex); + return jl_eval_global_var(m, (jl_sym_t*)ex, ct->world_age); } else if (head == NULL) { JL_GC_POP(); @@ -804,7 +807,7 @@ JL_DLLEXPORT void jl_check_top_level_effect(jl_module_t *m, char *fname) } } JL_UNLOCK(&jl_modules_mutex); - if (!open && !jl_is__toplevel__mod(m)) { + if (!open && !jl_is__toplevel__mod(m, jl_current_task)) { const char* name = jl_symbol_name(m->name); jl_errorf("Evaluation into the closed module `%s` breaks incremental compilation " "because the side effects will not be permanent. " From 48c0e7f99c4e83d98761898e090ca3b5c6d3be90 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 2 Jun 2025 21:47:28 +0200 Subject: [PATCH 359/662] Update readlines(::Cmd) test to not rely on the filesystem (#58607) --- test/spawn.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/spawn.jl b/test/spawn.jl index 099f0670ce5f7..bfb7c9a83ffb6 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -620,7 +620,9 @@ end @test reduce(&, [`$echocmd abc`, `$echocmd def`, `$echocmd hij`]) == `$echocmd abc` & `$echocmd def` & `$echocmd hij` # readlines(::Cmd), accidentally broken in #20203 -@test sort(readlines(`$lscmd -A`)) == sort(readdir()) +let str = "foo\nbar" + @test readlines(`$echocmd $str`) == split(str) +end # issue #19864 (PR #20497) let c19864 = readchomp(pipeline(ignorestatus( From 9108dd08a572e394854d85aa0b2b680cc6a591c3 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 3 Jun 2025 00:15:17 +0200 Subject: [PATCH 360/662] test: Compiler: relax flaky codegen test (#57432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relaxing it should help prevent spurious CI failures until someone does a proper fix. Updates #44740 --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> Co-authored-by: Jameson Nash --- Compiler/test/codegen.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index c2e7abe34c34a..ea83f43382582 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -410,7 +410,7 @@ function g_dict_hash_alloc() end # Warm up f_dict_hash_alloc(); g_dict_hash_alloc(); -@test abs((@allocated f_dict_hash_alloc()) / (@allocated g_dict_hash_alloc()) - 1) < 0.1 # less that 10% difference +@test abs((@allocated f_dict_hash_alloc()) / (@allocated g_dict_hash_alloc()) - 1) < 0.3 # returning an argument shouldn't alloc a new box @noinline f33829(x) = (global called33829 = true; x) From 050284ceed86aa47b34d39ebccdee0beaf026146 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Tue, 3 Jun 2025 02:03:24 -0700 Subject: [PATCH 361/662] Fix `linearize` of global with complex type (#58611) --- src/julia-syntax.scm | 2 +- test/syntax.jl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index cf2c964cf2374..808943c05af5b 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -5019,7 +5019,7 @@ f(x) = yt(x) (if value (error "misplaced \"global\" declaration")) (if (or (length= e 2) (atom? (caddr e))) (emit e) (let ((rr (make-ssavalue))) - (emit `(= ,rr ,(caddr e))) + (emit `(= ,rr ,(compile (caddr e) break-labels #t #f))) (emit `(globaldecl ,(cadr e) ,rr)))) (if (null? (cadr lam)) (emit `(latestworld)))) diff --git a/test/syntax.jl b/test/syntax.jl index 01b48918b5fde..92d8391345312 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4299,6 +4299,16 @@ end @test letf_57470(3) == 5 @test letT_57470 === Int64 +end # M57470_sub + +# lowering globaldecl with complex type +module M58609 +using Test +global x::T where T +global y::Type{<:Number} + +@test Core.get_binding_type(M58609, :x) === Any +@test Core.get_binding_type(M58609, :y) == Type{<:Number} end # #57574 From 1ea959a8b12f06391c305d600fb6942cb25905f6 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Tue, 3 Jun 2025 04:05:17 -0500 Subject: [PATCH 362/662] ci: Don't add status label to prs automatically (#58606) This status label will be helpful once it has a well defined semantic and we have more statuses to transition between. As is, it is mostly noise and I think it should be (temporarily) removed. --- .github/workflows/PrAssignee.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 62d8e192a59bb..97cc963843619 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -172,15 +172,16 @@ jobs: assignees: selectedAssignee, }); - // Add the "pr review" label - const prReviewLabel = 'status: waiting for PR reviewer'; - console.log('Attempting to add prReviewLabel to this PR...'); - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - labels: [prReviewLabel], - }); + // The following is commented out because the label only makes sense in the presence of a larger state machine + // // Add the "pr review" label + // const prReviewLabel = 'status: waiting for PR reviewer'; + // console.log('Attempting to add prReviewLabel to this PR...'); + // await github.rest.issues.addLabels({ + // owner: context.repo.owner, + // repo: context.repo.repo, + // issue_number: context.payload.pull_request.number, + // labels: [prReviewLabel], + // }); // Now get the updated PR info, and see if we were successful: const updatedPrData = await github.rest.pulls.get({ From 6f1da031bd81d4ee2c7bebfd85dfc4f1596e0ce6 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 3 Jun 2025 16:18:15 -0400 Subject: [PATCH 363/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=2088629b552=20to=205577f68d6=20(#58619)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 | 1 + .../Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 | 1 + .../Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 | 1 - .../Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 create mode 100644 deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 diff --git a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 new file mode 100644 index 0000000000000..6724025f60360 --- /dev/null +++ b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 @@ -0,0 +1 @@ +de84d77a95619c7127bf333a1a3e1c81 diff --git a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 new file mode 100644 index 0000000000000..17f80096c03eb --- /dev/null +++ b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 @@ -0,0 +1 @@ +7e904d8d984e7bcb48fb740d8bd35d3e26525dcbfd112b17e38d24ee6b72d4d0e96d07979f12004675097007b4bce58312c3822913c61c7fcf0441863d75c8de diff --git a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 deleted file mode 100644 index 67dc729b08313..0000000000000 --- a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -2b20498f7b8114ec06e809af6051679c diff --git a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 b/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 deleted file mode 100644 index 009cd139a3f2c..0000000000000 --- a/deps/checksums/Pkg-88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -5fa605f7af3378a6b7037cb01c08cbeac050d0dfa0d3c1db70b69a464801034c896334cac4234c43b91e78a3da2b1bbf0e598c7100a12ef57d4d662e1fb2324b diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index a5b7ea80e9d56..def2c01b96494 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0 +PKG_SHA1 = 5577f68d612139693282c037d070f515bf160d1b PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 8d2708c861ac84de16afe0efc58fd05ac023ee42 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Tue, 3 Jun 2025 17:36:19 -0300 Subject: [PATCH 364/662] reland: hard heap limit flag (#58600) Relands: https://github.com/JuliaLang/julia/pull/58487. --- base/options.jl | 2 ++ src/gc-mmtk.c | 2 +- src/gc-stock.c | 28 +++++++++++++++++-- src/jloptions.c | 48 ++++++++++++++++++++++++++------- src/jloptions.h | 2 ++ src/julia.h | 2 +- test/cmdlineargs.jl | 66 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 136 insertions(+), 14 deletions(-) diff --git a/base/options.jl b/base/options.jl index 818fd33d6f7fc..e3865e076d5bb 100644 --- a/base/options.jl +++ b/base/options.jl @@ -60,6 +60,8 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 + hard_heap_limit::UInt64 + heap_target_increment::UInt64 trace_compile_timing::Int8 trim::Int8 task_metrics::Int8 diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index a6650dd7cb68c..edf74dcc65443 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -78,7 +78,7 @@ void jl_gc_init(void) { if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); } #ifdef _P64 if (hint == 0) { diff --git a/src/gc-stock.c b/src/gc-stock.c index 0132bbfeb1931..d518df6dfb52a 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3214,7 +3214,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS uint64_t target_heap; const char *reason = ""; (void)reason; // for GC_TIME output stats old_heap_size = heap_size; // TODO: Update these values dynamically instead of just during the GC - if (collection == JL_GC_AUTO) { + if (collection == JL_GC_AUTO && jl_options.hard_heap_limit == 0) { // update any heuristics only when the user does not force the GC // but still update the timings, since GC was run and reset, even if it was too early uint64_t target_allocs = 0.0; @@ -3295,6 +3295,27 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS target_heap = jl_atomic_load_relaxed(&gc_heap_stats.heap_target); } + // Kill the process if we are above the hard heap limit + if (jl_options.hard_heap_limit != 0) { + if (heap_size > jl_options.hard_heap_limit) { + // Can't use `jl_errorf` here, because it will try to allocate memory + // and we are already at the hard limit. + jl_safe_printf("Heap size exceeded hard limit of %" PRIu64 " bytes.\n", + jl_options.hard_heap_limit); + abort(); + } + } + // Ignore heap limit computation from MemBalancer-like heuristics + // if the heap target increment goes above the value specified through + // `--heap-target-increment`. + // Note that if we reach this code, we can guarantee that the heap size + // is less than the hard limit, so there will be some room to grow the heap + // until the next GC without hitting the hard limit. + if (jl_options.heap_target_increment != 0) { + target_heap = heap_size + jl_options.heap_target_increment; + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap); + } + double old_ratio = (double)promoted_bytes/(double)heap_size; if (heap_size > user_max) { next_sweep_full = 1; @@ -3692,6 +3713,9 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); jl_atomic_store_relaxed(&gc_heap_stats.heap_target, default_collect_interval); + if (jl_options.hard_heap_limit != 0) { + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, jl_options.hard_heap_limit); + } gc_num.interval = default_collect_interval; gc_num.allocd = 0; gc_num.max_pause = 0; @@ -3705,7 +3729,7 @@ void jl_gc_init(void) if (jl_options.heap_size_hint == 0) { char *cp = getenv(HEAP_SIZE_HINT); if (cp) - hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + hint = parse_heap_size_option(cp, "JULIA_HEAP_SIZE_HINT=\"[]\"", 1); } #ifdef _P64 total_mem = uv_get_total_memory(); diff --git a/src/jloptions.c b/src/jloptions.c index 21d034a666b1d..2e9f020ee3977 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -36,7 +36,7 @@ JL_DLLEXPORT const char *jl_get_default_sysimg_path(void) /* This function is also used by gc-stock.c to parse the * JULIA_HEAP_SIZE_HINT environment variable. */ -uint64_t parse_heap_size_hint(const char *optarg, const char *option_name) +uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct) { long double value = 0.0; char unit[4] = {0}; @@ -62,14 +62,16 @@ uint64_t parse_heap_size_hint(const char *optarg, const char *option_name) multiplier <<= 40; break; case '%': - if (value > 100) - jl_errorf("julia: invalid percentage specified in %s", option_name); - uint64_t mem = uv_get_total_memory(); - uint64_t cmem = uv_get_constrained_memory(); - if (cmem > 0 && cmem < mem) - mem = cmem; - multiplier = mem/100; - break; + if (allow_pct) { + if (value > 100) + jl_errorf("julia: invalid percentage specified in %s", option_name); + uint64_t mem = uv_get_total_memory(); + uint64_t cmem = uv_get_constrained_memory(); + if (cmem > 0 && cmem < mem) + mem = cmem; + multiplier = mem/100; + break; + } default: jl_errorf("julia: invalid argument to %s (%s)", option_name, optarg); break; @@ -151,6 +153,8 @@ JL_DLLEXPORT void jl_init_options(void) 0, // strip-ir 0, // permalloc_pkgimg 0, // heap-size-hint + 0, // hard-heap-limit + 0, // heap-target-increment 0, // trace_compile_timing JL_TRIM_NO, // trim 0, // task_metrics @@ -289,6 +293,14 @@ static const char opts[] = " number of bytes, optionally in units of: B, K (kibibytes),\n" " M (mebibytes), G (gibibytes), T (tebibytes), or % (percentage\n" " of physical memory).\n\n" + " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever go above this\n" + " limit, we will abort. The value may be specified as a\n" + " number of bytes, optionally in units of: B, K (kibibytes),\n" + " M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" + " --heap-target-increment=[] Set an upper bound on how much the heap target\n" + " can increase between consecutive collections. The value may be\n" + " specified as a number of bytes, optionally in units of: B,\n" + " K (kibibytes), M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" ; static const char opts_hidden[] = @@ -380,6 +392,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_metadata, opt_strip_ir, opt_heap_size_hint, + opt_hard_heap_limit, + opt_heap_target_increment, opt_gc_threads, opt_permalloc_pkgimg, opt_trim, @@ -451,6 +465,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "hard-heap-limit", required_argument, 0, opt_hard_heap_limit }, + { "heap-target-increment", required_argument, 0, opt_heap_target_increment }, { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -960,11 +976,23 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) break; case opt_heap_size_hint: if (optarg != NULL) - jl_options.heap_size_hint = parse_heap_size_hint(optarg, "--heap-size-hint=[]"); + jl_options.heap_size_hint = parse_heap_size_option(optarg, "--heap-size-hint=[]", 1); if (jl_options.heap_size_hint == 0) jl_errorf("julia: invalid memory size specified in --heap-size-hint=[]"); break; + case opt_hard_heap_limit: + if (optarg != NULL) + jl_options.hard_heap_limit = parse_heap_size_option(optarg, "--hard-heap-limit=[]", 0); + if (jl_options.hard_heap_limit == 0) + jl_errorf("julia: invalid memory size specified in --hard-heap-limit=[]"); + break; + case opt_heap_target_increment: + if (optarg != NULL) + jl_options.heap_target_increment = parse_heap_size_option(optarg, "--heap-target-increment=[]", 0); + if (jl_options.heap_target_increment == 0) + jl_errorf("julia: invalid memory size specified in --heap-target-increment=[]"); + break; case opt_gc_threads: errno = 0; long nmarkthreads = strtol(optarg, &endptr, 10); diff --git a/src/jloptions.h b/src/jloptions.h index 06e00e9309dba..35cc8c3e13375 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -64,6 +64,8 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; + uint64_t hard_heap_limit; + uint64_t heap_target_increment; int8_t trace_compile_timing; int8_t trim; int8_t task_metrics; diff --git a/src/julia.h b/src/julia.h index 03a0185800d12..6c1c8af0a788b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2578,7 +2578,7 @@ JL_DLLEXPORT ssize_t jl_sizeof_jl_options(void); JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp); JL_DLLEXPORT char *jl_format_filename(const char *output_pattern); -uint64_t parse_heap_size_hint(const char *optarg, const char *option_name); +uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int allow_pct); // Set julia-level ARGS array according to the arguments provided in // argc/argv diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 12b13c1984a5e..8f286741186fd 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1214,6 +1214,12 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end + +@testset "hard heap limit" begin + cmd = `$(Base.julia_cmd()) --hard-heap-limit=30M -E "mutable struct ListNode; v::Int64; next::Union{ListNode, Nothing}; end\n + l = ListNode(0, nothing); while true; l = ListNode(0, l); end"` + @test !success(cmd) +end end ## `Main.main` entrypoint @@ -1253,6 +1259,66 @@ end end end +@testset "--hard-heap-limit" begin + exename = `$(Base.julia_cmd())` + @test errors_not_signals(`$exename --hard-heap-limit -e "exit(0)"`) + @testset "--hard-heap-limit=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] + @test errors_not_signals(`$exename --hard-heap-limit=$str -e "exit(0)"`) + end + k = UInt64(1) << 10 + m = UInt64(1) << 20 + g = UInt64(1) << 30 + t = UInt64(1) << 40 + one_hundred_mb_strs_and_vals = [ + ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), + ] + @testset "--hard-heap-limit=$str" for (str, val) in one_hundred_mb_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end + two_and_a_half_gigabytes_strs_and_vals = [ + ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), + ] + @testset "--hard-heap-limit=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end + one_terabyte_strs_and_vals = [ + ("1TB", 1t), ("1024GB", 1t), + ] + @testset "--hard-heap-limit=$str" for (str, val) in one_terabyte_strs_and_vals + @test parse(UInt64,read(`$exename --hard-heap-limit=$str -E "Base.JLOptions().hard_heap_limit"`, String)) == val + end +end + +@testset "--heap-target-increment" begin + exename = `$(Base.julia_cmd())` + @test errors_not_signals(`$exename --heap-target-increment -e "exit(0)"`) + @testset "--heap-target-increment=$str" for str in ["asdf","","0","1.2vb","b","GB","2.5GB̂","1.2gb2","42gigabytes","5gig","2GiB","NaNt"] + @test errors_not_signals(`$exename --heap-target-increment=$str -e "exit(0)"`) + end + k = UInt64(1) << 10 + m = UInt64(1) << 20 + g = UInt64(1) << 30 + t = UInt64(1) << 40 + one_hundred_mb_strs_and_vals = [ + ("100000000", 100000000), ("1e8", 1e8), ("100MB", 100m), ("100m", 100m), ("1e5kB", 1e5k), + ] + @testset "--heap-target-increment=$str" for (str, val) in one_hundred_mb_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end + two_and_a_half_gigabytes_strs_and_vals = [ + ("2500000000", 2500000000), ("2.5e9", 2.5e9), ("2.5g", 2.5g), ("2.5GB", 2.5g), ("2.5e6mB", 2.5e6m), + ] + @testset "--heap-target-increment=$str" for (str, val) in two_and_a_half_gigabytes_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end + one_terabyte_strs_and_vals = [ + ("1TB", 1t), ("1024GB", 1t), + ] + @testset "--heap-target-increment=$str" for (str, val) in one_terabyte_strs_and_vals + @test parse(UInt64,read(`$exename --heap-target-increment=$str -E "Base.JLOptions().heap_target_increment"`, String)) == val + end +end + @testset "--timeout-for-safepoint-straggler" begin exename = `$(Base.julia_cmd())` timeout = 120 From 921f17a349dd1c20ae266925fc3c8d459a731b77 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 3 Jun 2025 17:34:46 -0400 Subject: [PATCH 365/662] CI: PrAssignee.yml: Comment out some more label-related code (#58621) Follow-up to https://github.com/JuliaLang/julia/pull/58606 --- .github/workflows/PrAssignee.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 97cc963843619..5a7a74dd8c503 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -200,17 +200,17 @@ jobs: weDidEncounterError = true; console.log(`ERROR: Failed to assign @${selectedAssignee}`); } - const newPrLabels = updatedPrData - .data - .labels - .map(element => element.name) - console.log('newPrLabels: ', newPrLabels); - if (newPrLabels.includes(prReviewLabel)) { - console.log('Successfully added prReviewLabel'); - } else { - weDidEncounterError = true; - console.log('ERROR: Failed to add add prReviewLabel'); - } + // const newPrLabels = updatedPrData + // .data + // .labels + // .map(element => element.name) + // console.log('newPrLabels: ', newPrLabels); + // if (newPrLabels.includes(prReviewLabel)) { + // console.log('Successfully added prReviewLabel'); + // } else { + // weDidEncounterError = true; + // console.log('ERROR: Failed to add add prReviewLabel'); + // } // Post a comment const commentBody = `Hello! I am a bot.\n\nThank you for your pull request!\n\nI have assigned \`@${selectedAssignee}\` to this pull request.\n\n\`@${selectedAssignee}\` can either choose to review this pull request themselves, or they can choose to find someone else to review this pull request.` From e9e7931675bea6bc77eaa5f48fe7687464a195cb Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 3 Jun 2025 18:50:23 -0400 Subject: [PATCH 366/662] CI: `PrAssignee.yml`: don't skip PRs from triagers (#58612) Co-authored-by: Lilith Orion Hafner --- .github/workflows/PrAssignee.yml | 62 ++++++++++++-------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 5a7a74dd8c503..ba65fd9cee960 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -49,45 +49,29 @@ jobs: console.log('oldPrAssignees: ', oldPrAssignees); const prAuthor = context.payload.pull_request.user.login; - // Check if the PR is opened by a collaborator on the repo (aka a committer). - // We use the /repos/{owner}/{repo}/collaborators/{username} endpoint to avoid - // neeing org scope permissions. - const isCollaboratorResponseObj = await github.request('GET /repos/{owner}/{repo}/collaborators/{username}', { - owner: context.repo.owner, - repo: context.repo.repo, - username: prAuthor, - headers: { - 'X-GitHub-Api-Version': '2022-11-28' - } - }).catch((error) => { - // So, I'm not sure if the error is being thrown by `@octokit/request.js` or - // `@octokit/plugin-retry.js`. I suspect that the initial error is thrown by - // `request.js`, and is subsequently propagated by `plugin-retry.js`. - // - // Anyway, the docs for `request.js` say the following: - // - // > If an error occurs, the promise is rejected with an `error` object - //> containing 3 keys to help with debugging: - // > - // > - `error.status` The http response status code - // > - `error.request` The request options such as `method`, `url` and `data` - // > - `error.response` The http response object with `url`, `headers`, and `data` - // - // Source: https://github.com/octokit/request.js/blob/main/README.md#request - // - // So, inside this `.catch()`, we simply return that `error` object. - - return error; - }); - if (isCollaboratorResponseObj.status == 204) { - var isCollaborator = true; - } else if (isCollaboratorResponseObj.status == 404) { - var isCollaborator = false; - } else { - console.error('Unable to process the response from checkCollaborator'); - console.error('isCollaboratorResponseObj: ', isCollaboratorResponseObj); - var isCollaborator = false; - } + // Check if the PR is opened by a collaborator on the repo, aka someone with write (commit) permissions or higher. + const relevantPerms = [ + // 'triage', // Uncomment this line if you don't want PRs from triagers to get auto-assignees. + 'push', + 'maintain', + 'admin', + ] + const allCollaboratorsNestedPromises = relevantPerms.map( + (perm) => github.paginate( + // We use the `/repos/{owner}/{repo}/collaborators` endpoint to avoid needing org scope permissions: + '/repos/{owner}/{repo}/collaborators', + { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + permission: perm, + }, + (response) => response.data.map((collaboratorInfo) => collaboratorInfo.login), + ) + ) + const allCollaboratorsNested = await Promise.all(allCollaboratorsNestedPromises); + const allCollaboratorsFlattened = allCollaboratorsNested.flat(); + const isCollaborator = allCollaboratorsFlattened.includes(prAuthor); console.log('prAuthor: ', prAuthor); console.log('isCollaborator: ', isCollaborator); From b04b1040203d4093163d285e25a71bb9bf5b9f0c Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 3 Jun 2025 23:03:21 -0300 Subject: [PATCH 367/662] If the user explicitly asked for 1 thread don't add an interactive one. (#57454) Co-authored-by: Ian Butterworth --- HISTORY.md | 7 ++++--- doc/src/manual/multi-threading.md | 5 ++++- src/jloptions.c | 3 +++ src/threading.c | 2 ++ test/cmdlineargs.jl | 6 ++++++ 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 6c2ca03662233..ce4e9982a2c6b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -36,9 +36,10 @@ Language changes * Julia now defaults to 1 "interactive" thread, in addition to the 1 default "worker" thread. i.e. `-t1,1`. This means in default configuration the main task and repl (when in interactive mode), which both run on thread 1, now run within the `interactive` threadpool. The libuv IO loop also runs on thread 1, - helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Pass `0` to disable the - interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0`, or `-tauto,0` etc. The zero is explicitly - required to disable it, `-t2` will set the equivalent of `-t2,1` ([#57087]). + helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Asking for specifically 1 thread + (`-t1`/`JULIA_NUM_THREADS=1`) or passing `0` will disable the interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0` + , or `-tauto,0` etc. Asking for more than 1 thread will enable the interactive thread so + `-t2` will set the equivalent of `-t2,1` ([#57087]). * When a method is replaced with an exactly equivalent one, the old method is not deleted. Instead, the new method takes priority and becomes more specific than the old method. Thus if the new method is deleted later, the old method will resume operating. This can be useful in mocking frameworks (as in SparseArrays, diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index ec470f867cc47..401169e9c2132 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -8,7 +8,7 @@ of Julia multi-threading features. By default, Julia starts up with 2 threads of execution; 1 worker thread and 1 interactive thread. This can be verified by using the command [`Threads.nthreads()`](@ref): -```jldoctest +```julia julia> Threads.nthreads(:default) 1 julia> Threads.nthreads(:interactive) @@ -37,6 +37,7 @@ each threadpool. !!! compat "Julia 1.12" Starting by default with 1 interactive thread, as well as the 1 worker thread, was made as such in Julia 1.12 + If the number of threads is set to 1 by either doing `-t1` or `JULIA_NUM_THREADS=1` an interactive thread will not be spawned. Lets start Julia with 4 threads: @@ -144,6 +145,8 @@ julia> nthreads(:interactive) julia> nthreads() 3 ``` +!!! note + Explicitly asking for 1 thread by doing `-t1` or `JULIA_NUM_THREADS=1` does not add an interactive thread. !!! note The zero-argument version of `nthreads` returns the number of threads diff --git a/src/jloptions.c b/src/jloptions.c index 2e9f020ee3977..ca52d3c7f311c 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -685,6 +685,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) if (nthreadsi == 0) jl_options.nthreadpools = 1; } + } else if (nthreads == 1) { // User asked for 1 thread so don't add an interactive one + jl_options.nthreadpools = 1; + nthreadsi = 0; } jl_options.nthreads = nthreads + nthreadsi; } diff --git a/src/threading.c b/src/threading.c index 11726f5452cf0..6cd8eef558eb6 100644 --- a/src/threading.c +++ b/src/threading.c @@ -740,6 +740,8 @@ void jl_init_threading(void) if (errno != 0 || endptr == cp || nthreads <= 0) nthreads = 1; cp = endptr; + if (nthreads == 1) // User asked for 1 thread so lets assume they dont want an interactive thread + nthreadsi = 0; } if (*cp == ',') { cp++; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 8f286741186fd..5aee455ae75b4 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -348,6 +348,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` # -t, --threads code = "print(Threads.threadpoolsize())" + code2 = "print(Threads.maxthreadid())" cpu_threads = ccall(:jl_effective_threads, Int32, ()) @test string(cpu_threads) == read(`$exename --threads auto -e $code`, String) == @@ -358,6 +359,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` withenv("JULIA_NUM_THREADS" => nt) do @test read(`$exename --threads=2 -e $code`, String) == read(`$exename -t 2 -e $code`, String) == "2" + if nt === nothing + @test read(`$exename -e $code2`, String) == "2" #default + interactive + elseif nt == "1" + @test read(`$exename -e $code2`, String) == "1" #if user asks for 1 give 1 + end end end # We want to test oversubscription, but on manycore machines, this can From 602d6ee987486ec2b0953bb2bf895ca7986b7a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Wed, 4 Jun 2025 15:20:36 +0200 Subject: [PATCH 368/662] InteractiveUtils: Fully support broadcasting expressions for code introspection macros (#58349) This PR includes a fix and a feature for code introspection macros (`@code_typed`, `@code_llvm` and friends *but not* `@which`, `@edit`, etc): - Fixes a bug for expressions of the form `f.(x; y = 3)`, for which keyword arguments were not properly handled and led to an internal error. - Adds support for broadcasting assignments of the form `x .= f(y)`, `x .<<= f.(y, z)`, etc. The way this was (and still is) implemented is by constructing a temporary function, `f(x1, x2, x3, ...) = ` and feeding that to code introspection functions. This trick doesn't apply to `@which` and `@edit`, which need to map to a single function call (we could arguably choose to target `materialize`/`materialize!`, but this behavior could be a bit surprising and difficult to support). The switch differentiating the families of macro `@code_typed`/`@code_llvm` and `@which`/`@edit` etc was further exposed as an additional argument to `gen_call_with_extracted_types` and `gen_call_with_extracted_types_and_kwargs`, which default to the previous behavior (differentiating them based on whether their name starts with `code_`). The intent is to allow other macros such as `Cthulhu.@descend` to register themselves as code introspection macros. Quick tests indicate that it works as intended, e.g. with this PR Cthulhu supports `@descend [1, 2] .+= [2, 3]` (or equivalently, as added in #57909, `@descend ::Vector{Int} .+= ::Vector{Int}`). I originally just went for the fix, and after some refactoring I realized the feature was very straightforward to implement. --- Compiler/test/irutils.jl | 4 +- NEWS.md | 1 + stdlib/InteractiveUtils/src/macros.jl | 173 ++++++++++++++--------- stdlib/InteractiveUtils/test/runtests.jl | 24 +++- stdlib/Test/src/Test.jl | 2 +- 5 files changed, 137 insertions(+), 67 deletions(-) diff --git a/Compiler/test/irutils.jl b/Compiler/test/irutils.jl index c1616ad4a8fd0..e50491420c338 100644 --- a/Compiler/test/irutils.jl +++ b/Compiler/test/irutils.jl @@ -14,7 +14,7 @@ macro code_typed1(ex0...) end get_code(args...; kwargs...) = code_typed1(args...; kwargs...).code macro get_code(ex0...) - return gen_call_with_extracted_types_and_kwargs(__module__, :get_code, ex0) + return gen_call_with_extracted_types_and_kwargs(__module__, :get_code, ex0; is_source_reflection = false) end # check if `x` is a statement with a given `head` @@ -58,7 +58,7 @@ function fully_eliminated(code::Vector{Any}; retval=(@__FILE__), kwargs...) return retval′ == retval end macro fully_eliminated(ex0...) - return gen_call_with_extracted_types_and_kwargs(__module__, :fully_eliminated, ex0) + return gen_call_with_extracted_types_and_kwargs(__module__, :fully_eliminated, ex0; is_source_reflection = false) end let m = Meta.@lower 1 + 1 diff --git a/NEWS.md b/NEWS.md index 92514cff37390..d38da1cfe1300 100644 --- a/NEWS.md +++ b/NEWS.md @@ -72,6 +72,7 @@ Standard library changes #### InteractiveUtils * Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}`. Type-annotated variables as in `f(val::Int; kw::Float64)` are not evaluated if the type annotation provides the necessary information, making this syntax compatible with signatures found in stacktraces ([#57909], [#58222]). +* Code introspection macros such as `@code_lowered` and `@code_typed` now have a much better support for broadcasting expressions, including broadcasting assignments of the form `x .+= f(y)` ([#58349]). External dependencies --------------------- diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 49d31e48a8198..753e84beda06e 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -32,30 +32,65 @@ function get_typeof(@nospecialize ex) return :(Core.Typeof($(esc(ex)))) end +function is_broadcasting_call(ex) + isa(ex, Expr) || return false + # Standard broadcasting: f.(x) + isexpr(ex, :.) && length(ex.args) ≥ 2 && isexpr(ex.args[2], :tuple) && return true + # Infix broadcasting: x .+ y, x .<< y, etc. + if isexpr(ex, :call) + f = ex.args[1] + f == :.. && return false + string(f)[1] == '.' && return true + end + return false +end +is_broadcasting_expr(ex) = is_broadcasting_call(ex) || is_broadcasting_assignment(ex) +function is_broadcasting_assignment(ex) + isa(ex, Expr) || return false + isexpr(ex, :.) && return false + head = string(ex.head) + # x .= y, x .+= y, x .<<= y, etc. + head[begin] == '.' && head[end] == '=' && return true + return false +end + """ Transform a dot expression into one where each argument has been replaced by a variable "xj" (with j an integer from 1 to the returned i). The list `args` contains the original arguments that have been replaced. """ function recursive_dotcalls!(ex, args, i=1) - if !(ex isa Expr) || ((ex.head !== :. || !(ex.args[2] isa Expr)) && - (ex.head !== :call || string(ex.args[1])[1] != '.')) - newarg = Symbol('x', i) - if isexpr(ex, :...) - push!(args, only(ex.args)) - return Expr(:..., newarg), i+1 + if is_broadcasting_expr(ex) + if is_broadcasting_assignment(ex) + (start, branches) = (1, ex.args) + elseif isexpr(ex, :.) + (start, branches) = (1, ex.args[2].args) else - push!(args, ex) - return newarg, i+1 + (start, branches) = (2, ex.args) end + for j in start:length(branches)::Int + branch, i = recursive_dotcalls!(branches[j], args, i) + branches[j] = branch + end + return ex, i + elseif isexpr(ex, :parameters) + for j in eachindex(ex.args) + param, i = recursive_dotcalls!(ex.args[j], args, i) + ex.args[j] = param + end + return ex, i end - (start, branches) = ex.head === :. ? (1, ex.args[2].args) : (2, ex.args) - length_branches = length(branches)::Int - for j in start:length_branches - branch, i = recursive_dotcalls!(branches[j], args, i) - branches[j] = branch + newarg = Symbol('x', i) + if isexpr(ex, :...) + newarg = Expr(:..., newarg) + push!(args, only(ex.args)) + elseif isexpr(ex, :kw) + newarg = Expr(:kw, ex.args[1], newarg) + push!(args, ex.args[end]) + else + push!(args, ex) end - return ex, i + return newarg, i+1 end function extract_farg(@nospecialize arg) @@ -158,13 +193,15 @@ function merge_namedtuple_types(nt::Type{<:NamedTuple}, nts::Type{<:NamedTuple}. NamedTuple{Tuple(names), Tuple{types...}} end -function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) +is_code_macro(fcn) = startswith(string(fcn), "code_") + +function gen_call_with_extracted_types(__module__, fcn, ex0, kws = Expr[]; is_source_reflection = !is_code_macro(fcn), supports_binding_reflection = false) if isexpr(ex0, :ref) ex0 = replace_ref_begin_end!(ex0) end # assignments get bypassed: @edit a = f(x) <=> @edit f(x) if isa(ex0, Expr) && ex0.head == :(=) && isa(ex0.args[1], Symbol) && isempty(kws) - return gen_call_with_extracted_types(__module__, fcn, ex0.args[2]) + return gen_call_with_extracted_types(__module__, fcn, ex0.args[2], kws; is_source_reflection, supports_binding_reflection) end where_params = nothing if isa(ex0, Expr) @@ -172,6 +209,7 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) end if isa(ex0, Expr) if ex0.head === :do && isexpr(get(ex0.args, 1, nothing), :call) + # Normalize `f(args...) do ... end` calls to `f(do_anonymous_function, args...)` if length(ex0.args) != 2 return Expr(:call, :error, "ill-formed do call") end @@ -180,53 +218,60 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) insert!(args, (isnothing(i) ? 2 : 1+i::Int), ex0.args[2]) ex0 = Expr(:call, args...) end - if ex0.head === :. || (ex0.head === :call && ex0.args[1] !== :.. && string(ex0.args[1])[1] == '.') - codemacro = startswith(string(fcn), "code_") - if codemacro && (ex0.head === :call || ex0.args[2] isa Expr) - # Manually wrap a dot call in a function - args = Any[] - ex, i = recursive_dotcalls!(copy(ex0), args) - xargs = [Symbol('x', j) for j in 1:i-1] - dotfuncname = gensym("dotfunction") - dotfuncdef = Expr(:local, Expr(:(=), Expr(:call, dotfuncname, xargs...), ex)) - return quote - $(esc(dotfuncdef)) - $(fcn)($(esc(dotfuncname)), $(typesof_expr(args, where_params)); $(kws...)) - end - elseif !codemacro - fully_qualified_symbol = true # of the form A.B.C.D - ex1 = ex0 - while ex1 isa Expr && ex1.head === :. - fully_qualified_symbol = (length(ex1.args) == 2 && - ex1.args[2] isa QuoteNode && - ex1.args[2].value isa Symbol) - fully_qualified_symbol || break - ex1 = ex1.args[1] + if is_broadcasting_expr(ex0) && !is_source_reflection + # Manually wrap top-level broadcasts in a function. + # We don't do that if `fcn` reflects into the source, + # because that destroys provenance information. + args = Any[] + ex, i = recursive_dotcalls!(copy(ex0), args) + xargs = [Symbol('x', j) for j in 1:i-1] + dotfuncname = gensym("dotfunction") + dotfuncdef = :(local $dotfuncname($(xargs...)) = $ex) + return quote + $(esc(dotfuncdef)) + $(gen_call_with_extracted_types(__module__, fcn, :($dotfuncname($(args...))), kws; is_source_reflection, supports_binding_reflection)) + end + elseif isexpr(ex0, :.) && is_source_reflection + # If `ex0` has the form A.B (or some chain A.B.C.D) and `fcn` reflects into the source, + # `A` (or `A.B.C`) may be a module, in which case `fcn` is probably more interested in + # the binding rather than the `getproperty` call. + # If binding reflection is not supported, we generate an error; `getproperty(::Module, field)` + # is not going to be interesting to reflect into, so best to allow future non-breaking support + # for binding reflection in case the macro may eventually support that. + fully_qualified_symbol = true + ex1 = ex0 + while ex1 isa Expr && ex1.head === :. + fully_qualified_symbol = (length(ex1.args) == 2 && + ex1.args[2] isa QuoteNode && + ex1.args[2].value isa Symbol) + fully_qualified_symbol || break + ex1 = ex1.args[1] + end + fully_qualified_symbol &= ex1 isa Symbol + if fully_qualified_symbol || isexpr(ex1, :(::), 1) + call_reflection = :($(fcn)(Base.getproperty, $(typesof_expr(ex0.args, where_params)))) + isexpr(ex0.args[1], :(::), 1) && return call_reflection + if supports_binding_reflection + binding_reflection = :($fcn(arg1, $(ex0.args[2]))) + else + binding_reflection = :(error("expression is not a function call")) end - fully_qualified_symbol &= ex1 isa Symbol - if fully_qualified_symbol || isexpr(ex1, :(::), 1) - getproperty_ex = :($(fcn)(Base.getproperty, $(typesof_expr(ex0.args, where_params)))) - isexpr(ex0.args[1], :(::), 1) && return getproperty_ex - return quote - local arg1 = $(esc(ex0.args[1])) - if isa(arg1, Module) - $(if string(fcn) == "which" - :(which(arg1, $(ex0.args[2]))) - else - :(error("expression is not a function call")) - end) - else - $getproperty_ex - end + return quote + local arg1 = $(esc(ex0.args[1])) + if isa(arg1, Module) + $binding_reflection + else + $call_reflection end - else - return Expr(:call, :error, "dot expressions are not lowered to " - * "a single function call, so @$fcn cannot analyze " - * "them. You may want to use Meta.@lower to identify " - * "which function call to target.") end end end + if is_broadcasting_expr(ex0) + return Expr(:call, :error, "dot expressions are not lowered to " + * "a single function call, so @$fcn cannot analyze " + * "them. You may want to use Meta.@lower to identify " + * "which function call to target.") + end if any(@nospecialize(a)->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args) args, kwargs = separate_kwargs(ex0.args) are_kwargs_valid(kwargs) || return quote @@ -318,7 +363,7 @@ Same behaviour as `gen_call_with_extracted_types` except that keyword arguments of the form "foo=bar" are passed on to the called function as well. The keyword arguments must be given before the mandatory argument. """ -function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0) +function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0; is_source_reflection = !is_code_macro(fcn), supports_binding_reflection = false) kws = Expr[] arg = ex0[end] # Mandatory argument for i in 1:length(ex0)-1 @@ -332,13 +377,15 @@ function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0) return Expr(:call, :error, "@$fcn expects only one non-keyword argument") end end - return gen_call_with_extracted_types(__module__, fcn, arg, kws) + return gen_call_with_extracted_types(__module__, fcn, arg, kws; is_source_reflection, supports_binding_reflection) end for fname in [:which, :less, :edit, :functionloc] @eval begin macro ($fname)(ex0) - gen_call_with_extracted_types(__module__, $(Expr(:quote, fname)), ex0) + gen_call_with_extracted_types(__module__, $(Expr(:quote, fname)), ex0, Expr[]; + is_source_reflection = true, + supports_binding_reflection = $(fname === :which)) end end end @@ -351,13 +398,13 @@ end for fname in [:code_warntype, :code_llvm, :code_native, :infer_return_type, :infer_effects, :infer_exception_type] @eval macro ($fname)(ex0...) - gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) + gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0; is_source_reflection = false) end end for fname in [:code_typed, :code_lowered, :code_ircode] @eval macro ($fname)(ex0...) - thecall = gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) + thecall = gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0; is_source_reflection = false) quote local results = $thecall length(results) == 1 ? results[1] : results diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 4002e792fcf81..ea845c012f529 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -373,6 +373,13 @@ end @test (@which round(1.2; digits = ::Float64, kwargs_1...)).name === :round @test (@which round(1.2; sigdigits = ::Int, kwargs_1...)).name === :round @test (@which round(1.2; kwargs_1..., kwargs_2..., base)).name === :round + @test (@code_typed optimize=false round.([1.0, 2.0]; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false round.(::Vector{Float64}, base = 2; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false round.(base = ::Int64, ::Vector{Float64}; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false [1, 2] .= ::Int)[2] == Vector{Int} + @test (@code_typed optimize=false ::Vector{Int} .= ::Int)[2] == Vector{Int} + @test (@code_typed optimize=false ::Vector{Float64} .= 1 .+ ::Vector{Int})[2] == Vector{Float64} + @test (@code_typed optimize=false ::Vector{Float64} .= 1 .+ round.(base = ::Int, ::Vector{Int}; digits = 3))[2] == Vector{Float64} end module MacroTest @@ -505,7 +512,22 @@ a14637 = A14637(0) @test (@code_typed optimize=true max.([1,7], UInt.([4])))[2] == Vector{UInt} @test (@code_typed Ref.([1,2])[1].x)[2] == Int @test (@code_typed max.(Ref(true).x))[2] == Bool +@test (@code_typed optimize=false round.([1.0, 2.0]; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false round.([1.0, 2.0], base = 2; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false round.(base = 2, [1.0, 2.0], digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false [1, 2] .= 2)[2] == Vector{Int} +@test (@code_typed optimize=false [1, 2] .<<= 2)[2] == Vector{Int} +@test (@code_typed optimize=false [1, 2.0] .= 1 .+ [2, 3])[2] == Vector{Float64} +@test (@code_typed optimize=false [1, 2.0] .= 1 .+ round.(base = 1, [1, 3]; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false [1] .+ [2])[2] == Vector{Int} @test !isempty(@code_typed optimize=false max.(Ref.([5, 6])...)) +expansion = string(@macroexpand @code_typed optimize=false max.(Ref.([5, 6])...)) +@test contains(expansion, "(x1) =") # presence of wrapper function +# Make sure broadcasts in nested arguments are not processed. +v = Any[1] +expansion = string(@macroexpand @code_typed v[1] = rand.(Ref(1))) +@test contains(expansion, "Typeof(rand.(Ref(1)))") +@test !contains(expansion, "(x1) =") # Issue # 45889 @test !isempty(@code_typed 3 .+ 6) @@ -619,7 +641,7 @@ end @test_throws err @code_lowered 1 @test_throws err @code_lowered 1.0 - @test_throws "is too complex" @code_lowered a .= 1 + 2 + @test_throws "dot expressions are not lowered to a single function call" @which a .= 1 + 2 @test_throws "invalid keyword argument syntax" @eval @which round(1; digits(3)) end diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 8e15d10512029..f379c4dd15ac3 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -2134,7 +2134,7 @@ function _inferred(ex, mod, allow = :(Union{})) kwargs = gensym() quote $(esc(args)), $(esc(kwargs)), result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) - inftype = $(gen_call_with_extracted_types(mod, Base.infer_return_type, :($(ex.args[1])($(args)...; $(kwargs)...)))) + inftype = $(gen_call_with_extracted_types(mod, Base.infer_return_type, :($(ex.args[1])($(args)...; $(kwargs)...)); is_source_reflection = false)) end else # No keywords From 7498f3454379c1c2fa77dc27dc90b9faf5751853 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 4 Jun 2025 16:38:54 -0400 Subject: [PATCH 369/662] CI: `PrAssignee.yml`: Add note about public membership in the bot comment (#58626) --- .github/workflows/PrAssignee.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index ba65fd9cee960..a5d8e67fe3706 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -197,7 +197,7 @@ jobs: // } // Post a comment - const commentBody = `Hello! I am a bot.\n\nThank you for your pull request!\n\nI have assigned \`@${selectedAssignee}\` to this pull request.\n\n\`@${selectedAssignee}\` can either choose to review this pull request themselves, or they can choose to find someone else to review this pull request.` + const commentBody = `Hello! I am a bot.\n\nThank you for your pull request!\n\nI have assigned \`@${selectedAssignee}\` to this pull request.\n\n\`@${selectedAssignee}\` can either choose to review this pull request themselves, or they can choose to find someone else to review this pull request.\n\nNote: If you are a Julia committer, please make sure that your organization membership is public.` console.log('Attempting to post bot comment on the PR...'); await github.rest.issues.createComment({ owner: context.repo.owner, From edb2aa8676b2ddf30fe3aba7bca399ed703576eb Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 4 Jun 2025 19:25:51 -0400 Subject: [PATCH 370/662] codegen: More robustness for invalid :invoke IR (#58631) See #58429 for general discussion of these kinds of errors. We don't verify this one in the IR verifier, because it's currently not really part of the structural invariants of the IR. One could imagine using IRCode with a non-codegen backend that uses something else here. Nevertheless, codegen needs to complain of course if someone put something here it doesn't understand. --- src/codegen.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 5aa175df270f3..a736449813608 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5184,10 +5184,12 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR if (jl_is_method_instance(lival.constant)) { mi = (jl_method_instance_t*)lival.constant; } - else { + else if (jl_is_code_instance(lival.constant)) { ci = lival.constant; - assert(jl_is_code_instance(ci)); mi = jl_get_ci_mi((jl_code_instance_t*)ci); + } else { + emit_error(ctx, "(Internal ERROR - IR Validity): Invoke target is not a method instance or code instance"); + return jl_cgval_t(); } assert(jl_is_method_instance(mi)); if (mi == ctx.linfo) { From fe7dc7d38dc86eefea15b04dacf2e05230b53ab9 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 4 Jun 2025 19:26:03 -0400 Subject: [PATCH 371/662] docs: add commit message formatting guidelines to AGENTS.md (#58629) Add two additional guidelines for commit message formatting: - When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., owner/repo#123) - When fixing CI failures, include the link to the specific CI failure in the commit message These guidelines help improve commit message clarity and traceability. --- AGENTS.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index e5d9875856df4..38a978617a966 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,17 @@ corresponding test to ensure that the test is still passing with your changes. - Write one comment at the top of the test to explain what is being tested. Otherwise keep comments minimal. +### External dependencies + +When modifying external dependencies (patches in `deps/patches/` or version updates in `deps/`): + +1. Always test builds with `USE_BINARYBUILDER=0` to ensure source builds work correctly +2. For patches to external libraries: + - Verify the patch applies cleanly by running the extraction and patch steps + - Test the full build of the dependency: `make -C deps USE_BINARYBUILDER=0 compile-` + - Prefer using the full upstream commit in `git am` format (e.g., `git format-patch`) which includes proper commit metadata +3. When updating dependency versions, ensure all associated patches still apply + ### Writing code After writing code, look up the docstring for each function you used. If there are recommendations or additional considerations that apply to these functions, @@ -69,6 +80,9 @@ documentation, etc., unless this is the main purpose of the change. Do not menti the test plan, unless it differs from what you were instructed to do in AGENTS.md. If your change fixes one or more issues, use the syntax "Fixes #" at the end of the commit message, but do not include it in the title. +When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., `owner/repo#123` for PRs/issues). +When fixing CI failures, include the link to the specific CI failure in the commit message. + When creating pull requests, if the pull request consists of one commit only, use the body of the commit for the body of the pull request. If there are multiple commits in the pull request, follow the same guidelines for the pull request From 906d3482d318868e7d3eef6b1db97ef5bedd6f82 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 4 Jun 2025 20:30:23 -0300 Subject: [PATCH 372/662] Make late gc lower handle insertelement of alloca use. (#58637) This was in DAECompiler.jl code found by @serenity4. He also mentioned that writing up how one might go and fix a bug like this so i'll give a quick writeup (this was a very simple bug so it might not be too interesting) The original crash which looked something like > %19 = alloca [10 x i64], align 8 %155 = insertelement <4 x ptr> poison, ptr %19, i32 0 Unexpected instruction > [898844] signal 6 (-6): Aborted in expression starting at /home/gbaraldi/DAECompiler.jl/test/reflection.jl:28 pthread_kill at /lib/x86_64-linux-gnu/libc.so.6 (unknown line) gsignal at /lib/x86_64-linux-gnu/libc.so.6 (unknown line) abort at /lib/x86_64-linux-gnu/libc.so.6 (unknown line) RecursivelyVisit, int, State&, std::map >):::: > at /home/gbaraldi/julia4/src/llvm-late-gc-lowering.cpp:803 operator() at /home/gbaraldi/julia4/src/llvm-late-gc-lowering.cpp:2560 [inlined] PlaceRootsAndUpdateCalls at /home/gbaraldi/julia4/src/llvm-late-gc-lowering.cpp:2576 runOnFunction at /home/gbaraldi/julia4/src/llvm-late-gc-lowering.cpp:2638 run at /home/gbaraldi/julia4/src/llvm-late-gc-lowering.cpp:2675 run at /home/gbaraldi/julia4/usr/include/llvm/IR/PassManagerInternal.h:91 which means it was crashing inside of late-gc-lowering, so the first thing I did was ran julia and the same test with LLVM_ASSERTIONS=1 and FORCE_ASSERTIONS=1 to see if LLVM complained about a malformed module, and both were fine. Next step was trying to get the failing code out for inspection. Easiest way is to do `export JULIA_LLVM_ARGS="--print-before=LateLowerGCFrame --print-module-scope"` and pipe the output to a file. The file is huge, but since it's a crash in LLVM we know that the last thing is what we want, and that gave me the IR I wanted. To verify that this is failing I did `make -C src install-analysis-deps` to install the LLVM machinery (opt...). That gets put in the `tools` directory of a julia build. Then I checked if this crashed outside of julia by doing `./opt -load-pass-plugin=../lib/libjulia-codegen.dylib --passes=LateLowerGCFrame -S test.ll -o tmp3.ll `. This is run from inside the tools dir so your paths might vary (the -S is so LLVM doesn't generate bitcode) and my code did crash, however it was over 500 lines of IR which makes it harder to debug and to write a test. Next step then is to minimize the crash by doing [`llvm-reduce`](https://llvm.org/docs/CommandGuide/llvm-reduce.html) over it (it's basically creduce but optimized for LLVM IR) which gave me a 2 line reproducer (in this case apparently just having the insertelement was enough for the pass to fail). One thing to be wary is that llvm-reduce will usually make very weird code, so it might be useful to modify the code slightly so it doesn't look odd (it will have unreachable basic-blocks and such). After the cleanup fixing the bug here wasn't interesting but this doesn't apply generally. And also always transform your reduced IR into a test to put in llvmpasses. --- src/llvm-late-gc-lowering.cpp | 2 +- test/llvmpasses/late-lower-gc.ll | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 4ee7e4b30e61c..22d730621b80c 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -790,7 +790,7 @@ void RecursivelyVisit(callback f, Value *V) { if (isa(TheUser) || isa(TheUser) || isa(TheUser) || isa(TheUser) || // TODO: should these be removed from this list? isa(TheUser) || isa(TheUser) || - isa(TheUser) || // ICmpEQ/ICmpNE can be used with ptr types + isa(TheUser) || isa(TheUser)|| // ICmpEQ/ICmpNE can be used with ptr types isa(TheUser) || isa(TheUser)) continue; if (isa(TheUser) || isa(TheUser) || isa(TheUser)) { diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index af71d3c8d2c75..c3d6ea10c1511 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -199,6 +199,20 @@ define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { ; CHECK: %r = call i32 @callee_root(ptr addrspace(10) %l0, ptr addrspace(10) %l1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) +define swiftcc ptr addrspace(10) @insert_element(ptr swiftself %0) { +; CHECK-LABEL: @insert_element + %2 = alloca [10 x i64], i32 1, align 8 +; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 10) +; CHECK: [[gc_slot_addr_:%.*]] = call ptr @julia.get_gc_frame_slot(ptr %gcframe, i32 0) +; CHECK: call void @julia.push_gc_frame(ptr %gcframe, i32 10) + call void null(ptr sret([2 x [5 x ptr addrspace(10)]]) %2, ptr null, ptr addrspace(11) null, ptr null) + %4 = insertelement <4 x ptr> zeroinitializer, ptr %2, i32 0 +; CHECK: [[gc_slot_addr_:%.*]] = insertelement <4 x ptr> zeroinitializer, ptr [[gc_slot_addr_:%.*]], i32 0 +; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) + ret ptr addrspace(10) null +} + + !0 = !{i64 0, i64 23} !1 = !{!1} !2 = !{!7} ; scope list From 7cb88d6951c9725f05426fcb110ee49bec4b147e Mon Sep 17 00:00:00 2001 From: Max Horn Date: Thu, 5 Jun 2025 05:55:19 +0200 Subject: [PATCH 373/662] Detect Apple M4 and some related changes (#58301) The A15 was detected as M2; added codenames for easier future updates. Sources: - https://asahilinux.org/docs/hw/soc/soc-codenames/#socs - https://github.com/apple-oss-distributions/xnu/blob/main/osfmk/arm/cpuid.h - https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64Processors.td#L428 Missing: - the M4 Pro and Max are missing (because they are missing from Apple's `cpuid.h`) Resolves #58278 --------- Co-authored-by: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> --- src/processor_arm.cpp | 79 +++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index 66704a718a14d..b73f5c3e4bbac 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -166,9 +166,11 @@ enum class CPU : uint32_t { apple_a14, apple_a15, apple_a16, + apple_a17, apple_m1, apple_m2, apple_m3, + apple_m4, apple_s4, apple_s5, @@ -355,9 +357,11 @@ constexpr auto apple_a13 = armv8_4a_crypto | get_feature_masks(fp16fml, fullfp16 constexpr auto apple_a14 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3); constexpr auto apple_a15 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_a16 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); +constexpr auto apple_a17 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_m1 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3); constexpr auto apple_m2 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_m3 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); +constexpr auto apple_m4 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); // Features based on https://github.com/llvm/llvm-project/blob/82507f1798768280cf5d5aab95caaafbc7fe6f47/llvm/include/llvm/Support/AArch64TargetParser.def // and sysctl -a hw.optional constexpr auto apple_s4 = apple_a12; @@ -441,9 +445,11 @@ static constexpr CPUSpec cpus[] = { {"apple-a14", CPU::apple_a14, CPU::apple_a13, 120000, Feature::apple_a14}, {"apple-a15", CPU::apple_a15, CPU::apple_a14, 160000, Feature::apple_a15}, {"apple-a16", CPU::apple_a16, CPU::apple_a14, 160000, Feature::apple_a16}, + {"apple-a17", CPU::apple_a17, CPU::apple_a16, 190000, Feature::apple_a17}, {"apple-m1", CPU::apple_m1, CPU::apple_a14, 130000, Feature::apple_m1}, {"apple-m2", CPU::apple_m2, CPU::apple_m1, 160000, Feature::apple_m2}, {"apple-m3", CPU::apple_m3, CPU::apple_m2, 180000, Feature::apple_m3}, + {"apple-m4", CPU::apple_m4, CPU::apple_m3, 190000, Feature::apple_m4}, {"apple-s4", CPU::apple_s4, CPU::generic, 100000, Feature::apple_s4}, {"apple-s5", CPU::apple_s5, CPU::generic, 100000, Feature::apple_s5}, {"thunderx3t110", CPU::marvell_thunderx3t110, CPU::cavium_thunderx2t99, 110000, @@ -722,6 +728,8 @@ static NOINLINE std::pair> _get_host_cpu() return std::make_pair((uint32_t)CPU::apple_m2, Feature::apple_m2); else if (cpu_name.find("M3") != StringRef ::npos) return std::make_pair((uint32_t)CPU::apple_m3, Feature::apple_m3); + else if (cpu_name.find("M4") != StringRef ::npos) + return std::make_pair((uint32_t)CPU::apple_m4, Feature::apple_m4); else return std::make_pair((uint32_t)CPU::apple_m1, Feature::apple_m1); } @@ -1042,7 +1050,10 @@ static CPU get_cpu_name(CPUID cpuid) default: return CPU::generic; } case 0x61: // 'a': Apple - // https://opensource.apple.com/source/xnu/xnu-7195.141.2/osfmk/arm/cpuid.h.auto.html + // Data here is partially based on these sources: + // https://github.com/apple-oss-distributions/xnu/blob/main/osfmk/arm/cpuid.h + // https://asahilinux.org/docs/hw/soc/soc-codenames/#socs + // https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64Processors.td switch (cpuid.part) { case 0x0: // Swift return CPU::apple_swift; @@ -1067,31 +1078,57 @@ static CPU get_cpu_name(CPUID cpuid) return CPU::apple_a12; case 0xF: // Tempest M9 return CPU::apple_s4; - case 0x12: // Lightning - case 0x13: // Thunder + case 0x12: // H12 Cebu p-Core "Lightning" + case 0x13: // H12 Cebu e-Core "Thunder" return CPU::apple_a13; - case 0x20: // Icestorm - case 0x21: // Firestorm + case 0x20: // H13 Sicily e-Core "Icestorm" + case 0x21: // H13 Sicily p-Core "Firestorm" return CPU::apple_a14; - case 0x22: // Icestorm m1 - case 0x23: // Firestorm m1 - case 0x24: - case 0x25: // From https://github.com/AsahiLinux/m1n1/blob/3b9a71422e45209ef57c563e418f877bf54358be/src/chickens.c#L9 - case 0x28: - case 0x29: + case 0x22: // H13G Tonga e-Core "Icestorm" used in Apple M1 + case 0x23: // H13G Tonga p-Core "Firestorm" used in Apple M1 + case 0x24: // H13J Jade Chop e-Core "Icestorm" used in Apple M1 Pro + case 0x25: // H13J Jade Chop p-Core "Firestorm" used in Apple M1 Pro + case 0x28: // H13J Jade Die e-Core "Icestorm" used in Apple M1 Max / Ultra + case 0x29: // H13J Jade Die p-Core "Firestorm" used in Apple M1 Max / Ultra return CPU::apple_m1; - case 0x30: // Blizzard m2 - case 0x31: // Avalanche m2 - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x38: - case 0x39: + case 0x30: // H14 Ellis e-Core "Blizzard" used in Apple A15 + case 0x31: // H14 Ellis p-Core "Avalanche" used in Apple A15 + return CPU::apple_a15; + case 0x32: // H14G Staten e-Core "Blizzard" used in Apple M2 + case 0x33: // H14G Staten p-Core "Avalanche" used in Apple M2 + case 0x34: // H14S Rhodes Chop e-Core "Blizzard" used in Apple M2 Pro + case 0x35: // H14S Rhodes Chop p-Core "Avalanche" used in Apple M2 Pro + case 0x38: // H14C Rhodes Die e-Core "Blizzard" used in Apple M2 Max / Ultra + case 0x39: // H14C Rhodes Die p-Core "Avalanche" used in Apple M2 Max / Ultra return CPU::apple_m2; - case 0x49: // Everest m3 - case 0x48: // Sawtooth m3 + case 0x40: // H15 Crete e-Core "Sawtooth" used in Apple A16 + case 0x41: // H15 Crete p-Core "Everest" used in Apple A16 + return CPU::apple_a16; + case 0x42: // H15 Ibiza e-Core "Sawtooth" used in Apple M3 + case 0x43: // H15 Ibiza p-Core "Everest" used in Apple M3 + case 0x44: // H15 Lobos e-Core "Sawtooth" used in Apple M3 Pro + case 0x45: // H15 Lobos p-Core "Everest" used in Apple M3 Pro + case 0x49: // H15 Palma e-Core "Sawtooth" used in Apple M3 Max + case 0x48: // H15 Palma p-Core "Everest" used in Apple M3 Max return CPU::apple_m3; + //case 0x46: // M11 e-Core "Sawtooth" used in Apple S9 + //case 0x47: does not exist + //return CPU::apple_s9; + case 0x50: // H15 Coll e-Core "Sawtooth" used in Apple A17 Pro + case 0x51: // H15 Coll p-Core "Everest" used in Apple A17 Pro + return CPU::apple_a17; + case 0x52: // H16G Donan e-Core used in Apple M4 + case 0x53: // H16H Donan p-Core used in Apple M4 + case 0x54: // H16S Brava S e-Core used in Apple M4 Pro + case 0x55: // H16S Brava S p-Core used in Apple M4 Pro + case 0x58: // H16C Brava C e-Core used in Apple M4 Max + case 0x59: // H16C Brava C p-Core used in Apple M4 Max + return CPU::apple_m4; + //case 0x60: // H17P Tahiti e-Core used in Apple A18 Pro + //case 0x61: // H17P Tahiti p-Core used in Apple A18 Pro + //case 0x6a: // H17A Tupai e-Core used in Apple A18 + //case 0x6b: // H17A Tupai p-Core used in Apple A18 + //return CPU::apple_a18; default: return CPU::generic; } case 0x68: // 'h': Huaxintong Semiconductor From 4632cd28b9acfac832970c6562c1aab0e3169c3a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 5 Jun 2025 03:15:49 -0400 Subject: [PATCH 374/662] deps: fix libunwind build with GCC < 11 (#58628) Apply patch from libunwind/libunwind#853 to add missing parameter names in _UPT_ptrauth_insn_mask function to comply with ISO/IEC 9899. Prior to GCC 11, omitting parameter names in function definitions was an error. GCC 11 changed this behavior in commit https://gcc.gnu.org/pipermail/gcc-cvs/2020-October/336068.html to allow omitted parameter names as a C2x extension, making it a pedantic warning instead of an error. Since we build without -Wpedantic, the code compiles fine with GCC 11+ but fails with older GCC versions. Fixes scheduled CI failures in libunwind build: https://buildkite.com/julialang/julia-master-scheduled/builds/1160 --- .../libunwind-missing-parameter-names.patch | 28 +++++++++++++++++++ deps/unwind.mk | 6 +++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 deps/patches/libunwind-missing-parameter-names.patch diff --git a/deps/patches/libunwind-missing-parameter-names.patch b/deps/patches/libunwind-missing-parameter-names.patch new file mode 100644 index 0000000000000..643de108bf84b --- /dev/null +++ b/deps/patches/libunwind-missing-parameter-names.patch @@ -0,0 +1,28 @@ +From 9ebe5e12b8a0063953f9ef196b2433eca9933559 Mon Sep 17 00:00:00 2001 +From: Stephen Webb +Date: Tue, 15 Apr 2025 10:48:20 -0400 +Subject: [PATCH] Fix FTBFS in src/ptrace/_UPT_ptrauth_insn_mask.c + +Added missing parameter names to make C code comply to ISO/IEC 9899. +--- + src/ptrace/_UPT_ptrauth_insn_mask.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/ptrace/_UPT_ptrauth_insn_mask.c b/src/ptrace/_UPT_ptrauth_insn_mask.c +index dcc512370..e7b3a514b 100644 +--- a/src/ptrace/_UPT_ptrauth_insn_mask.c ++++ b/src/ptrace/_UPT_ptrauth_insn_mask.c +@@ -49,9 +49,10 @@ unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, void *arg) + + #else + +-unw_word_t _UPT_ptrauth_insn_mask (unw_addr_space_t, void *) ++unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, UNUSED void *arg) + { + return 0; + } + +-#endif +\ No newline at end of file ++#endif ++ \ No newline at end of file diff --git a/deps/unwind.mk b/deps/unwind.mk index 08d0059c89731..0808ae6ca4b1c 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -39,10 +39,14 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-app cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-disable-initial-exec-tls.patch echo 1 > $@ +$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied + cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-missing-parameter-names.patch + echo 1 > $@ + # note minidebuginfo requires liblzma, which we do not have a source build for # (it will be enabled in BinaryBuilder-based downloads however) # since https://github.com/JuliaPackaging/Yggdrasil/commit/0149e021be9badcb331007c62442a4f554f3003c -$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied +$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache From f5e983eafe6c879027217808a29a9c12bb27f5af Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 5 Jun 2025 08:49:12 -0400 Subject: [PATCH 375/662] Update install link in warning (#58638) --- stdlib/InteractiveUtils/src/InteractiveUtils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index d9d00851dbfa2..0f69860f39d78 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -120,7 +120,7 @@ function versioninfo(io::IO=stdout; verbose::Bool=false) Note: This is an unofficial build, please report bugs to the project responsible for this build and not to the Julia project unless you can - reproduce the issue using official builds available at https://julialang.org/downloads + reproduce the issue using official builds available at https://julialang.org """ ) end From a4ab11016c0d19cd56b1662afae906afaf8f7a3d Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 5 Jun 2025 09:23:26 -0400 Subject: [PATCH 376/662] test: replcompletions: Replace timedwait by proper condvar (#58643) Replace fragile timing-based synchronization with proper condition variable signaling to ensure PATH cache updates complete before testing completions. This eliminates test flakiness on systems where the cache update takes longer than 5s. The test failure was seen in CI: https://buildkite.com/julialang/julia-master/builds/48273 --- stdlib/REPL/src/REPLCompletions.jl | 12 +++++--- stdlib/REPL/test/replcompletions.jl | 47 +++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index d7b0b9c6d0b89..7de1246566b48 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -330,7 +330,8 @@ end const PATH_cache_lock = Base.ReentrantLock() const PATH_cache = Set{String}() -PATH_cache_task::Union{Task,Nothing} = nothing # used for sync in tests +PATH_cache_task::Union{Task,Nothing} = nothing +PATH_cache_condition::Union{Threads.Condition, Nothing} = nothing # used for sync in tests next_cache_update::Float64 = 0.0 function maybe_spawn_cache_PATH() global PATH_cache_task, next_cache_update @@ -339,7 +340,11 @@ function maybe_spawn_cache_PATH() time() < next_cache_update && return PATH_cache_task = Threads.@spawn begin REPLCompletions.cache_PATH() - @lock PATH_cache_lock PATH_cache_task = nothing # release memory when done + @lock PATH_cache_lock begin + next_cache_update = time() + 10 # earliest next update can run is 10s after + PATH_cache_task = nothing # release memory when done + PATH_cache_condition !== nothing && notify(PATH_cache_condition) + end end Base.errormonitor(PATH_cache_task) end @@ -350,8 +355,6 @@ function cache_PATH() path = get(ENV, "PATH", nothing) path isa String || return - global next_cache_update - # Calling empty! on PATH_cache would be annoying for async typing hints as completions would temporarily disappear. # So keep track of what's added this time and at the end remove any that didn't appear this time from the global cache. this_PATH_cache = Set{String}() @@ -414,7 +417,6 @@ function cache_PATH() @lock PATH_cache_lock begin intersect!(PATH_cache, this_PATH_cache) # remove entries from PATH_cache that weren't found this time - next_cache_update = time() + 10 # earliest next update can run is 10s after end @debug "caching PATH files took $t seconds" length(pathdirs) length(PATH_cache) diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 5f2990e970d97..43bea08e285f1 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -1073,6 +1073,39 @@ let c, r, res @test res === false end +# A pair of utility function for the REPL completions test to test PATH_cache +# dependent completions, which ordinarily happen asynchronously. +# Only to be used from the test suite +function test_only_arm_cache_refresh() + @lock REPL.REPLCompletions.PATH_cache_lock begin + @assert REPL.REPLCompletions.PATH_cache_condition === nothing + + # Arm a condition we can wait on + REPL.REPLCompletions.PATH_cache_condition = Threads.Condition(REPL.REPLCompletions.PATH_cache_lock) + + # Check if the previous update is still running - if so, wait for it to finish + while REPL.REPLCompletions.PATH_cache_task !== nothing + @assert !istaskdone(REPL.REPLCompletions.PATH_cache_task) + wait(REPL.REPLCompletions.PATH_cache_condition) + end + + # force the next cache update to happen immediately + REPL.REPLCompletions.next_cache_update = 0 + end + return REPL.REPLCompletions.PATH_cache_condition +end + +function test_only_wait_cache_path_done() + @lock REPL.REPLCompletions.PATH_cache_lock begin + @assert REPL.REPLCompletions.PATH_cache_condition !== nothing + + while REPL.REPLCompletions.next_cache_update == 0. + wait(REPL.REPLCompletions.PATH_cache_condition) + end + REPL.REPLCompletions.PATH_cache_condition = nothing + end +end + if Sys.isunix() let s, c, r #Assume that we can rely on the existence and accessibility of /tmp @@ -1204,12 +1237,9 @@ let s, c, r # Files reachable by PATH are cached async when PATH is seen to have been changed by `complete_path` # so changes are unlikely to appear in the first complete. For testing purposes we can wait for # caching to finish - @lock REPL.REPLCompletions.PATH_cache_lock begin - # force the next cache update to happen immediately - REPL.REPLCompletions.next_cache_update = 0 - end + test_only_arm_cache_refresh() c,r = test_scomplete(s) - timedwait(()->REPL.REPLCompletions.next_cache_update != 0, 5) # wait for caching to complete + test_only_wait_cache_path_done() c,r = test_scomplete(s) @test "tmp-executable" in c @test r == 1:9 @@ -1238,12 +1268,9 @@ let s, c, r withenv("PATH" => string(tempdir(), ":", dir)) do s = string("repl-completio") - @lock REPL.REPLCompletions.PATH_cache_lock begin - # force the next cache update to happen immediately - REPL.REPLCompletions.next_cache_update = 0 - end + test_only_arm_cache_refresh() c,r = test_scomplete(s) - timedwait(()->REPL.REPLCompletions.next_cache_update != 0, 5) # wait for caching to complete + test_only_wait_cache_path_done() c,r = test_scomplete(s) @test ["repl-completion"] == c @test s[r] == "repl-completio" From 7a86a28e52c4946b85cdb200a2171eed8593fd3a Mon Sep 17 00:00:00 2001 From: Arnav Kapoor Date: Thu, 5 Jun 2025 20:25:47 +0530 Subject: [PATCH 377/662] Serialization doc updated (#58575) This PR updates the documentation for Serialization.jl to recommend using the .jls extension for serialized Julia data, following common community conventions. Fixes: https://github.com/JuliaLang/julia/issues/58544 --- stdlib/Serialization/docs/src/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/stdlib/Serialization/docs/src/index.md b/stdlib/Serialization/docs/src/index.md index 0d00e47ed84ce..77c7558e0306a 100644 --- a/stdlib/Serialization/docs/src/index.md +++ b/stdlib/Serialization/docs/src/index.md @@ -11,3 +11,15 @@ Serialization.serialize Serialization.deserialize Serialization.writeheader ``` + +### Recommended File Extension + +While the Serialization module does not mandate a specific file extension, the Julia community commonly uses the `.jls` extension for serialized Julia files. + +Example: + +```julia +open("model.jls", "w") do io + serialize(io, my_model) +end +``` From 2e158a4d9a4c9600964168fd6243f9d7f224503f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 5 Jun 2025 11:21:08 -0400 Subject: [PATCH 378/662] ircode: handle content outside of a module (#58639) Specifically, content in an `__init__` block is handled by secret duplicate precompile logic, and any content generated by it was previously not eligible to be included into cache files. Fix #58449 --- src/precompile.c | 1 - src/staticdata.c | 6 ------ src/toplevel.c | 7 ++----- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/precompile.c b/src/precompile.c index a6ec4d550cfba..8d4e8495fc4bd 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -139,7 +139,6 @@ JL_DLLEXPORT void jl_write_compiler_output(void) } } - assert(jl_precompile_toplevel_module == NULL); void *native_code = NULL; bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm; diff --git a/src/staticdata.c b/src/staticdata.c index eb503fe0ffa78..92e7f494ad35d 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3419,9 +3419,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_array_t *mod_array, jl_array_t **udeps, int64_t *srctextpos, int64_t *checksumpos) { - assert(jl_precompile_toplevel_module == NULL); - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - *checksumpos = write_header(f, 0); write_uint8(f, jl_cache_flags()); // write description of contents (name, uuid, buildid) @@ -3479,9 +3476,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli // Generate _native_data` if (_native_data != NULL) { jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL, &query_cache); - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); *_native_data = jl_precompile_worklist(worklist, extext_methods, new_ext_cis); - jl_precompile_toplevel_module = NULL; extext_methods = NULL; new_ext_cis = NULL; } @@ -3528,7 +3523,6 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli // Re-enable running julia code for postoutput hooks, atexit, etc. jl_gc_enable_finalizers(ct, 1); ct->reentrant_timing &= ~0b1000u; - jl_precompile_toplevel_module = NULL; if (worklist) { // Go back and update the checksum in the header diff --git a/src/toplevel.c b/src/toplevel.c index 8540d8e5f3c56..41ce5a1901dbf 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -34,7 +34,7 @@ htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; // During incremental compilation, the following gets set -jl_module_t *jl_precompile_toplevel_module = NULL; // the toplevel module currently being defined +jl_module_t *jl_precompile_toplevel_module = NULL; // the first toplevel module being defined jl_module_t *jl_add_standard_imports(jl_module_t *m) { @@ -172,7 +172,6 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex } } - jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; size_t last_age = ct->world_age; if (parent_module == jl_main_module && name == jl_symbol("Base") && jl_base_module == NULL) { @@ -182,7 +181,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex if (is_parent__toplevel__) { jl_register_root_module(newm); - if (jl_options.incremental) { + if (jl_options.incremental && jl_precompile_toplevel_module == NULL) { jl_precompile_toplevel_module = newm; } } @@ -241,8 +240,6 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex } } - jl_precompile_toplevel_module = old_toplevel_module; - JL_GC_POP(); return (jl_value_t*)newm; } From 995ea2c8845f0551fd900ba9cd9a882748ee306d Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 5 Jun 2025 21:19:07 -0400 Subject: [PATCH 379/662] Fix whitespace in new libunwind patch (#58649) This whitespace somewhat got lost in the final version that got pushed. Fix it. --- deps/patches/libunwind-missing-parameter-names.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/patches/libunwind-missing-parameter-names.patch b/deps/patches/libunwind-missing-parameter-names.patch index 643de108bf84b..59a33f1a56880 100644 --- a/deps/patches/libunwind-missing-parameter-names.patch +++ b/deps/patches/libunwind-missing-parameter-names.patch @@ -25,4 +25,4 @@ index dcc512370..e7b3a514b 100644 -#endif \ No newline at end of file +#endif -+ \ No newline at end of file ++ From 2a9f33c94dd66ffb96beaea86d595a138a1a8bc7 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:49:05 +0900 Subject: [PATCH 380/662] add `print_signature_only` argument to `Base.show_method` (#58647) --- base/methodshow.jl | 16 ++++++++++------ test/show.jl | 12 ++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index dd391a46d170a..f58a4911494b3 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -214,7 +214,9 @@ show(io::IO, m::Method; kwargs...) = show_method(IOContext(io, :compact=>true), show(io::IO, ::MIME"text/plain", m::Method; kwargs...) = show_method(io, m; kwargs...) -function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_width = 1) +function show_method(io::IO, m::Method; + modulecolor = :light_black, digit_align_width = 1, + print_signature_only::Bool = get(io, :print_method_signature_only, false)::Bool) tv, decls, file, line = arg_decl_parts(m) sig = unwrap_unionall(m.sig) if sig === Tuple @@ -250,12 +252,14 @@ function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_ show_method_params(io, tv) end - if !(get(io, :compact, false)::Bool) # single-line mode - println(io) - digit_align_width += 4 + if !print_signature_only + if !(get(io, :compact, false)::Bool) # single-line mode + println(io) + digit_align_width += 4 + end + # module & file, re-using function from errorshow.jl + print_module_path_file(io, parentmodule(m), string(file), line; modulecolor, digit_align_width) end - # module & file, re-using function from errorshow.jl - print_module_path_file(io, parentmodule(m), string(file), line; modulecolor, digit_align_width) end function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) diff --git a/test/show.jl b/test/show.jl index 8abd99cf95455..063deaefb376e 100644 --- a/test/show.jl +++ b/test/show.jl @@ -2885,3 +2885,15 @@ end @test_repr """:(var"\a\b\t\n\v\f\r\e" = 1)""" @test_repr """:(var"\x01\u03c0\U03c0" = 1)""" end + +# test `print_signature_only::Bool` argument of `Base.show_method` +f_show_method(x::T) where T<:Integer = :integer +let m = only(methods(f_show_method)) + let io = IOBuffer() + Base.show_method(io, m; print_signature_only=true) + @test "f_show_method(x::T) where T<:Integer" == String(take!(io)) + end + let s = sprint(show, m; context=:print_method_signature_only=>true) + @test "f_show_method(x::T) where T<:Integer" == s + end +end From 347fb7c971df89116e92e0831ea2f5322865a411 Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Fri, 6 Jun 2025 14:49:18 +0200 Subject: [PATCH 381/662] Use type wrapper directly rather than typename in `FieldError` (#58507) --- base/errorshow.jl | 4 ++-- test/errorshow.jl | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 8f8a0afbe58ed..64601ea35f548 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -382,7 +382,7 @@ end function showerror(io::IO, exc::FieldError) @nospecialize - print(io, "FieldError: type $(exc.type |> nameof) has no field `$(exc.field)`") + print(io, "FieldError: type $(exc.type.name.wrapper) has no field `$(exc.field)`") Base.Experimental.show_error_hints(io, exc) end @@ -1127,7 +1127,7 @@ Experimental.register_error_hint(fielderror_dict_hint_handler, FieldError) function fielderror_listfields_hint_handler(io, exc) fields = fieldnames(exc.type) if isempty(fields) - print(io, "; $(nameof(exc.type)) has no fields at all.") + print(io, "; $(exc.type.name.wrapper) has no fields at all.") else print(io, ", available fields: $(join(map(k -> "`$k`", fields), ", "))") end diff --git a/test/errorshow.jl b/test/errorshow.jl index 8b225e6907ebc..e37d22718b491 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -857,7 +857,8 @@ end # Check error message first errorMsg = sprint(Base.showerror, ex) - @test occursin("FieldError: type FieldFoo has no field `c`", errorMsg) + @test occursin("FieldError: type", errorMsg) + @test occursin("FieldFoo has no field `c`", errorMsg) @test occursin("available fields: `a`, `b`", errorMsg) @test occursin("Available properties: `x`, `y`", errorMsg) @@ -882,6 +883,24 @@ end @test occursin(hintExpected, errorMsg) end +module FieldErrorTest +struct Point end +p = Point() +end + +@testset "FieldError with changing fields" begin + # https://discourse.julialang.org/t/better-error-message-for-modified-structs-in-julia-1-12/129265 + err_str1 = @except_str FieldErrorTest.p.x FieldError + @test occursin("FieldErrorTest.Point", err_str1) + @eval FieldErrorTest struct Point{T} + x::T + y::T + end + err_str2 = @except_str FieldErrorTest.p.x FieldError + @test occursin("@world", err_str2) + @test occursin("FieldErrorTest.Point", err_str2) +end + # UndefVar error hints module A53000 export f From d2cc06193ef4161e4ac161bd4b5b57a51686a89a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 6 Jun 2025 13:27:54 -0400 Subject: [PATCH 382/662] module: Tweak backdate-warning-turned error (#58651) This makes two changes to the backdate-warning-turned-error (#58266): 1. Fix a bug where the error would only trigger the first time. We do only want to print once per process, but of course, we do want to error every time if enabled. 2. If we are in speculative execution context (generators and speculatively run functions during inference), always use the UndefVarError. Effects from these functions are not supposed to be observable, and it's very confusing if the printed warning goes away when depwarns are enabled. This is marginally more breaking, but the burden is on generated function authors (which already have to be world-age aware and are somewhat more regularly broken) and is consistent with other things that are stronger errors in pure context. Fixes #58648 --- src/module.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 8c6c86b144f38..7c83d9329ab50 100644 --- a/src/module.c +++ b/src/module.c @@ -846,8 +846,6 @@ JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT { - if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) - jl_undefined_var_error(b->globalref->name, (jl_value_t*)b->globalref->mod); jl_safe_printf( "WARNING: Detected access to binding `%s.%s` in a world prior to its definition world.\n" " Julia 1.12 has introduced more strict world age semantics for global bindings.\n" @@ -860,9 +858,13 @@ static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT static inline void check_backdated_binding(jl_binding_t *b, enum jl_partition_kind kind) JL_NOTSAFEPOINT { - if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST) && - !(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) { - print_backdate_admonition(b); + if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST)) { + // We don't want functions that inference executes speculatively to print this warning, so turn those into + // an error for inference purposes. + if (jl_current_task->ptls->in_pure_callback || jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_undefined_var_error(b->globalref->name, (jl_value_t*)b->globalref->mod); + if (!(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) + print_backdate_admonition(b); } } From 67b69dd56ba7da9141b05ef094849f52a2372248 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 6 Jun 2025 14:05:35 -0400 Subject: [PATCH 383/662] CI: `PrAssignee.yml`: Skip BumpStdlibs.jl PRs (PRs authored by `@DilumAluthgeBot`) (#58625) Hat-tip to `@IanButterworth` for reporting this. --------- Co-authored-by: Lilith Orion Hafner --- .github/workflows/PrAssignee.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index a5d8e67fe3706..553a49351d924 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -71,6 +71,12 @@ jobs: ) const allCollaboratorsNested = await Promise.all(allCollaboratorsNestedPromises); const allCollaboratorsFlattened = allCollaboratorsNested.flat(); + + // Skip BumpStdlibs.jl PRs + allCollaboratorsFlattened.push('DilumAluthgeBot'); + // Skip Dependabot PRs + allCollaboratorsFlattened.push('dependabot'); + const isCollaborator = allCollaboratorsFlattened.includes(prAuthor); console.log('prAuthor: ', prAuthor); From 8a0a21362e683a2acc8178b60ff67df01070436b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 14:06:27 -0400 Subject: [PATCH 384/662] Bump julia-actions/setup-julia from 2.6.0 to 2.6.1 (#58656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [julia-actions/setup-julia](https://github.com/julia-actions/setup-julia) from 2.6.0 to 2.6.1.
Release notes

Sourced from julia-actions/setup-julia's releases.

v2.6.1 - Add warning for x64 on apple silicon runners

What's Changed

Maintenance

Full Changelog: https://github.com/julia-actions/setup-julia/compare/v2.6.0...v2.6.1

Commits
  • 5c9647d Bump @​types/node from 22.7.5 to 22.9.0 (#301)
  • 17468e8 Bump @​types/jest from 29.5.13 to 29.5.14 (#298)
  • c861e46 add warning if requesting x64 on apple silicon runners (#300)
  • 2fa1802 README examples: '1' for the latest v1 (#296)
  • 05e75bd Bump julia-actions/setup-julia from 2.5.0 to 2.6.0 (#295)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=julia-actions/setup-julia&package-manager=github_actions&previous-version=2.6.0&new-version=2.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/Whitespace.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Whitespace.yml b/.github/workflows/Whitespace.yml index 37c9dbfd39a3c..0855811c4ece6 100644 --- a/.github/workflows/Whitespace.yml +++ b/.github/workflows/Whitespace.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: persist-credentials: false - - uses: julia-actions/setup-julia@9b79636afcfb07ab02c256cede01fe2db6ba808c # v2.6.0 + - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1 with: version: '1' - name: Check whitespace From 3682c0693142c499595b26b03a7b7c9d9440f048 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 14:06:46 -0400 Subject: [PATCH 385/662] Bump actions/checkout from 4.1.7 to 4.2.2 (#58657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.2.
Release notes

Sourced from actions/checkout's releases.

v4.2.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v4.2.1...v4.2.2

v4.2.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.2.0...v4.2.1

v4.2.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.1.7...v4.2.0

Changelog

Sourced from actions/checkout's changelog.

Changelog

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

v4.1.4

v4.1.3

v4.1.2

v4.1.1

v4.1.0

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=4.1.7&new-version=4.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/Typos.yml | 2 +- .github/workflows/Whitespace.yml | 2 +- .github/workflows/cffconvert.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Typos.yml b/.github/workflows/Typos.yml index 6c9eeacc21800..4dcfdcf0095b9 100644 --- a/.github/workflows/Typos.yml +++ b/.github/workflows/Typos.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check spelling with typos diff --git a/.github/workflows/Whitespace.yml b/.github/workflows/Whitespace.yml index 0855811c4ece6..7414365322292 100644 --- a/.github/workflows/Whitespace.yml +++ b/.github/workflows/Whitespace.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 2 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1 diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml index 4c9debb246f3f..3e481f18e6f75 100644 --- a/.github/workflows/cffconvert.yml +++ b/.github/workflows/cffconvert.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false From dece143a94665f4511b9faddc5b160e56a20bc8f Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Fri, 6 Jun 2025 17:40:51 -0300 Subject: [PATCH 386/662] Create 'JL_GC_DECODE_NROOTS' macro to avoid writing explicit bitshifts in a few places (#58658) See PR title. CC: @udesou, @qinsoon. --- src/julia.h | 1 + src/subtype.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/julia.h b/src/julia.h index 6c1c8af0a788b..fff465b0ab114 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1139,6 +1139,7 @@ struct _jl_gcframe_t { #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<2) #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<2)|1) +#define JL_GC_DECODE_NROOTS(n) (n >> 2) #ifdef __clang_gcanalyzer__ diff --git a/src/subtype.c b/src/subtype.c index 19e5b4bd222b2..d2331b289818c 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -274,7 +274,7 @@ static void re_save_env(jl_stenv_t *e, jl_savedenv_t *se, int root) } else { roots = se->roots; - nroots = se->gcframe.nroots >> 2; + nroots = JL_GC_DECODE_NROOTS(se->gcframe.nroots); } } jl_varbinding_t *v = e->vars; @@ -367,7 +367,7 @@ static void restore_env(jl_stenv_t *e, jl_savedenv_t *se, int root) JL_NOTSAFEPO } else { roots = se->roots; - nroots = se->gcframe.nroots >> 2; + nroots = JL_GC_DECODE_NROOTS(se->gcframe.nroots); } } jl_varbinding_t *v = e->vars; @@ -4295,7 +4295,7 @@ static int merge_env(jl_stenv_t *e, jl_savedenv_t *me, jl_savedenv_t *se, int co else { saved = se->roots; merged = me->roots; - nroots = se->gcframe.nroots >> 2; + nroots = JL_GC_DECODE_NROOTS(se->gcframe.nroots); } assert(nroots == current_env_length(e) * 3); assert(nroots % 3 == 0); From 51214a752404a639e68a632a64a7b9e75ce1a4fa Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Sat, 7 Jun 2025 06:07:19 -0300 Subject: [PATCH 387/662] Make pool allocator stats from gc_page_fragmentation_stats more visible for Julia user code (#58659) - Mark `gc_page_fragmentation_stats` DLL_EXPORT so that we can load it through `cglobal`. - Avoid resetting data from `gc_page_fragmentation_stats` after sweeping to ensure it reflects the page stats from the last GC. - Some minor/cosmetic function renaming. --- src/gc-stock.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/gc-stock.c b/src/gc-stock.c index d518df6dfb52a..d7491f8d6bcf1 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -818,20 +818,28 @@ int jl_gc_classify_pools(size_t sz, int *osize) // sweep phase -gc_fragmentation_stat_t gc_page_fragmentation_stats[JL_GC_N_POOLS]; +JL_DLLEXPORT gc_fragmentation_stat_t jl_gc_page_fragmentation_stats[JL_GC_N_POOLS]; JL_DLLEXPORT double jl_gc_page_utilization_stats[JL_GC_N_MAX_POOLS]; -STATIC_INLINE void gc_update_page_fragmentation_data(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +STATIC_INLINE void gc_update_fragmentation_data_for_size_class(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { - gc_fragmentation_stat_t *stats = &gc_page_fragmentation_stats[pg->pool_n]; + gc_fragmentation_stat_t *stats = &jl_gc_page_fragmentation_stats[pg->pool_n]; jl_atomic_fetch_add_relaxed(&stats->n_freed_objs, pg->nfree); jl_atomic_fetch_add_relaxed(&stats->n_pages_allocd, 1); } -STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT +STATIC_INLINE void gc_reset_fragmentation_data_for_size_classes(void) JL_NOTSAFEPOINT { for (int i = 0; i < JL_GC_N_POOLS; i++) { - gc_fragmentation_stat_t *stats = &gc_page_fragmentation_stats[i]; + jl_atomic_store_relaxed(&jl_gc_page_fragmentation_stats[i].n_freed_objs, 0); + jl_atomic_store_relaxed(&jl_gc_page_fragmentation_stats[i].n_pages_allocd, 0); + } +} + +STATIC_INLINE void gc_compute_utilization_data_for_size_classes(void) JL_NOTSAFEPOINT +{ + for (int i = 0; i < JL_GC_N_POOLS; i++) { + gc_fragmentation_stat_t *stats = &jl_gc_page_fragmentation_stats[i]; double utilization = 1.0; size_t n_freed_objs = jl_atomic_load_relaxed(&stats->n_freed_objs); size_t n_pages_allocd = jl_atomic_load_relaxed(&stats->n_pages_allocd); @@ -839,8 +847,6 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT utilization -= ((double)n_freed_objs * (double)jl_gc_sizeclasses[i]) / (double)n_pages_allocd / (double)GC_PAGE_SZ; } jl_gc_page_utilization_stats[i] = utilization; - jl_atomic_store_relaxed(&stats->n_freed_objs, 0); - jl_atomic_store_relaxed(&stats->n_pages_allocd, 0); } } @@ -948,7 +954,7 @@ static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_ done: if (re_use_page) { - gc_update_page_fragmentation_data(pg); + gc_update_fragmentation_data_for_size_class(pg); push_lf_back(allocd, pg); } else { @@ -1388,6 +1394,7 @@ static void gc_sweep_pool(void) // the actual sweeping jl_gc_padded_page_stack_t *new_gc_allocd_scratch = (jl_gc_padded_page_stack_t *) calloc_s(n_threads * sizeof(jl_gc_padded_page_stack_t)); jl_ptls_t ptls = jl_current_task->ptls; + gc_reset_fragmentation_data_for_size_classes(); gc_sweep_wake_all_pages(ptls, new_gc_allocd_scratch); gc_sweep_pool_parallel(ptls); gc_sweep_wait_for_all_pages(); @@ -1455,7 +1462,7 @@ static void gc_sweep_pool(void) #else gc_free_pages(); #endif - gc_dump_page_utilization_data(); + gc_compute_utilization_data_for_size_classes(); gc_time_pool_end(current_sweep_full); } From dda37f97d6d41c01ecea793e9603fc51ff4d6789 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 7 Jun 2025 23:10:32 -0700 Subject: [PATCH 388/662] Try workaround for https://github.com/JuliaLang/www.julialang.org/issues/2291 This dodges the issue on my machine, let's see if it works for everyone. --- contrib/mac/app/startup.applescript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mac/app/startup.applescript b/contrib/mac/app/startup.applescript index 9964049f34ed6..45ccbbb977d25 100644 --- a/contrib/mac/app/startup.applescript +++ b/contrib/mac/app/startup.applescript @@ -1,4 +1,4 @@ set RootPath to (path to me) set JuliaPath to POSIX path of ((RootPath as text) & "Contents:Resources:julia:bin:julia") set JuliaFile to POSIX file JuliaPath -tell application id "com.apple.finder" to open JuliaFile +do shell script "open -a Terminal '" & JuliaFile & "'" From 5610e21199d61623ac8fda7c9ac99ab3a4a32c8b Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 8 Jun 2025 06:29:56 -0400 Subject: [PATCH 389/662] Fix and test stdlib JLL deps on Windows (#58560) --- base/binaryplatforms.jl | 16 +- .../src/CompilerSupportLibraries_jll.jl | 186 ++++++---- stdlib/GMP_jll/src/GMP_jll.jl | 61 ++-- .../src/LLVMLibUnwind_jll.jl | 6 +- stdlib/LibCURL_jll/Project.toml | 8 +- stdlib/LibCURL_jll/src/LibCURL_jll.jl | 49 ++- stdlib/LibGit2_jll/Project.toml | 2 + stdlib/LibGit2_jll/src/LibGit2_jll.jl | 49 ++- stdlib/LibSSH2_jll/Project.toml | 2 + stdlib/LibSSH2_jll/src/LibSSH2_jll.jl | 60 ++-- stdlib/LibUnwind_jll/src/LibUnwind_jll.jl | 29 +- stdlib/MPFR_jll/Project.toml | 4 +- stdlib/MPFR_jll/src/MPFR_jll.jl | 38 +- stdlib/Manifest.toml | 16 +- stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl | 46 +-- stdlib/OpenLibm_jll/Project.toml | 4 +- stdlib/OpenLibm_jll/src/OpenLibm_jll.jl | 33 +- stdlib/OpenSSL_jll/src/OpenSSL_jll.jl | 54 +-- stdlib/PCRE2_jll/src/PCRE2_jll.jl | 24 +- stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl | 307 +++++++++++----- stdlib/Zlib_jll/src/Zlib_jll.jl | 23 +- stdlib/Zstd_jll/Project.toml | 2 + stdlib/Zstd_jll/src/Zstd_jll.jl | 35 +- stdlib/dSFMT_jll/src/dSFMT_jll.jl | 22 +- stdlib/libLLVM_jll/src/libLLVM_jll.jl | 46 +-- .../src/libblastrampoline_jll.jl | 27 +- stdlib/nghttp2_jll/Project.toml | 4 +- stdlib/nghttp2_jll/src/nghttp2_jll.jl | 33 +- test/stdlib_dependencies.jl | 328 +++++++++++------- 29 files changed, 948 insertions(+), 566 deletions(-) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 86fd9118b6738..1d01fcf2d202f 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -796,6 +796,17 @@ function platform_dlext(p::AbstractPlatform = HostPlatform()) end end +# Not general purpose, just for parse_dl_name_version +function _this_os_name() + if Sys.iswindows() + return "windows" + elseif Sys.isapple() + return "macos" + else + return "other" + end +end + """ parse_dl_name_version(path::String, platform::AbstractPlatform) @@ -806,9 +817,10 @@ valid dynamic library, this method throws an error. If no soversion can be extracted from the filename, as in "libbar.so" this method returns `"libbar", nothing`. """ -function parse_dl_name_version(path::String, os::String) +function parse_dl_name_version(path::String, os::String=_this_os_name()) # Use an extraction regex that matches the given OS local dlregex + # Keep this up to date with _this_os_name if os == "windows" # On Windows, libraries look like `libnettle-6.dll` dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$"sa @@ -837,7 +849,7 @@ function parse_dl_name_version(path::String, os::String) end # Adapter for `AbstractString` -function parse_dl_name_version(path::AbstractString, os::AbstractString) +function parse_dl_name_version(path::AbstractString, os::AbstractString=_this_os_name()) return parse_dl_name_version(string(path)::String, string(os)::String) end diff --git a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl index b7f887da3799e..9a0729c50d01f 100644 --- a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl +++ b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl @@ -13,75 +13,123 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libgcc_s_path::String = "" -libgfortran_path::String = "" -libstdcxx_path::String = "" -libgomp_path::String = "" -if Sys.iswindows() - const _libatomic_path = BundledLazyLibraryPath("libatomic-1.dll") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath-0.dll") - if arch(HostPlatform()) == "x86_64" - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s_seh-1.dll") - else - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s_sjlj-1.dll") - end - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran-", libgfortran_version(HostPlatform()).major, ".dll")) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++-6.dll") - const _libgomp_path = BundledLazyLibraryPath("libgomp-1.dll") - const _libssp_path = BundledLazyLibraryPath("libssp-0.dll") -elseif Sys.isapple() - const _libatomic_path = BundledLazyLibraryPath("libatomic.1.dylib") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath.0.dylib") - if arch(HostPlatform()) == "aarch64" || libgfortran_version(HostPlatform()) == v"5" - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.1.1.dylib") - else - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.1.dylib") - end - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran.", libgfortran_version(HostPlatform()).major, ".dylib")) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++.6.dylib") - const _libgomp_path = BundledLazyLibraryPath("libgomp.1.dylib") - const _libssp_path = BundledLazyLibraryPath("libssp.0.dylib") -else - if Sys.isfreebsd() - const _libatomic_path = BundledLazyLibraryPath("libatomic.so.3") +libatomic_path::String = "" +const libatomic = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libatomic-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libatomic.1.dylib") + elseif Sys.isfreebsd() + BundledLazyLibraryPath("libatomic.so.3") + elseif Sys.islinux() + BundledLazyLibraryPath("libatomic.so.1") else - const _libatomic_path = BundledLazyLibraryPath("libatomic.so.1") - end - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.so.1") - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran.so.", libgfortran_version(HostPlatform()).major)) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++.so.6") - const _libgomp_path = BundledLazyLibraryPath("libgomp.so.1") - if libc(HostPlatform()) != "musl" - const _libssp_path = BundledLazyLibraryPath("libssp.so.0") + error("CompilerSupportLibraries_jll: Library 'libatomic' is not available for $(Sys.KERNEL)") end - if arch(HostPlatform()) ∈ ("x86_64", "i686") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath.so.0") - end -end +) -if @isdefined(_libatomic_path) - const libatomic = LazyLibrary(_libatomic_path) +if Sys.iswindows() || Sys.isapple() || arch(HostPlatform()) ∈ ("x86_64", "i686") + global libquadmath_path::String = "" + const libquadmath = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libquadmath-0.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libquadmath.0.dylib") + elseif (Sys.islinux() || Sys.isfreebsd()) && arch(HostPlatform()) ∈ ("x86_64", "i686") + BundledLazyLibraryPath("libquadmath.so.0") + else + error("CompilerSupportLibraries_jll: Library 'libquadmath' is not available for $(Sys.KERNEL)") + end + ) end -const libgcc_s = LazyLibrary(_libgcc_s_path) -_libgfortran_deps = [libgcc_s] -if @isdefined _libquadmath_path - const libquadmath = LazyLibrary(_libquadmath_path) - push!(_libgfortran_deps, libquadmath) -end +libgcc_s_path::String = "" +const libgcc_s = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libgcc_s_seh-1.dll") + else + BundledLazyLibraryPath("libgcc_s_sjlj-1.dll") + end + elseif Sys.isapple() + if arch(HostPlatform()) == "aarch64" || libgfortran_version(HostPlatform()) == v"5" + BundledLazyLibraryPath("libgcc_s.1.1.dylib") + else + BundledLazyLibraryPath("libgcc_s.1.dylib") + end + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgcc_s.so.1") + else + error("CompilerSupportLibraries_jll: Library 'libgcc_s' is not available for $(Sys.KERNEL)") + end +) -const libgfortran = LazyLibrary(_libgfortran_path, dependencies=_libgfortran_deps) +libgfortran_path::String = "" +const libgfortran = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath(string("libgfortran-", libgfortran_version(HostPlatform()).major, ".dll")) + elseif Sys.isapple() + BundledLazyLibraryPath(string("libgfortran.", libgfortran_version(HostPlatform()).major, ".dylib")) + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath(string("libgfortran.so.", libgfortran_version(HostPlatform()).major)) + else + error("CompilerSupportLibraries_jll: Library 'libgfortran' is not available for $(Sys.KERNEL)") + end; + dependencies = @static if @isdefined(libquadmath) + LazyLibrary[libgcc_s, libquadmath] + else + LazyLibrary[libgcc_s] + end +) -_libstdcxx_dependencies = LazyLibrary[libgcc_s] -const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=_libstdcxx_dependencies) +libstdcxx_path::String = "" +const libstdcxx = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libstdc++-6.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libstdc++.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libstdc++.so.6") + else + error("CompilerSupportLibraries_jll: Library 'libstdcxx' is not available for $(Sys.KERNEL)") + end; + dependencies = LazyLibrary[libgcc_s] +) -const libgomp = LazyLibrary(_libgomp_path) +libgomp_path::String = "" +const libgomp = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgomp-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgomp.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgomp.so.1") + else + error("CompilerSupportLibraries_jll: Library 'libgomp' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) -# Some installations (such as those from-source) may not have `libssp` -# So let's do a compile-time check to see if we've got it. -if @isdefined(_libssp_path) && isfile(string(_libssp_path)) - const libssp = LazyLibrary(_libssp_path) +# only define if isfile +let + if Sys.iswindows() || Sys.isapple() || libc(HostPlatform()) != "musl" + _libssp_path = if Sys.iswindows() + BundledLazyLibraryPath("libssp-0.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libssp.0.dylib") + elseif Sys.islinux() && libc(HostPlatform()) != "musl" + BundledLazyLibraryPath("libssp.so.0") + end + if isfile(string(_libssp_path)) + global libssp_path::String = "" + @eval const libssp = LazyLibrary($(_libssp_path)) + end + end end # Conform to LazyJLLWrappers API @@ -103,19 +151,17 @@ end is_available() = true function __init__() - if @isdefined _libatomic_path - global libatomic_path = string(_libatomic_path) - end - global libgcc_s_path = string(_libgcc_s_path) - global libgomp_path = string(_libgomp_path) - if @isdefined _libquadmath_path - global libquadmath_path = string(_libquadmath_path) + global libatomic_path = string(libatomic.path) + global libgcc_s_path = string(libgcc_s.path) + global libgomp_path = string(libgomp.path) + if @isdefined libquadmath_path + global libquadmath_path = string(libquadmath.path) end - if @isdefined _libssp_path - global libssp_path = string(_libssp_path) + if @isdefined libssp_path + global libssp_path = string(libssp.path) end - global libgfortran_path = string(_libgfortran_path) - global libstdcxx_path = string(_libstdcxx_path) + global libgfortran_path = string(libgfortran.path) + global libstdcxx_path = string(libstdcxx.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgcc_s_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/GMP_jll/src/GMP_jll.jl b/stdlib/GMP_jll/src/GMP_jll.jl index 3a2375b54e1d4..12a3fc21bd893 100644 --- a/stdlib/GMP_jll/src/GMP_jll.jl +++ b/stdlib/GMP_jll/src/GMP_jll.jl @@ -2,7 +2,10 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl baremodule GMP_jll -using Base, Libdl, CompilerSupportLibraries_jll +using Base, Libdl +if !Sys.isapple() + using CompilerSupportLibraries_jll +end export libgmp, libgmpxx @@ -12,44 +15,48 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libgmp_path::String = "" -libgmpxx_path::String = "" - -if Sys.iswindows() - const _libgmp_path = BundledLazyLibraryPath("libgmp-10.dll") - const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx-4.dll") -elseif Sys.isapple() - const _libgmp_path = BundledLazyLibraryPath("libgmp.10.dylib") - const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.4.dylib") -else - const _libgmp_path = BundledLazyLibraryPath("libgmp.so.10") - const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.so.4") -end -const libgmp = LazyLibrary(_libgmp_path) +libgmp_path::String = "" +const libgmp = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgmp-10.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgmp.10.dylib") + else + BundledLazyLibraryPath("libgmp.so.10") + end +) -if Sys.isfreebsd() - _libgmpxx_dependencies = LazyLibrary[libgmp, libgcc_s] -elseif Sys.isapple() - _libgmpxx_dependencies = LazyLibrary[libgmp] -else - _libgmpxx_dependencies = LazyLibrary[libgmp, libstdcxx, libgcc_s] -end +libgmpxx_path::String = "" const libgmpxx = LazyLibrary( - _libgmpxx_path, - dependencies=_libgmpxx_dependencies, + if Sys.iswindows() + BundledLazyLibraryPath("libgmpxx-4.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgmpxx.4.dylib") + else + BundledLazyLibraryPath("libgmpxx.so.4") + end, + dependencies = if Sys.isfreebsd() + LazyLibrary[libgmp, libgcc_s] + elseif Sys.isapple() + LazyLibrary[libgmp] + else + LazyLibrary[libgmp, libstdcxx, libgcc_s] + end ) function eager_mode() - CompilerSupportLibraries_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end dlopen(libgmp) dlopen(libgmpxx) end is_available() = true function __init__() - global libgmp_path = string(_libgmp_path) - global libgmpxx_path = string(_libgmpxx_path) + global libgmp_path = string(libgmp.path) + global libgmpxx_path = string(libgmpxx.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgmp_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl index 4adae99c520d4..c6e2750895c13 100644 --- a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl +++ b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl @@ -13,17 +13,17 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" + llvmlibunwind_path::String = "" +const llvmlibunwind = LazyLibrary(BundledLazyLibraryPath("libunwind")) -const _llvmlibunwind_path = BundledLazyLibraryPath("libunwind") -const llvmlibunwind = LazyLibrary(_llvmlibunwind_path) function eager_mode() dlopen(llvmlibunwind) end is_available() = @static Sys.isapple() ? true : false function __init__() - global llvmlibunwind_path = string(_llvmlibunwind_path) + global llvmlibunwind_path = string(llvmlibunwind.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(llvmlibunwind_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/LibCURL_jll/Project.toml b/stdlib/LibCURL_jll/Project.toml index 46ad6f18aaafa..5ee99517624cb 100644 --- a/stdlib/LibCURL_jll/Project.toml +++ b/stdlib/LibCURL_jll/Project.toml @@ -3,12 +3,13 @@ uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" version = "8.12.1+1" [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" -nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -17,4 +18,5 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" test = ["Test"] [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.11" diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index 92af7af03881c..43ae2dec24aab 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -5,9 +5,11 @@ baremodule LibCURL_jll using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll if !(Sys.iswindows() || Sys.isapple()) - # On Windows and macOS we use system SSL/crypto libraries using OpenSSL_jll end +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libcurl @@ -17,36 +19,47 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libcurl_path::String = "" - -_libcurl_dependencies = LazyLibrary[libz, libnghttp2, libssh2] -if !(Sys.iswindows() || Sys.isapple()) - append!(_libcurl_dependencies, [libssl, libcrypto]) -end - -if Sys.iswindows() - const _libcurl_path = BundledLazyLibraryPath("libcurl-4.dll") -elseif Sys.isapple() - const _libcurl_path = BundledLazyLibraryPath("libcurl.4.dylib") -else - const _libcurl_path = BundledLazyLibraryPath("libcurl.so.4") -end +libcurl_path::String = "" const libcurl = LazyLibrary( - _libcurl_path, - dependencies=_libcurl_dependencies, + if Sys.iswindows() + BundledLazyLibraryPath("libcurl-4.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcurl.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcurl.so.4") + else + error("LibCURL_jll: Library 'libcurl' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libz, libnghttp2, libssh2, libgcc_s] + else + LazyLibrary[libz, libnghttp2, libssh2] + end + elseif Sys.islinux() || Sys.isfreebsd() + LazyLibrary[libz, libnghttp2, libssh2, libssl, libcrypto] + elseif Sys.isapple() + LazyLibrary[libz, libnghttp2, libssh2] + end ) function eager_mode() Zlib_jll.eager_mode() nghttp2_jll.eager_mode() LibSSH2_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + @static if @isdefined OpenSSL_jll + OpenSSL_jll.eager_mode() + end dlopen(libcurl) end is_available() = true function __init__() - global libcurl_path = string(_libcurl_path) + global libcurl_path = string(libcurl.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libcurl_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/LibGit2_jll/Project.toml b/stdlib/LibGit2_jll/Project.toml index 216fe9c3c6b41..1b78c6d6b4877 100644 --- a/stdlib/LibGit2_jll/Project.toml +++ b/stdlib/LibGit2_jll/Project.toml @@ -7,9 +7,11 @@ OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" [compat] julia = "1.9" +CompilerSupportLibraries_jll = "1.3.0" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LibGit2_jll/src/LibGit2_jll.jl b/stdlib/LibGit2_jll/src/LibGit2_jll.jl index 26f679b2634e8..53663f1105220 100644 --- a/stdlib/LibGit2_jll/src/LibGit2_jll.jl +++ b/stdlib/LibGit2_jll/src/LibGit2_jll.jl @@ -5,9 +5,11 @@ baremodule LibGit2_jll using Base, Libdl, LibSSH2_jll if !(Sys.iswindows() || Sys.isapple()) - # On Windows and macOS we use system SSL/crypto libraries using OpenSSL_jll end +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libgit2 @@ -17,34 +19,45 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libgit2_path::String = "" - -if Sys.iswindows() - const _libgit2_path = BundledLazyLibraryPath("libgit2.dll") -elseif Sys.isapple() - const _libgit2_path = BundledLazyLibraryPath("libgit2.1.9.dylib") -else - const _libgit2_path = BundledLazyLibraryPath("libgit2.so.1.9") -end -if Sys.isfreebsd() - _libgit2_dependencies = LazyLibrary[libssh2, libssl, libcrypto] -else - _libgit2_dependencies = LazyLibrary[libssh2] -end -const libgit2 = LazyLibrary(_libgit2_path, dependencies=_libgit2_dependencies) +libgit2_path::String = "" +const libgit2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgit2.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgit2.1.9.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgit2.so.1.9") + else + error("LibGit2_jll: Library 'libgit2' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libssh2, libgcc_s] + else + LazyLibrary[libssh2] + end + elseif Sys.isfreebsd() || Sys.islinux() + LazyLibrary[libssh2, libssl, libcrypto] + else + LazyLibrary[libssh2] + end +) function eager_mode() LibSSH2_jll.eager_mode() - @static if !(Sys.iswindows() || Sys.isapple()) + @static if @isdefined OpenSSL_jll OpenSSL_jll.eager_mode() end + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end dlopen(libgit2) end is_available() = true function __init__() - global libgit2_path = string(_libgit2_path) + global libgit2_path = string(libgit2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgit2_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/LibSSH2_jll/Project.toml b/stdlib/LibSSH2_jll/Project.toml index f535847ae2f29..c4bf18ca39d7c 100644 --- a/stdlib/LibSSH2_jll/Project.toml +++ b/stdlib/LibSSH2_jll/Project.toml @@ -7,9 +7,11 @@ OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" [compat] julia = "1.8" +CompilerSupportLibraries_jll = "1.3.0" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl index 8c0884e8aca7b..6c273bbdecc4d 100644 --- a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl +++ b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl @@ -3,9 +3,14 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibSSH2_jll.jl baremodule LibSSH2_jll -using Base, Libdl, Zlib_jll +using Base, Libdl +if Sys.isfreebsd() || Sys.isapple() + using Zlib_jll +end +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end if !Sys.iswindows() - # On Windows we use system SSL/crypto libraries using OpenSSL_jll end @@ -17,28 +22,39 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libssh2_path::String = "" - - -if Sys.iswindows() - const _libssh2_path = BundledLazyLibraryPath("libssh2.dll") - _libssh2_dependencies = LazyLibrary[] -elseif Sys.isapple() - const _libssh2_path = BundledLazyLibraryPath("libssh2.1.dylib") - _libssh2_dependencies = LazyLibrary[libz, libcrypto] -elseif Sys.isfreebsd() - const _libssh2_path = BundledLazyLibraryPath("libssh2.so.1") - _libssh2_dependencies = LazyLibrary[libz, libcrypto] -else - const _libssh2_path = BundledLazyLibraryPath("libssh2.so.1") - _libssh2_dependencies = LazyLibrary[libcrypto] -end -const libssh2 = LazyLibrary(_libssh2_path, dependencies=_libssh2_dependencies) +libssh2_path::String = "" +const libssh2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libssh2.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libssh2.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libssh2.so.1") + else + error("LibSSH2_jll: Library 'libssh2' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end + elseif Sys.islinux() + LazyLibrary[libcrypto] + elseif Sys.isfreebsd() || Sys.isapple() + LazyLibrary[libz, libcrypto] + end +) function eager_mode() - Zlib_jll.eager_mode() - @static if !Sys.iswindows() + @static if @isdefined Zlib_jll + Zlib_jll.eager_mode() + end + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + @static if @isdefined OpenSSL_jll OpenSSL_jll.eager_mode() end dlopen(libssh2) @@ -46,7 +62,7 @@ end is_available() = true function __init__() - global libssh2_path = string(_libssh2_path) + global libssh2_path = string(libssh2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssh2_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl index 29659a18df0e0..d396540dbab46 100644 --- a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl +++ b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl @@ -4,8 +4,10 @@ baremodule LibUnwind_jll using Base, Libdl -using CompilerSupportLibraries_jll using Zlib_jll +if !Sys.isfreebsd() + using CompilerSupportLibraries_jll +end export libunwind @@ -15,27 +17,28 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libunwind_path::String = "" - -const _libunwind_path = BundledLazyLibraryPath("libunwind.so.8") - -if Sys.isfreebsd() - _libunwind_dependencies = LazyLibrary[libz] -else - _libunwind_dependencies = LazyLibrary[libgcc_s, libz] -end -const libunwind = LazyLibrary(_libunwind_path, dependencies=_libunwind_dependencies) +libunwind_path::String = "" +const libunwind = LazyLibrary( + BundledLazyLibraryPath("libunwind.so.8"), + dependencies = if Sys.isfreebsd() + LazyLibrary[libz] + else + LazyLibrary[libgcc_s, libz] + end +) function eager_mode() - CompilerSupportLibraries_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end Zlib_jll.eager_mode() dlopen(libunwind) end is_available() = @static(Sys.islinux() || Sys.isfreebsd()) ? true : false function __init__() - global libunwind_path = string(_libunwind_path) + global libunwind_path = string(libunwind.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libunwind_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/MPFR_jll/Project.toml b/stdlib/MPFR_jll/Project.toml index 9958383f4e65b..5d1524c1455b0 100644 --- a/stdlib/MPFR_jll/Project.toml +++ b/stdlib/MPFR_jll/Project.toml @@ -3,11 +3,13 @@ uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" version = "4.2.2+0" [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" GMP_jll = "781609d7-10c4-51f6-84f2-b8444358ff6d" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.6" [extras] diff --git a/stdlib/MPFR_jll/src/MPFR_jll.jl b/stdlib/MPFR_jll/src/MPFR_jll.jl index 58c79d1ab87ff..b7b379c543c7a 100644 --- a/stdlib/MPFR_jll/src/MPFR_jll.jl +++ b/stdlib/MPFR_jll/src/MPFR_jll.jl @@ -3,6 +3,9 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/MPFR_jll.jl baremodule MPFR_jll using Base, Libdl, GMP_jll +if Sys.iswindows() + using CompilerSupportLibraries_jll +end export libmpfr @@ -12,27 +15,36 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libmpfr_path::String = "" - -if Sys.iswindows() - const _libmpfr_path = BundledLazyLibraryPath("libmpfr-6.dll") -elseif Sys.isapple() - const _libmpfr_path = BundledLazyLibraryPath("libmpfr.6.dylib") -else - const _libmpfr_path = BundledLazyLibraryPath("libmpfr.so.6") -end -_libmpfr_dependencies = LazyLibrary[libgmp] - -const libmpfr = LazyLibrary(_libmpfr_path, dependencies=_libmpfr_dependencies) +libmpfr_path::String = "" +const libmpfr = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libmpfr-6.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libmpfr.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libmpfr.so.6") + else + error("MPFR_jll: Library 'libmpfr' is not available for $(Sys.KERNEL)") + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgmp, libgcc_s] + else + LazyLibrary[libgmp] + end +) function eager_mode() + GMP_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end dlopen(libmpfr) end is_available() = true function __init__() - global libmpfr_path = string(_libmpfr_path) + global libmpfr_path = string(libmpfr.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libmpfr_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 2866272e90073..54eb1ecd309ab 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -91,7 +91,7 @@ uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" version = "0.6.4" [[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" version = "8.12.1+1" @@ -101,12 +101,12 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" version = "1.9.0+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -134,7 +134,7 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" [[deps.MPFR_jll]] -deps = ["Artifacts", "GMP_jll", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "GMP_jll", "Libdl"] uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" version = "4.2.2+0" @@ -149,7 +149,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.2.25" +version = "2025.5.20" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -161,7 +161,7 @@ uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.29+0" [[deps.OpenLibm_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.5+0" @@ -275,7 +275,7 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.3.1+2" [[deps.Zstd_jll]] -deps = ["Libdl"] +deps = ["CompilerSupportLibraries_jll", "Libdl"] uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.7+1" @@ -295,7 +295,7 @@ uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.12.0+0" [[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.65.0+0" diff --git a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl index e3f4f14391369..238ad459a2dc9 100644 --- a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl +++ b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl @@ -13,7 +13,7 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libopenblas_path::String = "" + if Base.USE_BLAS64 const libsuffix = "64_" @@ -21,24 +21,30 @@ else const libsuffix = "" end -if Sys.iswindows() - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dll")) -elseif Sys.isapple() - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dylib")) -else - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".so")) -end - -_libopenblas_dependencies = LazyLibrary[libgfortran] -if Sys.isapple() - if isdefined(CompilerSupportLibraries_jll, :libquadmath) - push!(_libopenblas_dependencies, CompilerSupportLibraries_jll.libquadmath) - end - if Sys.ARCH != :aarch64 - push!(_libopenblas_dependencies, CompilerSupportLibraries_jll.libgcc_s) +libopenblas_path::String = "" +const libopenblas = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dll")) + elseif Sys.isapple() + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dylib")) + else + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".so")) + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgfortran, libgcc_s] + elseif Sys.isapple() + deps = LazyLibrary[libgfortran] + if isdefined(CompilerSupportLibraries_jll, :libquadmath) + push!(deps, CompilerSupportLibraries_jll.libquadmath) + end + if Sys.ARCH != :aarch64 + push!(deps, CompilerSupportLibraries_jll.libgcc_s) + end + deps + else + LazyLibrary[libgfortran] end -end -const libopenblas = LazyLibrary(_libopenblas_path, dependencies=_libopenblas_dependencies) +) # Conform to LazyJLLWrappers API function eager_mode() @@ -48,7 +54,7 @@ end is_available() = true function __init__() - global libopenblas_path = string(_libopenblas_path) + global libopenblas_path = string(libopenblas.path) # make sure OpenBLAS does not set CPU affinity (#1070, #9639) if !(haskey(ENV, "OPENBLAS_MAIN_FREE")) ENV["OPENBLAS_MAIN_FREE"] = "1" @@ -65,7 +71,7 @@ function __init__() ENV["OPENBLAS_DEFAULT_NUM_THREADS"] = "1" end - global libopenblas_path = string(_libopenblas_path) + global libopenblas_path = string(libopenblas.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libopenblas_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/OpenLibm_jll/Project.toml b/stdlib/OpenLibm_jll/Project.toml index 431528ee3f400..441ad36e79012 100644 --- a/stdlib/OpenLibm_jll/Project.toml +++ b/stdlib/OpenLibm_jll/Project.toml @@ -3,10 +3,12 @@ uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.5+0" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.0" [extras] diff --git a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl index 54c7e6d64b7d4..264dbbf9af8b9 100644 --- a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl +++ b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl @@ -3,6 +3,9 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenLibm_jll.jl baremodule OpenLibm_jll using Base, Libdl +if Sys.iswindows() + using CompilerSupportLibraries_jll +end export libopenlibm @@ -12,25 +15,33 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libopenlibm_path::String = "" -if Sys.iswindows() - const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.dll") -elseif Sys.isapple() - const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.4.dylib") -else - const _libopenlibm_path = BundledLazyLibraryPath("libopenlibm.so.4") -end - -const libopenlibm = LazyLibrary(_libopenlibm_path) +libopenlibm_path::String = "" +const libopenlibm = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libopenlibm.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libopenlibm.4.dylib") + else + BundledLazyLibraryPath("libopenlibm.so.4") + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) function eager_mode() dlopen(libopenlibm) + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end end is_available() = true function __init__() - global libopenlibm_path = string(_libopenlibm_path) + global libopenlibm_path = string(libopenlibm.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libopenlibm_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl index b9169b5f9c5cf..f4d11ee65b3bf 100644 --- a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl +++ b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl @@ -13,29 +13,41 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libcrypto_path::String = "" -libssl_path::String = "" -if Sys.iswindows() - if arch(HostPlatform()) == "x86_64" - const _libcrypto_path = BundledLazyLibraryPath("libcrypto-3-x64.dll") - const _libssl_path = BundledLazyLibraryPath("libssl-3-x64.dll") +libcrypto_path::String = "" +const libcrypto = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libcrypto-3-x64.dll") + else + BundledLazyLibraryPath("libcrypto-3.dll") + end + elseif Sys.isapple() + BundledLazyLibraryPath("libcrypto.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcrypto.so.3") else - const _libcrypto_path = BundledLazyLibraryPath("libcrypto-3.dll") - const _libssl_path = BundledLazyLibraryPath("libssl-3.dll") + error("OpenSSL_jll: Library 'libcrypto' is not available for $(Sys.KERNEL)") end -elseif Sys.isapple() - const _libcrypto_path = BundledLazyLibraryPath("libcrypto.3.dylib") - const _libssl_path = BundledLazyLibraryPath("libssl.3.dylib") -else - const _libcrypto_path = BundledLazyLibraryPath("libcrypto.so.3") - const _libssl_path = BundledLazyLibraryPath("libssl.so.3") -end +) -const libcrypto = LazyLibrary(_libcrypto_path) - -_libssl_dependencies = LazyLibrary[libcrypto] -const libssl = LazyLibrary(_libssl_path, dependencies=_libssl_dependencies) +libssl_path::String = "" +const libssl = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libssl-3-x64.dll") + else + BundledLazyLibraryPath("libssl-3.dll") + end + elseif Sys.isapple() + BundledLazyLibraryPath("libssl.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libssl.so.3") + else + error("OpenSSL_jll: Library 'libssl' is not available for $(Sys.KERNEL)") + end; + dependencies = LazyLibrary[libcrypto] +) function eager_mode() dlopen(libcrypto) @@ -44,8 +56,8 @@ end is_available() = true function __init__() - global libcrypto_path = string(_libcrypto_path) - global libssl_path = string(_libssl_path) + global libcrypto_path = string(libcrypto.path) + global libssl_path = string(libssl.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssl_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/PCRE2_jll/src/PCRE2_jll.jl b/stdlib/PCRE2_jll/src/PCRE2_jll.jl index 9af7fb3863a23..c6e32bf3e6672 100644 --- a/stdlib/PCRE2_jll/src/PCRE2_jll.jl +++ b/stdlib/PCRE2_jll/src/PCRE2_jll.jl @@ -12,17 +12,19 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libpcre2_8_path::String = "" - -if Sys.iswindows() - const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8-0.dll") -elseif Sys.isapple() - const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8.0.dylib") -else - const _libpcre2_8_path = BundledLazyLibraryPath("libpcre2-8.so.0") -end -const libpcre2_8 = LazyLibrary(_libpcre2_8_path) +libpcre2_8_path::String = "" +const libpcre2_8 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libpcre2-8.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libpcre2-8.0.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libpcre2-8.so.0") + else + error("PCRE2_jll: Library 'libpcre2_8' is not available for $(Sys.KERNEL)") + end +) function eager_mode() dlopen(libpcre2_8) @@ -30,7 +32,7 @@ end is_available() = true function __init__() - global libpcre2_8_path = string(_libpcre2_8_path) + global libpcre2_8_path = string(libpcre2_8.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libpcre2_8_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index 6fb4f4c53bfa6..1dcb2d24e4bc7 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -2,7 +2,11 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/SuiteSparse_jll.jl baremodule SuiteSparse_jll -using Base, Libdl, libblastrampoline_jll, CompilerSupportLibraries_jll +using Base, Libdl +using libblastrampoline_jll +if !(Sys.isfreebsd() || Sys.isapple()) + using CompilerSupportLibraries_jll +end export libamd, libbtf, libcamd, libccolamd, libcholmod, libcolamd, libklu, libldl, librbio, libspqr, libsuitesparseconfig, libumfpack @@ -13,99 +17,218 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libamd_path::String = "" -libbtf_path::String = "" -libcamd_path::String = "" -libccolamd_path::String = "" -libcholmod_path::String = "" -libcolamd_path::String = "" -libklu_path::String = "" -libldl_path::String = "" -librbio_path::String = "" -libspqr_path::String = "" + libsuitesparseconfig_path::String = "" -libumfpack_path::String = "" +const libsuitesparseconfig = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libsuitesparseconfig.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libsuitesparseconfig.7.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libsuitesparseconfig.so.7") + else + error("SuiteSparse_jll: Library 'libsuitesparseconfig' is not available for $(Sys.KERNEL)") + end +) -if Sys.iswindows() - const _libamd_path = BundledLazyLibraryPath("libamd.dll") - const _libbtf_path = BundledLazyLibraryPath("libbtf.dll") - const _libcamd_path = BundledLazyLibraryPath("libcamd.dll") - const _libccolamd_path = BundledLazyLibraryPath("libccolamd.dll") - const _libcholmod_path = BundledLazyLibraryPath("libcholmod.dll") - const _libcolamd_path = BundledLazyLibraryPath("libcolamd.dll") - const _libklu_path = BundledLazyLibraryPath("libklu.dll") - const _libldl_path = BundledLazyLibraryPath("libldl.dll") - const _librbio_path = BundledLazyLibraryPath("librbio.dll") - const _libspqr_path = BundledLazyLibraryPath("libspqr.dll") - const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.dll") - const _libumfpack_path = BundledLazyLibraryPath("libumfpack.dll") -elseif Sys.isapple() - const _libamd_path = BundledLazyLibraryPath("libamd.3.dylib") - const _libbtf_path = BundledLazyLibraryPath("libbtf.2.dylib") - const _libcamd_path = BundledLazyLibraryPath("libcamd.3.dylib") - const _libccolamd_path = BundledLazyLibraryPath("libccolamd.3.dylib") - const _libcholmod_path = BundledLazyLibraryPath("libcholmod.5.dylib") - const _libcolamd_path = BundledLazyLibraryPath("libcolamd.3.dylib") - const _libklu_path = BundledLazyLibraryPath("libklu.2.dylib") - const _libldl_path = BundledLazyLibraryPath("libldl.3.dylib") - const _librbio_path = BundledLazyLibraryPath("librbio.4.dylib") - const _libspqr_path = BundledLazyLibraryPath("libspqr.4.dylib") - const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.7.dylib") - const _libumfpack_path = BundledLazyLibraryPath("libumfpack.6.dylib") -else - const _libamd_path = BundledLazyLibraryPath("libamd.so.3") - const _libbtf_path = BundledLazyLibraryPath("libbtf.so.2") - const _libcamd_path = BundledLazyLibraryPath("libcamd.so.3") - const _libccolamd_path = BundledLazyLibraryPath("libccolamd.so.3") - const _libcholmod_path = BundledLazyLibraryPath("libcholmod.so.5") - const _libcolamd_path = BundledLazyLibraryPath("libcolamd.so.3") - const _libklu_path = BundledLazyLibraryPath("libklu.so.2") - const _libldl_path = BundledLazyLibraryPath("libldl.so.3") - const _librbio_path = BundledLazyLibraryPath("librbio.so.4") - const _libspqr_path = BundledLazyLibraryPath("libspqr.so.4") - const _libsuitesparseconfig_path = BundledLazyLibraryPath("libsuitesparseconfig.so.7") - const _libumfpack_path = BundledLazyLibraryPath("libumfpack.so.6") -end +libldl_path::String = "" +const libldl = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libldl.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libldl.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libldl.so.3") + else + error("SuiteSparse_jll: Library 'libldl' is not available for $(Sys.KERNEL)") + end +) -const libsuitesparseconfig = LazyLibrary(_libsuitesparseconfig_path) -const libldl = LazyLibrary(_libldl_path) -const libbtf = LazyLibrary(_libbtf_path) +libbtf_path::String = "" +const libbtf = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libbtf.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libbtf.2.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libbtf.so.2") + else + error("SuiteSparse_jll: Library 'libbtf' is not available for $(Sys.KERNEL)") + end +) -_libcolamd_dependencies = LazyLibrary[libsuitesparseconfig] -const libcolamd = LazyLibrary(_libcolamd_path; dependencies=_libcolamd_dependencies) +libcolamd_path::String = "" +const libcolamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcolamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcolamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcolamd.so.3") + else + error("SuiteSparse_jll: Library 'libcolamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) -_libamd_dependencies = LazyLibrary[libsuitesparseconfig] -const libamd = LazyLibrary(_libamd_path; dependencies=_libamd_dependencies) +libamd_path::String = "" +const libamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libamd.so.3") + else + error("SuiteSparse_jll: Library 'libamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) -_libcamd_dependencies = LazyLibrary[libsuitesparseconfig] -const libcamd = LazyLibrary(_libcamd_path; dependencies=_libcamd_dependencies) +libcamd_path::String = "" +const libcamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcamd.so.3") + else + error("SuiteSparse_jll: Library 'libcamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) -_libccolamd_dependencies = LazyLibrary[libsuitesparseconfig] -const libccolamd = LazyLibrary(_libccolamd_path; dependencies=_libccolamd_dependencies) +libccolamd_path::String = "" +const libccolamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libccolamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libccolamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libccolamd.so.3") + else + error("SuiteSparse_jll: Library 'libccolamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) -_librbio_dependencies = LazyLibrary[libsuitesparseconfig] -const librbio = LazyLibrary(_librbio_path; dependencies=_librbio_dependencies) +librbio_path::String = "" +const librbio = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("librbio.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("librbio.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("librbio.so.4") + else + error("SuiteSparse_jll: Library 'librbio' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) -_libcholmod_dependencies = LazyLibrary[ - libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline - ] -const libcholmod = LazyLibrary(_libcholmod_path; dependencies=_libcholmod_dependencies) +libcholmod_path::String = "" +const libcholmod = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcholmod.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcholmod.5.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcholmod.so.5") + else + error("SuiteSparse_jll: Library 'libcholmod' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[ + libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline, libgcc_s + ] + else + LazyLibrary[ + libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline + ] + end +) -_libklu_dependencies = LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf] -const libklu = LazyLibrary(_libklu_path; dependencies=_libklu_dependencies) +libklu_path::String = "" +const libklu = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libklu.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libklu.2.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libklu.so.2") + else + error("SuiteSparse_jll: Library 'libklu' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf, libgcc_s] + else + LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf] + end +) -if Sys.isfreebsd() || Sys.isapple() - _libspqr_dependencies = LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline] -else - _libspqr_dependencies = LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libstdcxx, libgcc_s] -end -const libspqr = LazyLibrary(_libspqr_path; dependencies=_libspqr_dependencies) +libspqr_path::String = "" +const libspqr = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libspqr.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libspqr.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libspqr.so.4") + else + error("SuiteSparse_jll: Library 'libspqr' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libgcc_s] + elseif Sys.isfreebsd() || Sys.isapple() + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline] + else + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libstdcxx, libgcc_s] + end +) -_libumfpack_dependencies = LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline] -const libumfpack = LazyLibrary(_libumfpack_path; dependencies=_libumfpack_dependencies) +libumfpack_path::String = "" +const libumfpack = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libumfpack.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libumfpack.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libumfpack.so.6") + else + error("SuiteSparse_jll: Library 'libumfpack' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline, libgcc_s] + else + LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline] + end +) function eager_mode() - CompilerSupportLibraries_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end libblastrampoline_jll.eager_mode() dlopen(libamd) @@ -125,23 +248,23 @@ is_available() = true function __init__() # BSD-3-Clause - global libamd_path = string(_libamd_path) - global libcamd_path = string(_libcamd_path) - global libccolamd_path = string(_libccolamd_path) - global libcolamd_path = string(_libcolamd_path) - global libsuitesparseconfig_path = string(_libsuitesparseconfig_path) + global libamd_path = string(libamd.path) + global libcamd_path = string(libcamd.path) + global libccolamd_path = string(libccolamd.path) + global libcolamd_path = string(libcolamd.path) + global libsuitesparseconfig_path = string(libsuitesparseconfig.path) # LGPL-2.1+ - global libbtf_path = string(_libbtf_path) - global libklu_path = string(_libklu_path) - global libldl_path = string(_libldl_path) + global libbtf_path = string(libbtf.path) + global libklu_path = string(libklu.path) + global libldl_path = string(libldl.path) # GPL-2.0+ if Base.USE_GPL_LIBS - global libcholmod_path = string(_libcholmod_path) - global librbio_path = string(_librbio_path) - global libspqr_path = string(_libspqr_path) - global libumfpack_path = string(_libumfpack_path) + global libcholmod_path = string(libcholmod.path) + global librbio_path = string(librbio.path) + global libspqr_path = string(libspqr.path) + global libumfpack_path = string(libumfpack.path) end global artifact_dir = dirname(Sys.BINDIR) end diff --git a/stdlib/Zlib_jll/src/Zlib_jll.jl b/stdlib/Zlib_jll/src/Zlib_jll.jl index 649cb93f862da..a52168bf244b0 100644 --- a/stdlib/Zlib_jll/src/Zlib_jll.jl +++ b/stdlib/Zlib_jll/src/Zlib_jll.jl @@ -12,16 +12,19 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libz_path::String = "" -if Sys.iswindows() - const _libz_path = BundledLazyLibraryPath("libz.dll") -elseif Sys.isapple() - const _libz_path = BundledLazyLibraryPath("libz.1.dylib") -else - const _libz_path = BundledLazyLibraryPath("libz.so.1") -end -const libz = LazyLibrary(_libz_path) +libz_path::String = "" +const libz = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libz.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libz.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libz.so.1") + else + error("Zlib_jll: Library 'libz' is not available for $(Sys.KERNEL)") + end +) function eager_mode() dlopen(libz) @@ -29,7 +32,7 @@ end is_available() = true function __init__() - global libz_path = string(_libz_path) + global libz_path = string(libz.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libz_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/Zstd_jll/Project.toml b/stdlib/Zstd_jll/Project.toml index 467516843390a..1f8172cdc75bc 100644 --- a/stdlib/Zstd_jll/Project.toml +++ b/stdlib/Zstd_jll/Project.toml @@ -3,9 +3,11 @@ uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.7+1" [deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.6" [extras] diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl index 004a884bed1ff..ce2dda69d5ee3 100644 --- a/stdlib/Zstd_jll/src/Zstd_jll.jl +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -4,6 +4,9 @@ # baremodule Zstd_jll using Base, Libdl +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libzstd, zstd, zstdmt @@ -13,17 +16,24 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libzstd_path::String = "" - -if Sys.iswindows() - const _libzstd_path = BundledLazyLibraryPath("libzstd-1.dll") -elseif Sys.isapple() - const _libzstd_path = BundledLazyLibraryPath("libzstd.1.dylib") -else - const _libzstd_path = BundledLazyLibraryPath("libzstd.so.1") -end -const libzstd = LazyLibrary(_libzstd_path) +libzstd_path::String = "" +const libzstd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libzstd-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libzstd.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libzstd.so.1") + else + error("Zstd_jll: Library 'libzstd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) if Sys.iswindows() const zstd_exe = "zstd.exe" @@ -74,13 +84,16 @@ zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_ex # Function to eagerly dlopen our library and thus resolve all dependencies function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end dlopen(libzstd) end is_available() = true function __init__() - global libzstd_path = string(_libzstd_path) + global libzstd_path = string(libzstd.path) global artifact_dir = dirname(Sys.BINDIR) end diff --git a/stdlib/dSFMT_jll/src/dSFMT_jll.jl b/stdlib/dSFMT_jll/src/dSFMT_jll.jl index b7e79c9d587ea..93fc26dc87a53 100644 --- a/stdlib/dSFMT_jll/src/dSFMT_jll.jl +++ b/stdlib/dSFMT_jll/src/dSFMT_jll.jl @@ -13,17 +13,17 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libdSFMT_path::String = "" - -if Sys.iswindows() - const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.dll") -elseif Sys.isapple() - const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.dylib") -else - const _libdSFMT_path = BundledLazyLibraryPath("libdSFMT.so") -end -const libdSFMT = LazyLibrary(_libdSFMT_path) +libdSFMT_path::String = "" +const libdSFMT = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libdSFMT.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libdSFMT.dylib") + else + BundledLazyLibraryPath("libdSFMT.so") + end +) function eager_mode() dlopen(libdSFMT) @@ -31,7 +31,7 @@ end is_available() = true function __init__() - global libdSFMT_path = string(_libdSFMT_path) + global libdSFMT_path = string(libdSFMT.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libdSFMT_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/libLLVM_jll/src/libLLVM_jll.jl b/stdlib/libLLVM_jll/src/libLLVM_jll.jl index c1d89d2ce886f..2edff186b13a1 100644 --- a/stdlib/libLLVM_jll/src/libLLVM_jll.jl +++ b/stdlib/libLLVM_jll/src/libLLVM_jll.jl @@ -3,7 +3,11 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/libLLVM_jll.jl baremodule libLLVM_jll -using Base, Libdl, Zlib_jll, Zstd_jll, CompilerSupportLibraries_jll +using Base, Libdl, Zlib_jll, Zstd_jll + +if !Sys.isapple() + using CompilerSupportLibraries_jll +end export libLLVM @@ -13,31 +17,29 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libLLVM_path::String = "" - -if Sys.iswindows() - const _libLLVM_path = BundledLazyLibraryPath("$(Base.libllvm_name).dll") -elseif Sys.isapple() - const _libLLVM_path = BundledLazyLibraryPath("libLLVM.dylib") -else - const _libLLVM_path = BundledLazyLibraryPath("$(Base.libllvm_name).so") -end - -if Sys.isapple() - _libLLVM_dependencies = LazyLibrary[libz, libzstd] -elseif Sys.isfreebsd() - _libLLVM_dependencies = LazyLibrary[libz, libzstd, libgcc_s] -else - _libLLVM_dependencies = LazyLibrary[libz, libzstd, libstdcxx, libgcc_s] -end +libLLVM_path::String = "" const libLLVM = LazyLibrary( - _libLLVM_path, - dependencies=_libLLVM_dependencies, + if Sys.iswindows() + BundledLazyLibraryPath("$(Base.libllvm_name).dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libLLVM.dylib") + else + BundledLazyLibraryPath("$(Base.libllvm_name).so") + end, + dependencies = if Sys.isapple() + LazyLibrary[libz, libzstd] + elseif Sys.isfreebsd() + LazyLibrary[libz, libzstd, libgcc_s] + else + LazyLibrary[libz, libzstd, libstdcxx, libgcc_s] + end ) function eager_mode() - CompilerSupportLibraries_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end Zlib_jll.eager_mode() # Zstd_jll.eager_mode() # Not lazy yet dlopen(libLLVM) @@ -45,7 +47,7 @@ end is_available() = true function __init__() - global libLLVM_path = string(_libLLVM_path) + global libLLVM_path = string(libLLVM.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libLLVM_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl index 01d222b8a472d..a75e2e30db2fd 100644 --- a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl +++ b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl @@ -13,8 +13,6 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libblastrampoline_path::String = "" - # Because LBT needs to have a weak-dependence on OpenBLAS (or any other BLAS) # we must manually construct a list of which modules and libraries we're going @@ -33,16 +31,19 @@ function add_dependency!(mod::Module, lib::LazyLibrary, on_load_callback::Functi push!(on_load_callbacks, on_load_callback) end -# NOTE: keep in sync with `Base.libblas_name` and `Base.liblapack_name`. -const _libblastrampoline_path = if Sys.iswindows() - BundledLazyLibraryPath("libblastrampoline-5.dll") -elseif Sys.isapple() - BundledLazyLibraryPath("libblastrampoline.5.dylib") -else - BundledLazyLibraryPath("libblastrampoline.so.5") -end -const libblastrampoline = LazyLibrary(_libblastrampoline_path, dependencies=[], - on_load_callback=libblastrampoline_on_load_callback) +libblastrampoline_path::String = "" +const libblastrampoline = LazyLibrary( + # NOTE: keep in sync with `Base.libblas_name` and `Base.liblapack_name`. + if Sys.iswindows() + BundledLazyLibraryPath("libblastrampoline-5.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libblastrampoline.5.dylib") + else + BundledLazyLibraryPath("libblastrampoline.so.5") + end, + dependencies = LazyLibrary[], + on_load_callback = libblastrampoline_on_load_callback +) function eager_mode() for mod in eager_mode_modules @@ -53,7 +54,7 @@ end is_available() = true function __init__() - global libblastrampoline_path = string(_libblastrampoline_path) + global libblastrampoline_path = string(libblastrampoline.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libblastrampoline_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/nghttp2_jll/Project.toml b/stdlib/nghttp2_jll/Project.toml index 4520427d7ca76..ea748ec6b6c81 100644 --- a/stdlib/nghttp2_jll/Project.toml +++ b/stdlib/nghttp2_jll/Project.toml @@ -3,10 +3,12 @@ uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.65.0+0" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.11" [extras] diff --git a/stdlib/nghttp2_jll/src/nghttp2_jll.jl b/stdlib/nghttp2_jll/src/nghttp2_jll.jl index 3257c9602822b..e0a1559a85628 100644 --- a/stdlib/nghttp2_jll/src/nghttp2_jll.jl +++ b/stdlib/nghttp2_jll/src/nghttp2_jll.jl @@ -3,6 +3,9 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/nghttp2_jll.jl baremodule nghttp2_jll using Base, Libdl +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libnghttp2 @@ -12,25 +15,33 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libnghttp2_path::String = "" -if Sys.iswindows() - const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2-14.dll") -elseif Sys.isapple() - const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2.14.dylib") -else - const _libnghttp2_path = BundledLazyLibraryPath("libnghttp2.so.14") -end - -const libnghttp2 = LazyLibrary(_libnghttp2_path) +libnghttp2_path::String = "" +const libnghttp2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libnghttp2-14.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libnghttp2.14.dylib") + else + BundledLazyLibraryPath("libnghttp2.so.14") + end, + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end dlopen(libnghttp2) end is_available() = true function __init__() - global libnghttp2_path = string(_libnghttp2_path) + global libnghttp2_path = string(libnghttp2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libnghttp2_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl index 46832aa404df1..9d0f6d7c05524 100644 --- a/test/stdlib_dependencies.jl +++ b/test/stdlib_dependencies.jl @@ -1,148 +1,198 @@ -using Pkg, Test, Libdl +using Libdl +using Pkg +using Test +prev_env = Base.active_project() +Pkg.activate(temp=true) +Pkg.add(Pkg.PackageSpec(name="ObjectFile", uuid="d8793406-e978-5875-9003-1fc021f44a92", version="0.4")) +using ObjectFile +try -# Remove `.X.dylib` or just `.dylib` -function strip_soversion_macos(lib) - m = match(r"^(.*?)(\.\d+)*\.dylib$", lib) - if m !== nothing - return m.captures[1] - end - return lib -end + strip_soversion(lib::AbstractString) = Base.BinaryPlatforms.parse_dl_name_version(lib)[1] -# Remove `.so.X` or just `.so` -function strip_soversion_linux(lib) - m = match(r"^(.*?)\.so(\.\d+)*$", lib) - if m !== nothing - return m.captures[1] - end - return lib -end + function get_deps_objectfile_macos(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = only(obj_handles) # If more than one its unclear what to do + raw_libs = String[] -# Remove `-X.dll` or just `.dll` -function strip_soversion_windows(lib) - m = match(r"^(.*?)(-\d+)*\.dll$", lib) - if m !== nothing - return m.captures[1] + # For Mach-O files, get load commands + if isa(obj, ObjectFile.MachOHandle) + for lc in ObjectFile.MachOLoadCmds(obj) + if lc isa ObjectFile.MachO.MachOLoadDylibCmd + # Extract the library name from the load command + lib_name = ObjectFile.dylib_name(lc) + if lib_name !== nothing + # Remove @rpath/ prefix if present + lib_name = last(split(lib_name, "@rpath/")) + # Get basename + lib_name = basename(lib_name) + isempty(splitext(lib_name)[2]) && continue # skip frameworks + push!(raw_libs, lib_name) + end + end + end + end + libs = strip_soversion.(raw_libs) + # Get rid of any self-referential links + self_lib = strip_soversion(basename(lib_path)) + libs = filter(!=(self_lib), libs) + return libs + end end - return lib -end + function get_deps_objectfile_linux_freebsd(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = first(obj_handles) # Take the first handle from the vector + raw_libs = String[] -function get_deps_otool(lib_path::String) - libs = split(readchomp(`otool -L $(lib_path)`), "\n")[2:end] - # Get rid of `(compatibility version x.y.x)` at the end - libs = first.(split.(libs, (" (compatibility",))) - # Get rid of any `@rpath/` stuff at the beginning - libs = last.(split.(libs, ("@rpath/",))) - - # If there are any absolute paths left, get rid of them here - libs = basename.(strip.(libs)) - - # Now that we've got the basenames of each library, remove `.x.dylib` if it exists: - libs = strip_soversion_macos.(libs) + # For ELF files, get dynamic dependencies + if isa(obj, ObjectFile.ELFHandle) + # Get all dynamic entries + dyn_entries = ObjectFile.ELFDynEntries(obj) + for entry in dyn_entries + # Check if the entry is of type DT_NEEDED + if ObjectFile.dyn_entry_type(entry) == ObjectFile.ELF.DT_NEEDED + lib_name = ObjectFile.strtab_lookup(entry) + if lib_name !== nothing && !isempty(lib_name) + push!(raw_libs, basename(lib_name)) + end + end + end + end - # Get rid of any self-referential links - self_lib = strip_soversion_macos(basename(lib_path)) - libs = filter(!=(self_lib), libs) - return libs -end + libs = strip_soversion.(raw_libs) + # Self-reference is typically not listed in NEEDED for ELF, so no explicit filter here. + return libs + end + end -function is_system_lib_macos(lib) - system_libs = [ - "libSystem.B", - "libc++", # While we package libstdc++, we do NOT package libc++. - "libiconv", # some things (like git) link against system libiconv - - # macOS frameworks used by things like LibCurl - "CoreFoundation", - "CoreServices", - "Security", - "SystemConfiguration" - ] - return lib ∈ system_libs -end + function get_deps_objectfile_windows(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = first(obj_handles) # Take the first handle from the vector + raw_libs_set = Set{String}() # Use Set for uniqueness of DLL names -function is_system_lib_linux(lib) - system_libs = [ - "libdl", - "libc", - "libm", - "librt", - "libpthread", - "ld-linux-x86-64", - "ld-linux-x86", - "ld-linux-aarch64", - "ld-linux-armhf", - "ld-linux-i386", - ] - return lib ∈ system_libs -end + # For COFF/PE files, get import table + if isa(obj, ObjectFile.COFFHandle) + # Get dynamic links + dls = ObjectFile.DynamicLinks(obj) + for link in dls + lib_name = ObjectFile.path(link) + if lib_name !== nothing && !isempty(lib_name) + # COFF library names are case-insensitive + push!(raw_libs_set, lowercase(lib_name)) + end + end + end -function is_system_lib_freebsd(lib) - system_libs = [ - "libdl", - "libc", - "libm", - "libthr", # primary threading library - "libpthread", # alias kept for compatibility - "librt", - "libutil", - "libexecinfo", - "libc++", - "libcxxrt", - ] - return lib ∈ system_libs -end + libs = strip_soversion.(collect(raw_libs_set)) + # Get rid of any self-referential links + self_lib = strip_soversion(lowercase(basename(lib_path))) + libs = filter(!=(self_lib), libs) + return libs + end + end -function get_deps_readelf(lib_path::String) - # Split into lines - libs = split(readchomp(`readelf -d $(lib_path)`), "\n") + function get_deps_objectfile(lib_path::String) + if Sys.isapple() + return get_deps_objectfile_macos(lib_path) + elseif Sys.islinux() || Sys.isfreebsd() + return get_deps_objectfile_linux_freebsd(lib_path) + elseif Sys.iswindows() + return get_deps_objectfile_windows(lib_path) + else + error("Unsupported platform for ObjectFile.jl dependency extraction") + end + end - # Only keep `(NEEDED)` lines - needed_str = Sys.isfreebsd() ? "NEEDED" : "(NEEDED)" - libs = filter(contains(needed_str), libs) + function is_system_lib_macos(lib) + system_libs = [ + "libSystem.B", + "libc++", # While we package libstdc++, we do NOT package libc++. + "libiconv", # some things (like git) link against system libiconv - # Grab the SONAME from "Shared library: [$SONAME]" - libs = map(libs) do lib - m = match(r"Shared library: \[(.*)\]$", lib) - if m !== nothing - return basename(m.captures[1]) - end - return "" + # macOS frameworks used by things like LibCurl + "CoreFoundation", + "CoreServices", + "Security", + "SystemConfiguration" + ] + return lib ∈ system_libs end - libs = filter(!isempty, strip.(libs)) - # Get rid of soversions in the filenames - libs = strip_soversion_linux.(libs) - return libs -end + function is_system_lib_linux(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "librt", + "libpthread", + "ld-linux", + "ld-linux-x86-64", + "ld-linux-x86", + "ld-linux-aarch64", + "ld-linux-armhf", + "ld-linux-i386", + ] + return lib ∈ system_libs + end + function is_system_lib_freebsd(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "libthr", # primary threading library + "libpthread", # alias kept for compatibility + "librt", + "libutil", + "libexecinfo", + "libc++", + "libcxxrt", + ] + return lib ∈ system_libs + end -skip = false -# On linux, we need `readelf` available, otherwise we refuse to attempt this -if Sys.islinux() || Sys.isfreebsd() - if Sys.which("readelf") === nothing - @debug("Silently skipping stdlib_dependencies.jl as `readelf` not available.") - skip = true + function is_system_lib_windows(lib) + system_libs = [ + "kernel32", + "user32", + "gdi32", + "advapi32", + "ole32", + "oleaut32", + "shell32", + "ws2_32", + "comdlg32", + "shlwapi", + "rpcrt4", + "msvcrt", + "comctl32", + "ucrtbase", + "vcruntime140", + "msvcp140", + "libwinpthread", + "ntdll", + "crypt32", + "bcrypt", + "winhttp", + "secur32", + ] + return any(syslib -> lowercase(lib) == syslib, system_libs) end - get_deps = get_deps_readelf - strip_soversion = strip_soversion_linux - is_system_lib = Sys.islinux() ? is_system_lib_linux : is_system_lib_freebsd -elseif Sys.isapple() - # On macOS, we need `otool` available - if Sys.which("otool") === nothing - @debug("Silently skipping stdlib_dependencies.jl as `otool` not available.") - skip = true + + # Set up platform-specific functions + if Sys.islinux() || Sys.isfreebsd() + is_system_lib = Sys.islinux() ? is_system_lib_linux : is_system_lib_freebsd + elseif Sys.isapple() + is_system_lib = is_system_lib_macos + elseif Sys.iswindows() + is_system_lib = is_system_lib_windows + else + error("Unsupported platform for `stdlib_dependencies.jl`. Only Linux, FreeBSD, macOS, and Windows are supported.") end - get_deps = get_deps_otool - strip_soversion = strip_soversion_macos - is_system_lib = is_system_lib_macos -else - @debug("Don't know how to run `stdlib_dependencies.jl` on this platform") - skip = true -end -if !skip # Iterate over all JLL stdlibs, check their lazy libraries to ensure # that they list all valid library dependencies, avoiding a situation # where the JLL wrapper code has fallen out of sync with the binaries @@ -164,7 +214,7 @@ if !skip if isa(prop, Libdl.LazyLibrary) lib_path = dlpath(prop) lazy_lib_deps = strip_soversion.(basename.(dlpath.(prop.dependencies))) - real_lib_deps = filter(!is_system_lib, get_deps(lib_path)) + real_lib_deps = filter(!is_system_lib, get_deps_objectfile(lib_path)) # See if there are missing dependencies in the lazy library deps missing_deps = setdiff(real_lib_deps, lazy_lib_deps) @@ -175,8 +225,8 @@ if !skip # This is a manually-managed special case if stdlib_name == "libblastrampoline_jll" && - prop_name == :libblastrampoline && - extraneous_deps == ["libopenblas64_"] + prop_name == :libblastrampoline && + extraneous_deps in (["libopenblas64_"], ["libopenblas"]) deps_mismatch = false end @@ -185,15 +235,27 @@ if !skip # Print out the deps mismatch if we find one if deps_mismatch @warn("Dependency mismatch", - jll=stdlib_name, - library=string(prop_name), - missing_deps=join(missing_deps, ", "), - extraneous_deps=join(extraneous_deps, ", "), - actual_deps=join(real_lib_deps, ", "), + jll = stdlib_name, + library = string(prop_name), + missing_deps = join(missing_deps, ", "), + extraneous_deps = join(extraneous_deps, ", "), + actual_deps = join(real_lib_deps, ", "), ) end end end + if isdefined(m, :eager_mode) + # If the JLL has an eager_mode function, call it + Base.invokelatest(getproperty(m, :eager_mode)) + end end end + +finally + if prev_env !== nothing + Pkg.activate(prev_env) + else + # If no previous environment, activate the default one + Pkg.activate() + end end From c759aa933ed041e678d370dede7030c8893bbd10 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 8 Jun 2025 16:25:59 -0700 Subject: [PATCH 390/662] Adjust applescript workaround It turns out that there are two path types in applescript, and I had mixed two of them in my previous patch. Annoyingly, things seemed to work when editing locally, unsure why. --- contrib/mac/app/startup.applescript | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/mac/app/startup.applescript b/contrib/mac/app/startup.applescript index 45ccbbb977d25..d7b46cec1a89d 100644 --- a/contrib/mac/app/startup.applescript +++ b/contrib/mac/app/startup.applescript @@ -1,4 +1,3 @@ set RootPath to (path to me) set JuliaPath to POSIX path of ((RootPath as text) & "Contents:Resources:julia:bin:julia") -set JuliaFile to POSIX file JuliaPath -do shell script "open -a Terminal '" & JuliaFile & "'" +do shell script "open -a Terminal '" & JuliaPath & "'" From 1c26f43717b613d639c2171a3f883a11bcb12196 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 9 Jun 2025 16:30:46 -0400 Subject: [PATCH 391/662] better handling of missing backedges (#58636) Manage a single dictionary (keyed by TypeName) instead of scattering this info into each TypeName scattered across the system. This makes it much easier to scan the whole table when required and to split it up better, so that all kwcalls and all constructors don't end up stuck into just one table. While not enormous (or even the largest) just using the REPL and Pkg, they are clearly larger than intended for a linear scan: ``` julia> length(Type.body.name.backedges) 1024 julia> length(typeof(Core.kwcall).name.backedges) 196 julia> length(typeof(convert).name.backedges) 1510 ``` --- src/datatype.c | 2 +- src/gf.c | 331 +++++++++++++++++++++++++++---------------- src/jltypes.c | 30 ++-- src/julia.h | 2 +- src/julia_internal.h | 2 - src/method.c | 38 ----- src/staticdata.c | 44 ++++-- test/misc.jl | 4 +- 8 files changed, 257 insertions(+), 196 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index eb25907647157..2df2bdb2aaa71 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -62,6 +62,7 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo mt->cache = mc; mt->name = name; mt->module = module; + mt->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; JL_GC_POP(); return mt; } @@ -88,7 +89,6 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; - tn->backedges = NULL; tn->max_methods = 0; jl_atomic_store_relaxed(&tn->max_args, 0); jl_atomic_store_relaxed(&tn->cache_entry_count, 0); diff --git a/src/gf.c b/src/gf.c index 8205cf70b99c3..57dc2760eecac 100644 --- a/src/gf.c +++ b/src/gf.c @@ -777,55 +777,117 @@ JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, return ret; } -static int foreach_typename_in_module( - jl_module_t *m, - int (*visit)(jl_typename_t *tn, void *env), - void *env) +enum top_typename_facts { + EXACTLY_ANY = 1 << 0, + HAVE_TYPE = 1 << 1, + EXACTLY_TYPE = 1 << 2, + HAVE_FUNCTION = 1 << 3, + EXACTLY_FUNCTION = 1 << 4, + HAVE_KWCALL = 1 << 5, + EXACTLY_KWCALL = 1 << 6, + SHORT_TUPLE = 1 << 7, +}; + +static void foreach_top_nth_typename(void (*f)(jl_typename_t*, int, void*), jl_value_t *a JL_PROPAGATES_ROOT, int n, unsigned *facts, void *env) { - jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); - for (size_t i = 0; i < jl_svec_len(table); i++) { - jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); - if ((void*)b == jl_nothing) - break; - jl_sym_t *name = b->globalref->name; - jl_value_t *v = jl_get_latest_binding_value_if_const(b); - if (v) { - jl_value_t *uw = jl_unwrap_unionall(v); - if (jl_is_datatype(uw)) { - jl_typename_t *tn = ((jl_datatype_t*)uw)->name; - if (tn->module == m && tn->name == name && tn->wrapper == v) { - // this is the original/primary binding for the type (name/wrapper) - if (!visit(((jl_datatype_t*)uw)->name, env)) - return 0; - } + if (jl_is_datatype(a)) { + if (n <= 0) { + jl_datatype_t *dt = ((jl_datatype_t*)a); + if (dt->name == jl_type_typename) { // key Type{T} on T instead of Type + *facts |= HAVE_TYPE; + foreach_top_nth_typename(f, jl_tparam0(a), -1, facts, env); } - else if (jl_is_module(v)) { - jl_module_t *child = (jl_module_t*)v; - if (child != m && child->parent == m && child->name == name) { - // this is the original/primary binding for the submodule - if (!foreach_typename_in_module(child, visit, env)) - return 0; + else if (dt == jl_function_type) { + if (n == -1) // key Type{>:Function} as Type instead of Function + *facts |= EXACTLY_TYPE; // HAVE_TYPE is already set + else + *facts |= HAVE_FUNCTION | EXACTLY_FUNCTION; + } + else if (dt == jl_any_type) { + if (n == -1) // key Type{>:Any} and kinds as Type instead of Any + *facts |= EXACTLY_TYPE; // HAVE_TYPE is already set + else + *facts |= EXACTLY_ANY; + } + else if (dt == jl_kwcall_type) { + if (n == -1) // key Type{>:typeof(kwcall)} as exactly kwcall + *facts |= EXACTLY_KWCALL; + else + *facts |= HAVE_KWCALL; + } + else { + while (1) { + jl_datatype_t *super = dt->super; + if (super == jl_function_type) { + *facts |= HAVE_FUNCTION; + break; + } + if (super == jl_any_type || super->super == dt) + break; + dt = super; } + f(dt->name, 1, env); } } - table = jl_atomic_load_relaxed(&m->bindings); - } + else if (jl_is_tuple_type(a)) { + if (jl_nparams(a) >= n) + foreach_top_nth_typename(f, jl_tparam(a, n - 1), 0, facts, env); + else + *facts |= SHORT_TUPLE; + } + } + else if (jl_is_typevar(a)) { + foreach_top_nth_typename(f, ((jl_tvar_t*)a)->ub, n, facts, env); + } + else if (jl_is_unionall(a)) { + foreach_top_nth_typename(f, ((jl_unionall_t*)a)->body, n, facts, env); + } + else if (jl_is_uniontype(a)) { + jl_uniontype_t *u = (jl_uniontype_t*)a; + foreach_top_nth_typename(f, u->a, n, facts, env); + foreach_top_nth_typename(f, u->b, n, facts, env); + } +} + +// Inspect type `argtypes` for all backedge keys that might be relevant to it, splitting it +// up on some commonly observed patterns to make a better distribution. +// (It could do some of that balancing automatically, but for now just hard-codes kwcall.) +// Along the way, record some facts about what was encountered, so that those additional +// calls can be added later if needed for completeness. +// The `int explct` argument instructs the caller if the callback is due to an exactly +// encountered type or if it rather encountered a subtype. +// This is not capable of walking to all top-typenames for an explicitly encountered +// Function or Any, so the caller a fallback that can scan the entire in that case. +// We do not de-duplicate calls when encountering a Union. +static int jl_foreach_top_typename_for(void (*f)(jl_typename_t*, int, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, int all_subtypes, void *env) +{ + unsigned facts = 0; + foreach_top_nth_typename(f, argtypes, 1, &facts, env); + if (facts & HAVE_KWCALL) { + // split kwcall on the 3rd argument instead, using the same logic + unsigned kwfacts = 0; + foreach_top_nth_typename(f, argtypes, 3, &kwfacts, env); + // copy kwfacts to original facts + if (kwfacts & SHORT_TUPLE) + kwfacts |= (all_subtypes ? EXACTLY_ANY : EXACTLY_KWCALL); + facts |= kwfacts; + } + if (all_subtypes && (facts & (EXACTLY_FUNCTION | EXACTLY_TYPE | EXACTLY_ANY))) + // flag that we have an explct match than is necessitating a full table scan + return 0; + // or inform caller of only which supertypes are applicable + if (facts & HAVE_FUNCTION) + f(jl_function_type->name, facts & EXACTLY_FUNCTION ? 1 : 0, env); + if (facts & HAVE_TYPE) + f(jl_type_typename, facts & EXACTLY_TYPE ? 1 : 0, env); + if (facts & (HAVE_KWCALL | EXACTLY_KWCALL)) + f(jl_kwcall_type->name, facts & EXACTLY_KWCALL ? 1 : 0, env); + f(jl_any_type->name, facts & EXACTLY_ANY ? 1 : 0, env); return 1; } -static int jl_foreach_reachable_typename(int (*visit)(jl_typename_t *tn, void *env), jl_array_t *mod_array, void *env) -{ - for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { - jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); - assert(jl_is_module(m)); - if (m->parent == m) // some toplevel modules (really just Base) aren't actually - if (!foreach_typename_in_module(m, visit, env)) - return 0; - } - return 1; -} -int foreach_mtable_in_module( +static int foreach_mtable_in_module( jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), void *env) @@ -2102,41 +2164,56 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, } +static int jl_foreach_top_typename_for(void (*f)(jl_typename_t*, int, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, int all_subtypes, void *env); + struct _typename_add_backedge { jl_value_t *typ; jl_value_t *caller; }; -static void _typename_add_backedge(jl_typename_t *tn, void *env0) +static void _typename_add_backedge(jl_typename_t *tn, int explct, void *env0) { struct _typename_add_backedge *env = (struct _typename_add_backedge*)env0; JL_GC_PROMISE_ROOTED(env->typ); JL_GC_PROMISE_ROOTED(env->caller); - if (jl_atomic_load_relaxed(&allow_new_worlds)) { - if (!tn->backedges) { - // lazy-init the backedges array - tn->backedges = jl_alloc_vec_any(2); - jl_gc_wb(tn, tn->backedges); - jl_array_ptr_set(tn->backedges, 0, env->typ); - jl_array_ptr_set(tn->backedges, 1, env->caller); + if (!explct) + return; + jl_genericmemory_t *allbackedges = jl_method_table->backedges; + jl_array_t *backedges = (jl_array_t*)jl_eqtable_get(allbackedges, (jl_value_t*)tn, NULL); + if (backedges == NULL) { + backedges = jl_alloc_vec_any(2); + JL_GC_PUSH1(&backedges); + jl_array_del_end(backedges, 2); + jl_genericmemory_t *newtable = jl_eqtable_put(allbackedges, (jl_value_t*)tn, (jl_value_t*)backedges, NULL); + JL_GC_POP(); + if (newtable != allbackedges) { + jl_method_table->backedges = newtable; + jl_gc_wb(jl_method_table, newtable); } - else { - // check if the edge is already present and avoid adding a duplicate - size_t i, l = jl_array_nrows(tn->backedges); - // reuse an already cached instance of this type, if possible - // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? - for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(tn->backedges, i) != env->caller) { - if (jl_types_equal(jl_array_ptr_ref(tn->backedges, i - 1), env->typ)) { - env->typ = jl_array_ptr_ref(tn->backedges, i - 1); - break; - } - } + } + // check if the edge is already present and avoid adding a duplicate + size_t i, l = jl_array_nrows(backedges); + // reuse an already cached instance of this type, if possible + // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? + // TODO: use as_global_root and de-dup edges array too + for (i = 1; i < l; i += 2) { + if (jl_array_ptr_ref(backedges, i) == env->caller) { + if (jl_types_equal(jl_array_ptr_ref(backedges, i - 1), env->typ)) { + env->typ = jl_array_ptr_ref(backedges, i - 1); + return; // this edge already recorded } - jl_array_ptr_1d_push(tn->backedges, env->typ); - jl_array_ptr_1d_push(tn->backedges, env->caller); } } + for (i = 1; i < l; i += 2) { + if (jl_array_ptr_ref(backedges, i) != env->caller) { + if (jl_types_equal(jl_array_ptr_ref(backedges, i - 1), env->typ)) { + env->typ = jl_array_ptr_ref(backedges, i - 1); + break; + } + } + } + jl_array_ptr_1d_push(backedges, env->typ); + jl_array_ptr_1d_push(backedges, env->caller); } // add a backedge from a non-existent signature to caller @@ -2146,11 +2223,13 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_value_t *typ, jl_code_instance if (!jl_atomic_load_relaxed(&allow_new_worlds)) return; // try to pick the best cache(s) for this typ edge - struct _typename_add_backedge env = {typ, (jl_value_t*)caller}; - jl_methcache_t *mc = jl_method_table->cache; + jl_methtable_t *mt = jl_method_table; + jl_methcache_t *mc = mt->cache; JL_LOCK(&mc->writelock); - if (jl_atomic_load_relaxed(&allow_new_worlds)) - jl_foreach_top_typename_for(_typename_add_backedge, typ, &env); + if (jl_atomic_load_relaxed(&allow_new_worlds)) { + struct _typename_add_backedge env = {typ, (jl_value_t*)caller}; + jl_foreach_top_typename_for(_typename_add_backedge, typ, 0, &env); + } JL_UNLOCK(&mc->writelock); } @@ -2164,65 +2243,66 @@ struct _typename_invalidate_backedge { int invalidated; }; -static void _typename_invalidate_backedges(jl_typename_t *tn, void *env0) +static void _typename_invalidate_backedges(jl_typename_t *tn, int explct, void *env0) { struct _typename_invalidate_backedge *env = (struct _typename_invalidate_backedge*)env0; JL_GC_PROMISE_ROOTED(env->type); JL_GC_PROMISE_ROOTED(env->isect); // isJuliaType considers jl_value_t** to be a julia object too JL_GC_PROMISE_ROOTED(env->isect2); // isJuliaType considers jl_value_t** to be a julia object too - if (tn->backedges) { - jl_value_t **backedges = jl_array_ptr_data(tn->backedges); - size_t i, na = jl_array_nrows(tn->backedges); - size_t ins = 0; - for (i = 1; i < na; i += 2) { - jl_value_t *backedgetyp = backedges[i - 1]; - JL_GC_PROMISE_ROOTED(backedgetyp); - int missing = 0; - if (jl_type_intersection2(backedgetyp, (jl_value_t*)env->type, env->isect, env->isect2)) { - // See if the intersection was actually already fully - // covered, but that the new method is ambiguous. - // -> no previous method: now there is one, need to update the missing edge - // -> one+ previously matching method(s): - // -> more specific then all of them: need to update the missing edge - // -> some may have been ambiguous: now there is a replacement - // -> some may have been called: now there is a replacement (also will be detected in the loop later) - // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) - // -> some may have been ambiguous: still are - // -> some may have been called: they may be partly replaced (will be detected in the loop later) - // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against - missing = 1; - for (size_t j = 0; j < env->n; j++) { - jl_method_t *m = env->d[j]; - JL_GC_PROMISE_ROOTED(m); - if (jl_subtype(*env->isect, m->sig) || (*env->isect2 && jl_subtype(*env->isect2, m->sig))) { - // We now know that there actually was a previous - // method for this part of the type intersection. - if (!jl_type_morespecific(env->type, m->sig)) { - missing = 0; - break; - } + jl_array_t *backedges = (jl_array_t*)jl_eqtable_get(jl_method_table->backedges, (jl_value_t*)tn, NULL); + if (backedges == NULL) + return; + jl_value_t **d = jl_array_ptr_data(backedges); + size_t i, na = jl_array_nrows(backedges); + size_t ins = 0; + for (i = 1; i < na; i += 2) { + jl_value_t *backedgetyp = d[i - 1]; + JL_GC_PROMISE_ROOTED(backedgetyp); + int missing = 0; + if (jl_type_intersection2(backedgetyp, (jl_value_t*)env->type, env->isect, env->isect2)) { + // See if the intersection was actually already fully + // covered, but that the new method is ambiguous. + // -> no previous method: now there is one, need to update the missing edge + // -> one+ previously matching method(s): + // -> more specific then all of them: need to update the missing edge + // -> some may have been ambiguous: now there is a replacement + // -> some may have been called: now there is a replacement (also will be detected in the loop later) + // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) + // -> some may have been ambiguous: still are + // -> some may have been called: they may be partly replaced (will be detected in the loop later) + // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against + missing = 1; + for (size_t j = 0; j < env->n; j++) { + jl_method_t *m = env->d[j]; + JL_GC_PROMISE_ROOTED(m); + if (jl_subtype(*env->isect, m->sig) || (*env->isect2 && jl_subtype(*env->isect2, m->sig))) { + // We now know that there actually was a previous + // method for this part of the type intersection. + if (!jl_type_morespecific(env->type, m->sig)) { + missing = 0; + break; } } } - *env->isect = *env->isect2 = NULL; - if (missing) { - jl_code_instance_t *backedge = (jl_code_instance_t*)backedges[i]; - JL_GC_PROMISE_ROOTED(backedge); - invalidate_code_instance(backedge, env->max_world, 0); - env->invalidated = 1; - if (_jl_debug_method_invalidation) - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); - } - else { - backedges[ins++] = backedges[i - 1]; - backedges[ins++] = backedges[i - 0]; - } } - if (ins == 0) - tn->backedges = NULL; - else - jl_array_del_end(tn->backedges, na - ins); + *env->isect = *env->isect2 = NULL; + if (missing) { + jl_code_instance_t *backedge = (jl_code_instance_t*)d[i]; + JL_GC_PROMISE_ROOTED(backedge); + invalidate_code_instance(backedge, env->max_world, 0); + env->invalidated = 1; + if (_jl_debug_method_invalidation) + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); + } + else { + d[ins++] = d[i - 1]; + d[ins++] = d[i - 0]; + } } + if (ins == 0) + jl_eqtable_pop(jl_method_table->backedges, (jl_value_t*)tn, NULL, NULL); + else if (na != ins) + jl_array_del_end(backedges, na - ins); } struct invalidate_mt_env { @@ -2390,14 +2470,6 @@ static int erase_all_backedges(jl_methtable_t *mt, void *env) return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), erase_method_backedges, env); } -static int erase_all_mc_backedges(jl_typename_t *tn, void *env) -{ - tn->backedges = NULL; - return 1; -} - -static int jl_foreach_reachable_typename(int (*visit)(jl_typename_t *tn, void *env), jl_array_t *mod_array, void *env); - JL_DLLEXPORT void jl_disable_new_worlds(void) { if (jl_generating_output()) @@ -2410,7 +2482,7 @@ JL_DLLEXPORT void jl_disable_new_worlds(void) jl_foreach_reachable_mtable(erase_all_backedges, mod_array, (void*)NULL); JL_LOCK(&jl_method_table->cache->writelock); - jl_foreach_reachable_typename(erase_all_mc_backedges, mod_array, (void*)NULL); + jl_method_table->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; JL_UNLOCK(&jl_method_table->cache->writelock); JL_GC_POP(); } @@ -2590,7 +2662,16 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) jl_methcache_t *mc = jl_method_table->cache; JL_LOCK(&mc->writelock); struct _typename_invalidate_backedge typename_env = {type, &isect, &isect2, d, n, max_world, invalidated}; - jl_foreach_top_typename_for(_typename_invalidate_backedges, type, &typename_env); + if (!jl_foreach_top_typename_for(_typename_invalidate_backedges, type, 1, &typename_env)) { + // if the new method cannot be split into exact backedges, scan the whole table for anything that might be affected + jl_genericmemory_t *allbackedges = jl_method_table->backedges; + for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i+1); + if (tn && tn != jl_nothing && backedges) + _typename_invalidate_backedges((jl_typename_t*)tn, 0, &typename_env); + } + } invalidated |= typename_env.invalidated; if (oldmi && jl_array_nrows(oldmi)) { // search mc->cache and leafcache and drop anything that might overlap with the new method diff --git a/src/jltypes.c b/src/jltypes.c index 7d731082bd786..9b664ce25861c 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3004,23 +3004,22 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name->wrapper = (jl_value_t*)jl_typename_type; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 19 - 2; - jl_typename_type->name->names = jl_perm_symsvec(19, "name", "module", "singletonname", + jl_typename_type->name->n_uninitialized = 18 - 2; + jl_typename_type->name->names = jl_perm_symsvec(18, "name", "module", "singletonname", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", - "backedges", "partial", - "hash", "max_args", "n_uninitialized", + "partial", "hash", "max_args", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", "cache_entry_count", "max_methods", "constprop_heuristic"); - const static uint32_t typename_constfields[1] = { 0b0001101000001001011 }; // TODO: put back atomicfields and constfields in this list - const static uint32_t typename_atomicfields[1] = { 0b0010010001110000000 }; + const static uint32_t typename_constfields[1] = { 0b000110100001001011 }; // TODO: put back atomicfields and constfields in this list + const static uint32_t typename_atomicfields[1] = { 0b001001001110000000 }; jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(19, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, + jl_typename_type->types = jl_svec(18, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, - jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, + jl_type_type, jl_simplevector_type, jl_simplevector_type, jl_methcache_type, jl_any_type, jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/, @@ -3046,13 +3045,13 @@ void jl_init_types(void) JL_GC_DISABLED jl_methtable_type->super = jl_any_type; jl_methtable_type->parameters = jl_emptysvec; jl_methtable_type->name->n_uninitialized = 0; - jl_methtable_type->name->names = jl_perm_symsvec(4, "defs", "cache", "name", "module"); - const static uint32_t methtable_constfields[1] = { 0b1110 }; - const static uint32_t methtable_atomicfields[1] = { 0b0001 }; + jl_methtable_type->name->names = jl_perm_symsvec(5, "defs", "cache", "name", "module", "backedges"); + const static uint32_t methtable_constfields[1] = { 0b01110 }; + const static uint32_t methtable_atomicfields[1] = { 0b00001 }; jl_methtable_type->name->constfields = methtable_constfields; jl_methtable_type->name->atomicfields = methtable_atomicfields; jl_precompute_memoized_dt(jl_methtable_type, 1); - jl_methtable_type->types = jl_svec(4, jl_any_type, jl_methcache_type, jl_symbol_type, jl_any_type /*jl_module_type*/); + jl_methtable_type->types = jl_svec(5, jl_any_type, jl_methcache_type, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_any_type); jl_symbol_type->name = jl_new_typename_in(jl_symbol("Symbol"), core, 0, 1); jl_symbol_type->name->wrapper = (jl_value_t*)jl_symbol_type; @@ -3378,6 +3377,7 @@ void jl_init_types(void) JL_GC_DISABLED core = jl_core_module; jl_method_table->module = core; jl_atomic_store_relaxed(&jl_method_table->cache->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_method_table->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); // export own name, so "using Foo" makes "Foo" itself visible jl_set_initial_const(core, core->name, (jl_value_t*)core, 1); @@ -3842,13 +3842,13 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_typename_type->types, 5, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 6, jl_type_type); jl_svecset(jl_typename_type->types, 7, jl_type_type); - jl_svecset(jl_typename_type->types, 12, jl_long_type); + jl_svecset(jl_typename_type->types, 11, jl_long_type); + jl_svecset(jl_typename_type->types, 12, jl_int32_type); jl_svecset(jl_typename_type->types, 13, jl_int32_type); - jl_svecset(jl_typename_type->types, 14, jl_int32_type); + jl_svecset(jl_typename_type->types, 14, jl_uint8_type); jl_svecset(jl_typename_type->types, 15, jl_uint8_type); jl_svecset(jl_typename_type->types, 16, jl_uint8_type); jl_svecset(jl_typename_type->types, 17, jl_uint8_type); - jl_svecset(jl_typename_type->types, 18, jl_uint8_type); jl_svecset(jl_methcache_type->types, 2, jl_long_type); // voidpointer jl_svecset(jl_methcache_type->types, 3, jl_long_type); // uint32_t plus alignment jl_svecset(jl_methtable_type->types, 3, jl_module_type); diff --git a/src/julia.h b/src/julia.h index fff465b0ab114..83c32aa5d85ec 100644 --- a/src/julia.h +++ b/src/julia.h @@ -520,7 +520,6 @@ typedef struct { _Atomic(jl_value_t*) Typeofwrapper; // cache for Type{wrapper} _Atomic(jl_svec_t*) cache; // sorted array _Atomic(jl_svec_t*) linearcache; // unsorted array - jl_array_t *backedges; // uncovered (sig => caller::CodeInstance) pairs with this type as the function jl_array_t *partial; // incomplete instantiations of this type intptr_t hash; _Atomic(int32_t) max_args; // max # of non-vararg arguments in a signature with this type as the function @@ -882,6 +881,7 @@ typedef struct _jl_methtable_t { jl_methcache_t *cache; jl_sym_t *name; // sometimes used for debug printing jl_module_t *module; // sometimes used for debug printing + jl_genericmemory_t *backedges; // IdDict{top typenames, Vector{uncovered (sig => caller::CodeInstance)}} } jl_methtable_t; typedef struct { diff --git a/src/julia_internal.h b/src/julia_internal.h index 24fb7fdb05f90..ed5aea6c80aef 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -857,7 +857,6 @@ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module, size_t new_world); jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st, size_t new_world); int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_array_t *mod_array, void *env); -int foreach_mtable_in_module(jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), void *env); void jl_init_main_module(void); JL_DLLEXPORT int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT; jl_array_t *jl_get_loaded_modules(void); @@ -939,7 +938,6 @@ JL_DLLEXPORT jl_methtable_t *jl_method_get_table( jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methcache_t *jl_method_get_cache( jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -void jl_foreach_top_typename_for(void (*f)(jl_typename_t*, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, void *env); JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t); JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; diff --git a/src/method.c b/src/method.c index 77863b27e24b6..eba485ff9eb69 100644 --- a/src/method.c +++ b/src/method.c @@ -1154,39 +1154,6 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) return gf; } -static void foreach_top_nth_typename(void (*f)(jl_typename_t*, void*), jl_value_t *a JL_PROPAGATES_ROOT, int n, void *env) -{ - if (jl_is_datatype(a)) { - if (n == 0) { - jl_datatype_t *dt = ((jl_datatype_t*)a); - jl_typename_t *tn = NULL; - while (1) { - if (dt != jl_any_type && dt != jl_function_type) - tn = dt->name; - if (dt->super == dt) - break; - dt = dt->super; - } - if (tn) - f(tn, env); - } - else if (jl_is_tuple_type(a)) { - if (jl_nparams(a) >= n) - foreach_top_nth_typename(f, jl_tparam(a, n - 1), 0, env); - } - } - else if (jl_is_typevar(a)) { - foreach_top_nth_typename(f, ((jl_tvar_t*)a)->ub, n, env); - } - else if (jl_is_unionall(a)) { - foreach_top_nth_typename(f, ((jl_unionall_t*)a)->body, n, env); - } - else if (jl_is_uniontype(a)) { - jl_uniontype_t *u = (jl_uniontype_t*)a; - foreach_top_nth_typename(f, u->a, n, env); - foreach_top_nth_typename(f, u->b, n, env); - } -} // get the MethodTable for dispatch, or `nothing` if cannot be determined JL_DLLEXPORT jl_methtable_t *jl_method_table_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT @@ -1200,11 +1167,6 @@ JL_DLLEXPORT jl_methcache_t *jl_method_cache_for(jl_value_t *argtypes JL_PROPAGA return jl_method_table->cache; } -void jl_foreach_top_typename_for(void (*f)(jl_typename_t*, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, void *env) -{ - foreach_top_nth_typename(f, argtypes, 1, env); -} - jl_methcache_t *jl_kwmethod_cache_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { return jl_method_table->cache; diff --git a/src/staticdata.c b/src/staticdata.c index 92e7f494ad35d..3df8e7414420c 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -867,20 +867,30 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->module)); assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } + } + if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; // Any back-edges will be re-validated and added by staticdata.jl, so // drop them from the image here if (s->incremental || jl_options.trim || jl_options.strip_ir) { - record_field_change((jl_value_t**)&tn->backedges, NULL); + record_field_change((jl_value_t**)&mt->backedges, jl_an_empty_memory_any); } else { // don't recurse into all backedges memory (yet) - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&tn->backedges, 1); - if (backedges) { - jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); - for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { - jl_value_t *t = jl_array_ptr_ref(backedges, i); - assert(!jl_is_code_instance(t)); - jl_queue_for_serialization(s, t); + jl_value_t *allbackedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + jl_queue_for_serialization_(s, allbackedges, 0, 1); + for (size_t i = 0, n = ((jl_genericmemory_t*)allbackedges)->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_queue_for_serialization(s, tn); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i + 1); + if (backedges && backedges != jl_nothing) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + jl_queue_for_serialization(s, backedges); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { + jl_value_t *t = jl_array_ptr_ref(backedges, i); + assert(!jl_is_code_instance(t)); + jl_queue_for_serialization(s, t); + } } } } @@ -2573,8 +2583,6 @@ static void jl_prune_mi_backedges(jl_array_t *backedges) static void jl_prune_tn_backedges(jl_array_t *backedges) { - if (backedges == NULL) - return; size_t i = 0, ins = 0, n = jl_array_nrows(backedges); for (i = 1; i < n; i += 2) { jl_value_t *ci = jl_array_ptr_ref(backedges, i); @@ -2586,6 +2594,15 @@ static void jl_prune_tn_backedges(jl_array_t *backedges) jl_array_del_end(backedges, n - ins); } +static void jl_prune_mt_backedges(jl_genericmemory_t *allbackedges) +{ + for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i + 1); + if (tn && tn != jl_nothing && backedges) + jl_prune_tn_backedges((jl_array_t*)backedges); + } +} static void jl_prune_binding_backedges(jl_array_t *backedges) { @@ -3240,8 +3257,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_prune_type_cache_hash(jl_atomic_load_relaxed(&tn->cache))); jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); - jl_value_t *backedges = get_replaceable_field((jl_value_t**)&tn->backedges, 1); - jl_prune_tn_backedges((jl_array_t*)backedges); } else if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; @@ -3253,6 +3268,11 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); jl_prune_binding_backedges((jl_array_t*)backedges); } + else if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + jl_prune_mt_backedges((jl_genericmemory_t*)backedges); + } } } diff --git a/test/misc.jl b/test/misc.jl index d72ad5b58ad12..2b87a2ad26f0f 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1632,10 +1632,10 @@ end let errs = IOBuffer() run(`$(Base.julia_cmd()) -e ' using Test - @test isdefined(Type.body.name, :backedges) + @test !isempty(Core.GlobalMethods.backedges) Base.Experimental.disable_new_worlds() @test_throws "disable_new_worlds" @eval f() = 1 - @test !isdefined(Type.body.name, :backedges) + @test isempty(Core.GlobalMethods.backedges) @test_throws "disable_new_worlds" Base.delete_method(which(+, (Int, Int))) @test 1+1 == 2 using Dates From 1052201ff1b1844029c6a3274ea12755646ac597 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 9 Jun 2025 23:55:09 +0200 Subject: [PATCH 392/662] improve inferred effects for `reshape` for `Array` (#58672) --- base/reshapedarray.jl | 2 +- test/arrayops.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index e831a75dfe3da..5bfa64d756a97 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -36,7 +36,7 @@ length(R::ReshapedArrayIterator) = length(R.iter) eltype(::Type{<:ReshapedArrayIterator{I}}) where {I} = @isdefined(I) ? ReshapedIndex{eltype(I)} : Any @noinline throw_dmrsa(dims, len) = - throw(DimensionMismatch("new dimensions $(dims) must be consistent with array length $len")) + throw(DimensionMismatch(LazyString("new dimensions ", dims, " must be consistent with array length ", len))) ## reshape(::Array, ::Dims) returns a new Array (to avoid conditionally aliasing the structure, only the data) # reshaping to same # of dimensions diff --git a/test/arrayops.jl b/test/arrayops.jl index 7e2454eabd93c..5d68f70e9caf1 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -100,6 +100,18 @@ using Dates @test Array{eltype(a)}(a) !== a @test Vector(a) !== a end +@testset "effect inference for `reshape` for `Array`" begin + for Arr ∈ (Array{<:Any, 0}, Vector, Matrix, Array{<:Any, 3}) + for Shape ∈ (Tuple{Int}, Tuple{Int, Int}) + effects = Base.infer_effects(reshape, Tuple{Arr{Float32}, Shape}) + @test Base.Compiler.is_effect_free(effects) + @test Base.Compiler.is_terminates(effects) + @test Base.Compiler.is_notaskstate(effects) + @test Base.Compiler.is_noub(effects) + @test Base.Compiler.is_nortcall(effects) + end + end +end @testset "reshaping SubArrays" begin a = Array(reshape(1:5, 1, 5)) @testset "linearfast" begin From d6294ba973db1dea9dc932779008fd66d27c4bd2 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 10 Jun 2025 00:27:37 -0400 Subject: [PATCH 393/662] Unicode: Force-inline isgraphemebreak! (#58674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When this API was added, this function inlined, which is important, because the API relies on the allocation of the `Ref` being elided. At some point (I went back to 1.8) this regressed. For example, it is currently responsible for substantially all non-Expr allocations in JuliaParser. Before (parsing all of Base with JuliaParser): ``` │ Memory estimate: 76.93 MiB, allocs estimate: 719922. ``` After: ``` │ Memory estimate: 53.31 MiB, allocs estimate: 156. ``` Also add a test to make sure this doesn't regress again. --- base/strings/unicode.jl | 2 +- stdlib/Unicode/test/runtests.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index 5d59b0fc3bff4..d8f3a2b6ad66b 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -800,7 +800,7 @@ isgraphemebreak(c1::AbstractChar, c2::AbstractChar) = # Stateful grapheme break required by Unicode-9 rules: the string # must be processed in sequence, with state initialized to Ref{Int32}(0). # Requires utf8proc v2.0 or later. -function isgraphemebreak!(state::Ref{Int32}, c1::AbstractChar, c2::AbstractChar) +@inline function isgraphemebreak!(state::Ref{Int32}, c1::AbstractChar, c2::AbstractChar) if ismalformed(c1) || ismalformed(c2) state[] = 0 return true diff --git a/stdlib/Unicode/test/runtests.jl b/stdlib/Unicode/test/runtests.jl index 7fa57508cffbf..2af7015afa249 100644 --- a/stdlib/Unicode/test/runtests.jl +++ b/stdlib/Unicode/test/runtests.jl @@ -284,6 +284,8 @@ end @test_throws BoundsError graphemes("äöüx", 2:5) @test_throws BoundsError graphemes("äöüx", 5:5) @test_throws ArgumentError graphemes("äöüx", 0:1) + + @test @allocated(length(graphemes("äöüx"))) == 0 end @testset "#3721, #6939 up-to-date character widths" begin From 3fa367b7b6cb6fe2cc8c1a3d68be64d004509858 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 10 Jun 2025 17:23:51 +0200 Subject: [PATCH 394/662] `Random`: make `randperm!` and `randcycle!` accept `AbstractArray` (#58596) --- NEWS.md | 4 ++++ stdlib/Random/src/misc.jl | 20 ++++++++++++++------ stdlib/Random/test/runtests.jl | 8 ++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index d38da1cfe1300..1e4e24b67a4db 100644 --- a/NEWS.md +++ b/NEWS.md @@ -61,6 +61,10 @@ Standard library changes #### Profile +#### Random + +* `randperm!` and `randcycle!` now support non-`Array` `AbstractArray` inputs, assuming they are mutable and their indices are one-based ([#58596]). + #### REPL * The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). diff --git a/stdlib/Random/src/misc.jl b/stdlib/Random/src/misc.jl index 16c1962f235d6..0f113e8153f04 100644 --- a/stdlib/Random/src/misc.jl +++ b/stdlib/Random/src/misc.jl @@ -297,13 +297,16 @@ randperm(r::AbstractRNG, n::T) where {T <: Integer} = randperm!(r, Vector{T}(und randperm(n::Integer) = randperm(default_rng(), n) """ - randperm!([rng=default_rng(),] A::Array{<:Integer}) + randperm!([rng=default_rng(),] A::AbstractArray{<:Integer}) Construct in `A` a random permutation of length `length(A)`. The optional `rng` argument specifies a random number generator (see [Random Numbers](@ref)). To randomly permute an arbitrary vector, see [`shuffle`](@ref) or [`shuffle!`](@ref). +!!! compat "Julia 1.13" + `A isa Array` was required prior to Julia v1.13. + # Examples ```jldoctest julia> randperm!(Xoshiro(123), Vector{Int}(undef, 4)) @@ -314,8 +317,9 @@ julia> randperm!(Xoshiro(123), Vector{Int}(undef, 4)) 3 ``` """ -function randperm!(r::AbstractRNG, a::Array{<:Integer}) +function randperm!(r::AbstractRNG, a::AbstractArray{<:Integer}) # keep it consistent with `shuffle!` and `randcycle!` if possible + Base.require_one_based_indexing(a) n = length(a) @assert n <= Int64(2)^52 n == 0 && return a @@ -332,7 +336,7 @@ function randperm!(r::AbstractRNG, a::Array{<:Integer}) return a end -randperm!(a::Array{<:Integer}) = randperm!(default_rng(), a) +randperm!(a::AbstractArray{<:Integer}) = randperm!(default_rng(), a) ## randcycle & randcycle! @@ -370,7 +374,7 @@ randcycle(r::AbstractRNG, n::T) where {T <: Integer} = randcycle!(r, Vector{T}(u randcycle(n::Integer) = randcycle(default_rng(), n) """ - randcycle!([rng=default_rng(),] A::Array{<:Integer}) + randcycle!([rng=default_rng(),] A::AbstractArray{<:Integer}) Construct in `A` a random cyclic permutation of length `n = length(A)`. The optional `rng` argument specifies a random number generator, see @@ -382,6 +386,9 @@ which are sampled uniformly. If `A` is empty, `randcycle!` leaves it unchanged. [`randcycle`](@ref) is a variant of this function that allocates a new vector. +!!! compat "Julia 1.13" + `A isa Array` was required prior to Julia v1.13. + # Examples ```jldoctest julia> randcycle!(Xoshiro(123), Vector{Int}(undef, 6)) @@ -394,8 +401,9 @@ julia> randcycle!(Xoshiro(123), Vector{Int}(undef, 6)) 1 ``` """ -function randcycle!(r::AbstractRNG, a::Array{<:Integer}) +function randcycle!(r::AbstractRNG, a::AbstractArray{<:Integer}) # keep it consistent with `shuffle!` and `randperm!` if possible + Base.require_one_based_indexing(a) n = length(a) @assert n <= Int64(2)^52 n == 0 && return a @@ -411,4 +419,4 @@ function randcycle!(r::AbstractRNG, a::Array{<:Integer}) return a end -randcycle!(a::Array{<:Integer}) = randcycle!(default_rng(), a) +randcycle!(a::AbstractArray{<:Integer}) = randcycle!(default_rng(), a) diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index cbd8eb41f5515..e929cd2d8a517 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -554,6 +554,14 @@ end @test randcycle!(mta, A) == randcycle!(mtb, B) @test randcycle!(A) === A + @testset "non-`Array` `randperm!` and `randcycle!`" begin + x, y = Memory{Int}(undef, 10), Memory{Int}(undef, 10) + @test randperm!(mta, x) == randperm!(mtb, y) + @test randperm!(x) === x + @test randcycle!(mta, x) == randcycle!(mtb, y) + @test randcycle!(x) === x + end + let p = randcycle(UInt16(10)) @test typeof(p) ≡ Vector{UInt16} @test sort!(p) == 1:10 From 66d3cf554747a3724c7ac9a5beb8a9dd9446f96b Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 10 Jun 2025 11:24:08 -0400 Subject: [PATCH 395/662] cfunction: store `fptr` / `world` contiguously (#58684) --- src/aotcompile.cpp | 15 ++++++++------- src/codegen.cpp | 25 +++++++++---------------- src/jitlayers.h | 1 - src/julia_internal.h | 2 +- src/runtime_ccall.cpp | 22 ++++++++++++---------- 5 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 33c0ad011e6f0..0251e9db4b38b 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -576,7 +576,7 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : params.cfuncs) { - Module *M = cfunc.theFptr->getParent(); + Module *M = cfunc.cfuncdata->getParent(); jl_value_t *sigt = cfunc.sigt; JL_GC_PROMISE_ROOTED(sigt); jl_value_t *declrt = cfunc.declrt; @@ -585,19 +585,20 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct jl_code_instance_t *codeinst = nullptr; auto assign_fptr = [¶ms, &cfunc, &codeinst, &unspec](Function *f) { ConstantArray *init = cast(cfunc.cfuncdata->getInitializer()); - SmallVector initvals; + SmallVector initvals; for (unsigned i = 0; i < init->getNumOperands(); ++i) initvals.push_back(init->getOperand(i)); - assert(initvals.size() == 6); + assert(initvals.size() == 8); assert(initvals[0]->isNullValue()); + assert(initvals[2]->isNullValue()); if (codeinst) { Constant *llvmcodeinst = literal_pointer_val_slot(params, f->getParent(), (jl_value_t*)codeinst); - initvals[0] = llvmcodeinst; // plast_codeinst + initvals[2] = llvmcodeinst; // plast_codeinst } - assert(initvals[2]->isNullValue()); - initvals[2] = unspec; + assert(initvals[4]->isNullValue()); + initvals[4] = unspec; + initvals[0] = f; cfunc.cfuncdata->setInitializer(ConstantArray::get(init->getType(), initvals)); - cfunc.theFptr->setInitializer(f); }; Module *defM = nullptr; StringRef func; diff --git a/src/codegen.cpp b/src/codegen.cpp index a736449813608..cb556f3b07a2e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1368,8 +1368,8 @@ static const auto jlgetabiconverter_func = new JuliaFunction{ @@ -7167,26 +7167,23 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ Type *T_size = ctx.types().T_size; Constant *Vnull = ConstantPointerNull::get(T_ptr); Module *M = jl_Module; - GlobalVariable *theFptr = new GlobalVariable(*M, T_ptr, false, - GlobalVariable::PrivateLinkage, - Vnull); - GlobalVariable *last_world_p = new GlobalVariable(*M, T_size, false, - GlobalVariable::PrivateLinkage, - ConstantInt::get(T_size, 0)); - ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 6); + ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 8); size_t flags = specsig; GlobalVariable *cfuncdata = new GlobalVariable(*M, T_cfuncdata, false, GlobalVariable::PrivateLinkage, ConstantArray::get(T_cfuncdata, { + Vnull, + Vnull, Vnull, Vnull, Vnull, literal_pointer_val_slot(ctx.emission_context, M, declrt), literal_pointer_val_slot(ctx.emission_context, M, sigt), literal_static_pointer_val((void*)flags, T_ptr)})); + Value *last_world_p = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, cfuncdata, 1); LoadInst *last_world_v = ctx.builder.CreateAlignedLoad(T_size, last_world_p, ctx.types().alignof_ptr); last_world_v->setOrdering(AtomicOrdering::Acquire); - LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, theFptr, ctx.types().alignof_ptr); + LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, cfuncdata, ctx.types().alignof_ptr); callee->setOrdering(AtomicOrdering::Monotonic); LoadInst *world_v = ctx.builder.CreateAlignedLoad(ctx.types().T_size, prepare_global_in(M, jlgetworld_global), ctx.types().alignof_ptr); @@ -7195,15 +7192,11 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ Value *age_not_ok = ctx.builder.CreateICmpNE(last_world_v, world_v); Value *target = emit_guarded_test(ctx, age_not_ok, callee, [&] { Function *getcaller = prepare_call(jlgetabiconverter_func); - CallInst *cw = ctx.builder.CreateCall(getcaller, { - get_current_task(ctx), - theFptr, - last_world_p, - cfuncdata}); + CallInst *cw = ctx.builder.CreateCall(getcaller, {get_current_task(ctx), cfuncdata}); cw->setAttributes(getcaller->getAttributes()); return cw; }); - ctx.emission_context.cfuncs.push_back({declrt, sigt, nargs, specsig, theFptr, cfuncdata}); + ctx.emission_context.cfuncs.push_back({declrt, sigt, nargs, specsig, cfuncdata}); if (specsig) { // TODO: could we force this to guarantee passing a box for `f` here (since we // know we had it here) and on the receiver end (emit_abi_converter / diff --git a/src/jitlayers.h b/src/jitlayers.h index 43fdc9b83b89b..7d0b6d9f02d6f 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -224,7 +224,6 @@ struct cfunc_decl_t { jl_value_t *sigt; size_t nargs; bool specsig; - llvm::GlobalVariable *theFptr; llvm::GlobalVariable *cfuncdata; }; diff --git a/src/julia_internal.h b/src/julia_internal.h index ed5aea6c80aef..7c73ac072251a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1628,7 +1628,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill, void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals); -JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); +JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data); JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 9b7374d95af50..0ef81357f8baf 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -327,6 +327,8 @@ jl_value_t *jl_get_cfunction_trampoline( JL_GCC_IGNORE_STOP struct cfuncdata_t { + _Atomic(void *) fptr; + _Atomic(size_t) last_world; jl_code_instance_t** plast_codeinst; jl_code_instance_t* last_codeinst; void *unspecialized; @@ -358,7 +360,7 @@ static jl_mutex_t cfun_lock; // read theFptr // acquire jl_world_counter extern "C" JL_DLLEXPORT -void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data) +void *jl_get_abi_converter(jl_task_t *ct, void *data) { cfuncdata_t *cfuncdata = (cfuncdata_t*)data; jl_value_t *sigt = *cfuncdata->sigt; @@ -373,8 +375,8 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) // check first, while behind this lock, of the validity of the current contents of this cfunc thunk JL_LOCK(&cfun_lock); do { - size_t last_world_v = jl_atomic_load_relaxed(last_world); - void *f = jl_atomic_load_relaxed(fptr); + size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); + void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); jl_code_instance_t *last_ci = cfuncdata->plast_codeinst ? *cfuncdata->plast_codeinst : nullptr; world = jl_atomic_load_acquire(&jl_world_counter); ct->world_age = world; @@ -386,14 +388,14 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) if (f != nullptr) { if (last_ci == nullptr) { if (mi == nullptr) { - jl_atomic_store_release(last_world, world); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; } } else { if (jl_get_ci_mi(last_ci) == mi && jl_atomic_load_relaxed(&last_ci->max_world) >= world) { // same dispatch and source - jl_atomic_store_release(last_world, world); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; } @@ -406,17 +408,17 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) JL_LOCK(&cfun_lock); } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart entirely, since jl_world_counter changed thus jl_get_specialization1 might have changed // double-check if the values were set on another thread - size_t last_world_v = jl_atomic_load_relaxed(last_world); - void *f = jl_atomic_load_relaxed(fptr); + size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); + void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); if (world == last_world_v) { JL_UNLOCK(&cfun_lock); return f; // another thread fixed this up while we were away } - auto assign_fptr = [fptr, last_world, cfuncdata, world, codeinst](void *f) { + auto assign_fptr = [cfuncdata, world, codeinst](void *f) { cfuncdata->plast_codeinst = &cfuncdata->last_codeinst; cfuncdata->last_codeinst = codeinst; - jl_atomic_store_relaxed(fptr, f); - jl_atomic_store_release(last_world, world); + jl_atomic_store_relaxed(&cfuncdata->fptr, f); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; }; From abb49424c66be651abbeb553a103c3a9be30b9de Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 10 Jun 2025 23:35:56 +0530 Subject: [PATCH 396/662] Specialize iterate for `IndexLinear` arrays (#58635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a specialized `iterate` method for linearly indexed arrays. Firstly, we explicitly evaluate the iteration over the range that corresponds to the linear indices. Secondly, we return `nothing` for an out-of-bounds state, instead of throwing a `BoundsError`. This also lets us annotate the actual indexing with `@inbounds`. On master ```julia julia> A = rand(1000,1000); v = view(A, :); v2 = view(A, 1:2:lastindex(A)); julia> using LinearAlgebra julia> @btime norm(Iterators.map(splat(-), zip($A, $A))); 434.982 μs (0 allocations: 0 bytes) julia> @btime norm(Iterators.map(splat(-), zip($v, $v))); 2.080 ms (0 allocations: 0 bytes) # v"1.13.0-DEV.709" 510.835 μs (0 allocations: 0 bytes) # this PR ``` The performance of iterating over the contiguous `view` becomes comparable to that of the `Array`. Fixes https://github.com/JuliaLang/julia/issues/43295 without the need to disable bounds-checking. --------- Co-authored-by: N5N3 <2642243996@qq.com> --- base/abstractarray.jl | 10 +++++++++- test/abstractarray.jl | 10 ++++++++++ test/stacktraces.jl | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index bf2a6ebabecba..d026f8082cfcd 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1234,11 +1234,19 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) # While the definitions for IndexLinear are all simple enough to inline on their # own, IndexCartesian's CartesianIndices is more complicated and requires explicit # inlining. -function iterate(A::AbstractArray, state=(eachindex(A),)) +iterate_starting_state(A) = iterate_starting_state(A, IndexStyle(A)) +iterate_starting_state(A, ::IndexLinear) = firstindex(A) +iterate_starting_state(A, ::IndexStyle) = (eachindex(A),) +iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate(A, state) +function _iterate(A::AbstractArray, state::Tuple) y = iterate(state...) y === nothing && return nothing A[y[1]], (state[1], tail(y)...) end +function _iterate(A::AbstractArray, state::Integer) + checkbounds(Bool, A, state) || return nothing + @inbounds(A[state]), state + one(state) +end isempty(a::AbstractArray) = (length(a) == 0) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 01e13f17460b5..bbac7812174da 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2286,3 +2286,13 @@ end @test_throws "no method matching $Int(::$Infinity)" similar(ones(2), OneToInf()) end end + +@testset "iterate for linear indexing" begin + A = [1 2; 3 4] + v = view(A, :) + @test sum(x for x in v) == sum(A) + v = view(A, 1:2:lastindex(A)) + @test sum(x for x in v) == sum(A[1:2:end]) + v2 = view(A, Base.IdentityUnitRange(1:length(A))) + @test sum(x for x in v2) == sum(A) +end diff --git a/test/stacktraces.jl b/test/stacktraces.jl index 3df0998fe88f6..f71cf9f616eaa 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -248,7 +248,7 @@ struct F49231{a,b,c,d,e,f,g} end stacktrace(catch_backtrace()) end str = sprint(Base.show_backtrace, st, context = (:limit=>true, :stacktrace_types_limited => Ref(false), :color=>true, :displaysize=>(50,105))) - @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0mTuple\e[90m{…}\e[39m\e[0m\e[1m)\e[22m\n\e[90m") + @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n\e[90m") st = try F49231{Vector,Val{'}'},Vector{Vector{Vector{Vector}}},Tuple{Int,Int,Int,Int,Int,Int,Int},Int,Int,Int}()(1,2,3) From cdb158b58c9f0be4281fef1631c9c6e796532900 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 10 Jun 2025 15:11:39 -0400 Subject: [PATCH 397/662] restore fallback 3-arg `setprecision` method (#58586) fixes #55899 --------- Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> --- HISTORY.md | 2 ++ base/mpfr.jl | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index ce4e9982a2c6b..dd666eb3312ed 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -134,6 +134,8 @@ New library features * `Timer` now has readable `timeout` and `interval` properties, and a more descriptive `show` method ([#57081]). * `sort` now supports `NTuple`s ([#54494]). * `map!(f, A)` now stores the results in `A`, like `map!(f, A, A)` or `A .= f.(A)` ([#40632]). +* `setprecision` with a function argument (typically a `do` block) is now thread safe. Other forms + should be avoided, and types should switch to an implementation using `ScopedValue` ([#51362]). Standard library changes ------------------------ diff --git a/base/mpfr.jl b/base/mpfr.jl index 25b7f0abe4b98..ee1e4eac39145 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -1183,9 +1183,29 @@ Often used as `setprecision(T, precision) do ... end` Note: `nextfloat()`, `prevfloat()` do not use the precision mentioned by `setprecision`. +!!! warning + There is a fallback implementation of this method that calls `precision` + and `setprecision`, but it should no longer be relied on. Instead, you + should define the 3-argument form directly in a way that uses `ScopedValue`, + or recommend that callers use `ScopedValue` and `@with` themselves. + !!! compat "Julia 1.8" The `base` keyword requires at least Julia 1.8. """ +function setprecision(f::Function, ::Type{T}, prec::Integer; kws...) where T + depwarn(""" + The fallback `setprecision(::Function, ...)` method is deprecated. Packages overloading this method should + implement their own specialization using `ScopedValue` instead. + """, :setprecision) + old_prec = precision(T) + setprecision(T, prec; kws...) + try + return f() + finally + setprecision(T, old_prec) + end +end + function setprecision(f::Function, ::Type{BigFloat}, prec::Integer; base::Integer=2) Base.ScopedValues.@with(CURRENT_PRECISION => _convert_precision_from_base(prec, base), f()) end From da004516e7b53a5cfd264170fcb3ea33f9ea0bc6 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 10 Jun 2025 16:49:55 -0300 Subject: [PATCH 398/662] Add 0 predecessor to entry basic block and handle it in inlining (#58683) --- Compiler/src/ssair/inlining.jl | 19 +++++++++++-------- Compiler/src/ssair/ir.jl | 3 +++ Compiler/test/ssair.jl | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index ec3879e27357f..251767d577157 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -126,10 +126,11 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn block = block_for_inst(ir, idx) inline_into_block!(state, block) - if !isempty(inlinee_cfg.blocks[1].preds) + if length(inlinee_cfg.blocks[1].preds) > 1 need_split_before = true + else + @assert inlinee_cfg.blocks[1].preds[1] == 0 end - last_block_idx = last(state.cfg.blocks[block].stmts) if false # TODO: ((idx+1) == last_block_idx && isa(ir[SSAValue(last_block_idx)], GotoNode)) need_split = false @@ -166,12 +167,18 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn end new_block_range = (length(state.new_cfg_blocks)-length(inlinee_cfg.blocks)+1):length(state.new_cfg_blocks) - # Fixup the edges of the newely added blocks + # Fixup the edges of the newly added blocks for (old_block, new_block) in enumerate(bb_rename_range) if old_block != 1 || need_split_before p = state.new_cfg_blocks[new_block].preds let bb_rename_range = bb_rename_range map!(p, p) do old_pred_block + # the meaning of predecessor 0 depends on the block we encounter it: + # - in the first block, it represents the function entry and so needs to be re-mapped + if old_block == 1 && old_pred_block == 0 + return first(bb_rename_range) - 1 + end + # - elsewhere, it represents external control-flow from a caught exception which is un-affected by inlining return old_pred_block == 0 ? 0 : bb_rename_range[old_pred_block] end end @@ -186,10 +193,6 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn end end - if need_split_before - push!(state.new_cfg_blocks[first(bb_rename_range)].preds, first(bb_rename_range)-1) - end - any_edges = false for (old_block, new_block) in enumerate(bb_rename_range) if (length(state.new_cfg_blocks[new_block].succs) == 0) @@ -399,7 +402,7 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector else bb_offset, post_bb_id = popfirst!(todo_bbs) # This implements the need_split_before flag above - need_split_before = !isempty(item.ir.cfg.blocks[1].preds) + need_split_before = length(item.ir.cfg.blocks[1].preds) > 1 if need_split_before finish_current_bb!(compact, 0) end diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index 87c8e7f0bc00d..e6a8ffe6d539e 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -105,6 +105,9 @@ function compute_basic_blocks(stmts::Vector{Any}) end # Compute successors/predecessors for (num, b) in enumerate(blocks) + if b.stmts.start == 1 + push!(b.preds, 0) # the entry block has a virtual predecessor + end terminator = stmts[last(b.stmts)] if isa(terminator, ReturnNode) # return never has any successors diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index a855439877f1b..7aca2b8977a4e 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -825,3 +825,23 @@ end @test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = "nonexisting pass name") @test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = typemax(Int)) + +#57153 check that the CFG has a #0 block predecessor and that we don't fail to compile code that observes that +function _worker_task57153() + while true + r = let + try + if @noinline rand(Bool) + return nothing + end + q, m + finally + missing + end + end + r[1]::Bool + end +end +let ir = Base.code_ircode(_worker_task57153, (), optimize_until="CC: COMPACT_2")[1].first + @test findfirst(x->x==0, ir.cfg.blocks[1].preds) !== nothing +end From 8567a3a10f4b746b91bf406bfe3171c3399aed8d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 11 Jun 2025 01:14:38 -0400 Subject: [PATCH 399/662] Test: Fix failfast for for loops (#58695) --- stdlib/Test/src/Test.jl | 10 ++++++++-- stdlib/Test/test/runtests.jl | 28 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index f379c4dd15ac3..7ad1c35294be7 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1726,6 +1726,10 @@ end trigger_test_failure_break(@nospecialize(err)) = ccall(:jl_test_failure_breakpoint, Cvoid, (Any,), err) +is_failfast_error(err::FailFastError) = true +is_failfast_error(err::LoadError) = is_failfast_error(err.error) # handle `include` barrier +is_failfast_error(err) = false + """ Generate the code for an `@testset` with a `let` argument. """ @@ -1837,7 +1841,7 @@ function testset_beginend_call(args, tests, source) # something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - if err isa FailFastError + if is_failfast_error(err) get_testset_depth() > 1 ? rethrow() : failfast_print() else record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) @@ -1925,7 +1929,9 @@ function testset_forloop(args, testloop, source) # Something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - if !isa(err, FailFastError) + if is_failfast_error(err) + get_testset_depth() > 1 ? rethrow() : failfast_print() + else record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) end end diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index cffaa778cf265..4a8a25f0089d4 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1394,7 +1394,7 @@ end @test occursin(expected, result) end end - @testset "failfast" begin + @testset "failfast begin-end" begin expected = r""" Test Summary: \| Fail Total +Time Foo \| 1 1 \s*\d*\.\ds @@ -1419,6 +1419,32 @@ end @test occursin(expected, result) end end + @testset "failfast for-loop" begin + expected = r""" + Test Summary: \| Fail Total +Time + Foo \| 1 1 \s*\d*\.\ds + 1 \| 1 1 \s*\d*\.\ds + """ + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Foo" failfast=true begin + @testset "\$x" for x in 1:2 + @test false + end + @testset "Bar" begin + @test false + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end @testset "failfast passes to child testsets" begin expected = r""" Test Summary: \| Fail Total +Time From a970e16c3f466415606a25b113ac5bb7d21c89de Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Wed, 11 Jun 2025 12:51:52 -0400 Subject: [PATCH 400/662] `Integer` can be hashed rapidly as well (#58440) as noted in https://github.com/JuliaLang/julia/pull/58388#discussion_r2085241015 , using `hash_bytes` for `BigInt` limbs in isolation will break some hashing invariants. accordingly this PR updates also the `hash_integer` fallback to use rapidhash some surgery was needed to make `BigFloat` and `Rational` and such still match, but I think (hope?) I got it all below are some benchmarks. the `y` axis is nanoseconds and the `x` axis is an input of size `1234^x`, so `length = 10` means input `hash(big(1234^10))` hashing `BigFloat` got a small bit slower, but this already allocates and already seems less common than hashing `BigInt` (whose hashing remains non-allocating) ![image](https://github.com/user-attachments/assets/df6de62d-5c8e-4779-b5fc-67143f868572) ![image](https://github.com/user-attachments/assets/5c7201b2-78a4-4e80-bee6-989fd29cfc86) --- base/gmp.jl | 44 ++++++++++++------------- base/hashing.jl | 80 +++++++++++++++++++++++++++++++++++++++------ base/irrationals.jl | 2 +- base/rational.jl | 2 +- test/gmp.jl | 12 +++++-- 5 files changed, 101 insertions(+), 39 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index 7ec41b189478b..1d3c86de5abf2 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -843,24 +843,25 @@ Base.deepcopy_internal(x::BigInt, stackdict::IdDict) = get!(() -> MPZ.set(x), st ## streamlined hashing for BigInt, by avoiding allocation from shifts ## +Base._hash_shl!(x::BigInt, n) = MPZ.mul_2exp!(x, n) + if Limb === UInt64 === UInt # On 64 bit systems we can define # an optimized version for BigInt of hash_integer (used e.g. for Rational{BigInt}), # and of hash - using .Base: hash_finalizer + using .Base: HASH_SECRET, hash_bytes, hash_finalizer function hash_integer(n::BigInt, h::UInt) GC.@preserve n begin s = n.size - s == 0 && return hash_integer(0, h) - p = convert(Ptr{UInt64}, n.d) - b = unsafe_load(p) - h ⊻= hash_finalizer(ifelse(s < 0, -b, b) ⊻ h) - for k = 2:abs(s) - h ⊻= hash_finalizer(unsafe_load(p, k) ⊻ h) - end - return h + h ⊻= (s < 0) + hash_bytes( + Ptr{UInt8}(n.d), + 8 * abs(s), + h, + HASH_SECRET + ) end end @@ -892,21 +893,16 @@ if Limb === UInt64 === UInt return hash(ldexp(flipsign(Float64(limb), sz), pow), h) end h = hash_integer(pow, h) - h ⊻= hash_finalizer(flipsign(limb, sz) ⊻ h) - for idx = idx+1:asz - if shift == 0 - limb = unsafe_load(ptr, idx) - else - limb1 = limb2 - if idx == asz - limb = limb1 >> shift - limb == 0 && break # don't hash leading zeros - else - limb2 = unsafe_load(ptr, idx+1) - limb = limb2 << upshift | limb1 >> shift - end - end - h ⊻= hash_finalizer(limb ⊻ h) + + h ⊻= (sz < 0) + trailing_zero_bytes = div(pow, 8) + GC.@preserve x begin + h = hash_bytes( + Ptr{UInt8}(x.d) + 8 * trailing_zero_bytes, + 8 * (asz - trailing_zero_bytes), + h, + HASH_SECRET + ) end return h end diff --git a/base/hashing.jl b/base/hashing.jl index 01274e3182245..b714013da8cb5 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -69,17 +69,72 @@ hash(x::UInt64, h::UInt) = hash_uint64(hash_mix_linear(x, h)) hash(x::Int64, h::UInt) = hash(bitcast(UInt64, x), h) hash(x::Union{Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32}, h::UInt) = hash(Int64(x), h) -function hash_integer(n::Integer, h::UInt) - h ⊻= hash_uint((n % UInt) ⊻ h) - n = abs(n) - n >>>= sizeof(UInt) << 3 - while n != 0 - h ⊻= hash_uint((n % UInt) ⊻ h) - n >>>= sizeof(UInt) << 3 +hash_integer(x::Integer, h::UInt) = _hash_integer(x, UInt64(h)) % UInt +function _hash_integer( + x::Integer, + seed::UInt64 = HASH_SEED, + secret::NTuple{3, UInt64} = HASH_SECRET + ) + seed ⊻= (x < 0) + u = abs(x) + + # always left-pad to multiple of 8 bytes + buflen = UInt(cld(top_set_bit(u), 64) * 8) + seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + + a = zero(UInt64) + b = zero(UInt64) + + if buflen ≤ 16 + a = (UInt64(u % UInt32) << 32) | + UInt64((u >>> ((buflen - 4) * 8)) % UInt32) + + delta = (buflen & 24) >>> (buflen >>> 3) + + b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) | + UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32) + else + a = (u >>> 8(buflen - 16)) % UInt + b = (u >>> 8(buflen - 8)) % UInt + + i = buflen + if i > 48 + see1 = seed + see2 = seed + while i ≥ 48 + l0 = u % UInt; u >>>= 64 + l1 = u % UInt; u >>>= 64 + l2 = u % UInt; u >>>= 64 + l3 = u % UInt; u >>>= 64 + l4 = u % UInt; u >>>= 64 + l5 = u % UInt; u >>>= 64 + + seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed) + see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1) + see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2) + end + seed = seed ⊻ see1 ⊻ see2 + i -= 48 + end + if i > 16 + l0 = u % UInt; u >>>= 64 + l1 = u % UInt; u >>>= 64 + seed = hash_mix(l0 ⊻ secret[3], l1 ⊻ seed ⊻ secret[2]) + if i > 32 + l2 = u % UInt; u >>>= 64 + l3 = u % UInt; u >>>= 64 + seed = hash_mix(l2 ⊻ secret[3], l3 ⊻ seed) + end + end end - return h + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) end + ## efficient value-based hashing of floats ## const hx_NaN = hash(reinterpret(UInt64, NaN)) @@ -117,6 +172,7 @@ function hash(x::Float16, h::UInt) end ## generic hashing for rational values ## +_hash_shl!(x, n) = (x << n) function hash(x::Real, h::UInt) # decompose x as num*2^pow/den num, pow, den = decompose(x) @@ -132,6 +188,7 @@ function hash(x::Real, h::UInt) den = -den end num_z = trailing_zeros(num) + num >>= num_z den_z = trailing_zeros(den) den >>= den_z @@ -156,7 +213,10 @@ function hash(x::Real, h::UInt) end # handle generic rational values h = hash_integer(pow, h) - h = hash_integer(num, h) + + # trimming only whole bytes of trailing zeros simplifies greatly + # some specializations for memory-backed bitintegers + h = hash_integer((pow > 0) ? _hash_shl!(num, pow % 8) : num, h) return h end @@ -209,7 +269,7 @@ end else pos = 1 i = buflen - while i ≥ 48 + if i > 48 see1 = seed see2 = seed while i ≥ 48 diff --git a/base/irrationals.jl b/base/irrationals.jl index f86b55e4faaa7..e0e7fc3bc2e1d 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -182,7 +182,7 @@ isinteger(::AbstractIrrational) = false iszero(::AbstractIrrational) = false isone(::AbstractIrrational) = false -hash(x::Irrational, h::UInt) = 3*objectid(x) - h +hash(x::Irrational, h::UInt) = 3h - objectid(x) widen(::Type{T}) where {T<:Irrational} = T diff --git a/base/rational.jl b/base/rational.jl index 8c5244ea4bad3..6e1edb457277b 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -620,7 +620,7 @@ function hash(x::Rational{<:BitInteger64}, h::UInt) end end h = hash_integer(pow, h) - h = hash_integer(num, h) + h = hash_integer((pow > 0) ? (num << (pow % 64)) : num, h) return h end diff --git a/test/gmp.jl b/test/gmp.jl index 991e4613bb621..296fd336527ed 100644 --- a/test/gmp.jl +++ b/test/gmp.jl @@ -811,8 +811,14 @@ end @testset "hashing" begin for i in 1:10:100 - bint = big(11)^i - bfloat = big(11.0)^i - @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + for shift in 0:3 + bint = big(11)^i << shift + bfloat = float(bint) + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + @test hash(bint, Base.HASH_SEED) == + @invoke(hash(bint::Real, Base.HASH_SEED)) + @test Base.hash_integer(bint, Base.HASH_SEED) == + @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) + end end end From c0ecfe9bccfdf1ea0e87c31fc2ddf3db69773c9b Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Wed, 11 Jun 2025 19:52:17 -0400 Subject: [PATCH 401/662] Follow-up to hashing BigInts; expand testing (#58714) fixes https://github.com/JuliaLang/julia/issues/58711 closes https://github.com/JuliaLang/julia/pull/58712 --- base/gmp.jl | 22 ++++++++++++---------- base/hashing.jl | 25 +++++++++++++++++-------- test/gmp.jl | 26 ++++++++++++++++++-------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index 1d3c86de5abf2..e4d9294766aaa 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -853,12 +853,16 @@ if Limb === UInt64 === UInt using .Base: HASH_SECRET, hash_bytes, hash_finalizer function hash_integer(n::BigInt, h::UInt) + iszero(n) && return hash_integer(0, h) GC.@preserve n begin s = n.size h ⊻= (s < 0) + + us = abs(s) + leading_zero_bytes = div(leading_zeros(unsafe_load(n.d, us)), 8) hash_bytes( Ptr{UInt8}(n.d), - 8 * abs(s), + 8 * us - leading_zero_bytes, h, HASH_SECRET ) @@ -895,16 +899,14 @@ if Limb === UInt64 === UInt h = hash_integer(pow, h) h ⊻= (sz < 0) + leading_zero_bytes = div(leading_zeros(unsafe_load(x.d, asz)), 8) trailing_zero_bytes = div(pow, 8) - GC.@preserve x begin - h = hash_bytes( - Ptr{UInt8}(x.d) + 8 * trailing_zero_bytes, - 8 * (asz - trailing_zero_bytes), - h, - HASH_SECRET - ) - end - return h + return hash_bytes( + Ptr{UInt8}(x.d) + trailing_zero_bytes, + 8 * asz - (leading_zero_bytes + trailing_zero_bytes), + h, + HASH_SECRET + ) end end end diff --git a/base/hashing.jl b/base/hashing.jl index b714013da8cb5..848b69fee7e23 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -78,21 +78,30 @@ function _hash_integer( seed ⊻= (x < 0) u = abs(x) - # always left-pad to multiple of 8 bytes - buflen = UInt(cld(top_set_bit(u), 64) * 8) + # always left-pad to full byte + buflen = UInt(max(cld(top_set_bit(u), 8), 1)) seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) a = zero(UInt64) b = zero(UInt64) if buflen ≤ 16 - a = (UInt64(u % UInt32) << 32) | - UInt64((u >>> ((buflen - 4) * 8)) % UInt32) + if buflen ≥ 4 + a = (UInt64(u % UInt32) << 32) | + UInt64((u >>> ((buflen - 4) * 8)) % UInt32) - delta = (buflen & 24) >>> (buflen >>> 3) + delta = (buflen & 24) >>> (buflen >>> 3) - b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) | - UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32) + b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) | + UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32) + else # buflen > 0 + b0 = u % UInt8 + b1 = (u >>> (8 * div(buflen, 2))) % UInt8 + b2 = (u >>> (8 * (buflen - 1))) % UInt8 + a = (UInt64(b0) << 56) | + (UInt64(b1) << 32) | + UInt64(b2) + end else a = (u >>> 8(buflen - 16)) % UInt b = (u >>> 8(buflen - 8)) % UInt @@ -112,9 +121,9 @@ function _hash_integer( seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed) see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1) see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2) + i -= 48 end seed = seed ⊻ see1 ⊻ see2 - i -= 48 end if i > 16 l0 = u % UInt; u >>>= 64 diff --git a/test/gmp.jl b/test/gmp.jl index 296fd336527ed..c0edb462298ea 100644 --- a/test/gmp.jl +++ b/test/gmp.jl @@ -811,14 +811,24 @@ end @testset "hashing" begin for i in 1:10:100 - for shift in 0:3 - bint = big(11)^i << shift - bfloat = float(bint) - @test (hash(bint) == hash(bfloat)) == (bint == bfloat) - @test hash(bint, Base.HASH_SEED) == - @invoke(hash(bint::Real, Base.HASH_SEED)) - @test Base.hash_integer(bint, Base.HASH_SEED) == - @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) + for shift in vcat(0:8, 9:8:81) + for sgn in (1, -1) + bint = sgn * (big(11)^i << shift) + bfloat = float(bint) + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + @test hash(bint, Base.HASH_SEED) == + @invoke(hash(bint::Real, Base.HASH_SEED)) + @test Base.hash_integer(bint, Base.HASH_SEED) == + @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) + end end end + + bint = big(0) + bfloat = float(bint) + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + @test hash(bint, Base.HASH_SEED) == + @invoke(hash(bint::Real, Base.HASH_SEED)) + @test Base.hash_integer(bint, Base.HASH_SEED) == + @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) end From b62132871a5fc77a399db4e9843390651b527beb Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:12:28 -0400 Subject: [PATCH 402/662] [NFC] codegen: introduce `jl_abi_t` type for ABI adapter API (#58697) With luck, this makes it a bit easier to read which parameters are grouped together and which apply to the caller vs. callee side of the ABI adapter. --- src/aotcompile.cpp | 24 +++++++++--------- src/codegen.cpp | 57 ++++++++++++++++++++++++------------------- src/jitlayers.cpp | 8 +++--- src/jitlayers.h | 13 ++++------ src/julia_internal.h | 12 ++++++++- src/runtime_ccall.cpp | 10 +++++--- 6 files changed, 70 insertions(+), 54 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 0251e9db4b38b..72b68c14636be 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -549,16 +549,16 @@ Function *IRLinker_copyFunctionProto(Module *DstM, Function *SF) { return F; } -static Function *aot_abi_converter(jl_codegen_params_t ¶ms, Module *M, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Module *defM, StringRef func, StringRef specfunc, bool target_specsig) +static Function *aot_abi_converter(jl_codegen_params_t ¶ms, Module *M, jl_abi_t from_abi, jl_code_instance_t *codeinst, Module *defM, StringRef func, StringRef specfunc, bool target_specsig) { std::string gf_thunk_name; if (!specfunc.empty()) { Value *llvmtarget = IRLinker_copyFunctionProto(M, defM->getFunction(specfunc)); - gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + gf_thunk_name = emit_abi_converter(M, params, from_abi, codeinst, llvmtarget, target_specsig); } else { Value *llvmtarget = func.empty() ? nullptr : IRLinker_copyFunctionProto(M, defM->getFunction(func)); - gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget); + gf_thunk_name = emit_abi_dispatcher(M, params, from_abi, codeinst, llvmtarget); } auto F = M->getFunction(gf_thunk_name); assert(F); @@ -577,11 +577,11 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : params.cfuncs) { Module *M = cfunc.cfuncdata->getParent(); - jl_value_t *sigt = cfunc.sigt; + jl_value_t *sigt = cfunc.abi.sigt; JL_GC_PROMISE_ROOTED(sigt); - jl_value_t *declrt = cfunc.declrt; + jl_value_t *declrt = cfunc.abi.rt; JL_GC_PROMISE_ROOTED(declrt); - Function *unspec = aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, nullptr, nullptr, "", "", false); + Function *unspec = aot_abi_converter(params, M, cfunc.abi, nullptr, nullptr, "", "", false); jl_code_instance_t *codeinst = nullptr; auto assign_fptr = [¶ms, &cfunc, &codeinst, &unspec](Function *f) { ConstantArray *init = cast(cfunc.cfuncdata->getInitializer()); @@ -622,7 +622,7 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); } if (func == "jl_fptr_const_return") { - std::string gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst->rettype_const); + std::string gf_thunk_name = emit_abi_constreturn(M, params, cfunc.abi, codeinst->rettype_const); auto F = M->getFunction(gf_thunk_name); assert(F); assign_fptr(F); @@ -630,11 +630,11 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } else if (func == "jl_fptr_args") { assert(!specfunc.empty()); - if (!cfunc.specsig && jl_subtype(astrt, declrt)) { + if (!cfunc.abi.specsig && jl_subtype(astrt, declrt)) { assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); continue; } - assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, false)); + assign_fptr(aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, specfunc, false)); continue; } else if (func == "jl_fptr_sparam" || func == "jl_f_opaque_closure_call") { @@ -646,12 +646,12 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); continue; } - assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, true)); + assign_fptr(aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, specfunc, true)); continue; } } } - Function *f = codeinst ? aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, "", false) : unspec; + Function *f = codeinst ? aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, "", false) : unspec; return assign_fptr(f); } } @@ -2483,7 +2483,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ jl_compiled_functions_t compiled_functions; size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : output.cfuncs) { - jl_value_t *sigt = cfunc.sigt; + jl_value_t *sigt = cfunc.abi.sigt; JL_GC_PROMISE_ROOTED(sigt); jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); if (mi == nullptr) diff --git a/src/codegen.cpp b/src/codegen.cpp index cb556f3b07a2e..372939825c2b3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7065,7 +7065,7 @@ static void emit_specsig_to_specsig( emit_specsig_to_specsig(gf_thunk, returninfo.cc, returninfo.return_roots, calltype, rettype, is_for_opaque_closure, nargs, params, target, targetsig, targetrt, targetspec, rettype_const); } -std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig) +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *target, bool target_specsig) { // this builds a method that calls a method with the same arguments but a different specsig // build a specsig -> specsig converter thunk @@ -7073,36 +7073,35 @@ std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_ // build a args1 -> specsig converter thunk (gen_invoke_wrapper) // build a args1 -> args1 converter thunk (to add typeassert on result) bool needsparams = false; - bool is_opaque_closure = false; + bool target_is_opaque_closure = false; jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - std::string gf_thunk_name = get_function_name(specsig, needsparams, name_from_method_instance(mi), params.TargetTriple); + std::string gf_thunk_name = get_function_name(from_abi.specsig, needsparams, name_from_method_instance(mi), params.TargetTriple); gf_thunk_name += "_gfthunk"; if (target_specsig) { jl_value_t *abi = get_ci_abi(codeinst); - jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, is_opaque_closure); - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, target_is_opaque_closure); + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, &targetspec, nullptr); else - gen_invoke_wrapper(mi, abi, codeinst->rettype, declrt, targetspec, nargs, -1, is_opaque_closure, gf_thunk_name, M, params); + gen_invoke_wrapper(mi, abi, codeinst->rettype, from_abi.rt, targetspec, from_abi.nargs, -1, from_abi.is_opaque_closure, gf_thunk_name, M, params); } else { - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst->rettype, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, from_abi.rt, codeinst->rettype, params); } return gf_thunk_name; } -std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke) +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *invoke) { // this builds a method that calls a method with the same arguments but a different specsig // build a specsig -> args1 (apply_generic) or invoke (emit_tojlinvoke) call // build a args1 -> args1 call (emit_fptr1_wrapper) // build a args1 -> invoke call (emit_tojlinvoke) - bool is_opaque_closure = false; Value *target; if (!codeinst) target = prepare_call_in(M, jlapplygeneric_func); @@ -7114,33 +7113,40 @@ std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value else raw_string_ostream(gf_thunk_name) << "j_"; raw_string_ostream(gf_thunk_name) << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1) << "_gfthunk"; - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, - target, sigt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, + target, from_abi.sigt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, from_abi.rt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); return gf_thunk_name; } -std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const) +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_value_t *rettype_const) { - bool is_opaque_closure = false; std::string gf_thunk_name; raw_string_ostream(gf_thunk_name) << "jconst_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); - if (specsig) { - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, - nullptr, sigt, jl_typeof(rettype_const), nullptr, rettype_const); + if (from_abi.specsig) { + emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, + nullptr, from_abi.sigt, jl_typeof(rettype_const), nullptr, rettype_const); } else { - emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, declrt, jl_typeof(rettype_const), params); + emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, from_abi.rt, jl_typeof(rettype_const), params); } return gf_thunk_name; } std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst) { - jl_value_t *abi = get_ci_abi(codeinst); - return emit_abi_constreturn(M, params, codeinst->rettype, abi, specsig ? jl_nparams(abi) : 0, specsig, codeinst->rettype_const); + jl_value_t *sigt = get_ci_abi(codeinst); + jl_value_t *rt = codeinst->rettype; + + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; + + size_t nargs = specsig ? jl_nparams(sigt) : 0; + jl_abi_t abi = {sigt, rt, nargs, specsig, is_opaque_closure}; + + return emit_abi_constreturn(M, params, abi, codeinst->rettype_const); } // release jl_world_counter @@ -7196,7 +7202,8 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ cw->setAttributes(getcaller->getAttributes()); return cw; }); - ctx.emission_context.cfuncs.push_back({declrt, sigt, nargs, specsig, cfuncdata}); + jl_abi_t cfuncabi = {sigt, declrt, nargs, specsig, is_opaque_closure}; + ctx.emission_context.cfuncs.push_back({cfuncabi, cfuncdata}); if (specsig) { // TODO: could we force this to guarantee passing a box for `f` here (since we // know we had it here) and on the receiver end (emit_abi_converter / diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index b0a1338ab81dd..4d7f8e0480479 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -256,7 +256,7 @@ static void finish_params(Module *M, jl_codegen_params_t ¶ms, SmallVectorgetContext(), 0)); - gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + gf_thunk_name = emit_abi_converter(M, params, from_abi, codeinst, llvmtarget, target_specsig); } else if (invoke == jl_fptr_const_return_addr) { - gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, nargs, specsig, codeinst->rettype_const); + gf_thunk_name = emit_abi_constreturn(M, params, from_abi, codeinst->rettype_const); } else { Value *llvminvoke = invoke ? literal_static_pointer_val((void*)invoke, PointerType::get(M->getContext(), 0)) : nullptr; - gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvminvoke); + gf_thunk_name = emit_abi_dispatcher(M, params, from_abi, codeinst, llvminvoke); } SmallVector sharedmodules; finish_params(M, params, sharedmodules); diff --git a/src/jitlayers.h b/src/jitlayers.h index 7d0b6d9f02d6f..961adde887289 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -220,10 +220,7 @@ struct jl_codegen_call_target_t { // reification of a call to jl_jit_abi_convert, so that it isn't necessary to parse the Modules to recover this info struct cfunc_decl_t { - jl_value_t *declrt; - jl_value_t *sigt; - size_t nargs; - bool specsig; + jl_abi_t abi; llvm::GlobalVariable *cfuncdata; }; @@ -324,10 +321,10 @@ Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t * jl_codegen_params_t ¶ms); extern "C" JL_DLLEXPORT_CODEGEN -void *jl_jit_abi_convert(jl_task_t *ct, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); -std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke); -std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig); -std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const); +void *jl_jit_abi_convert(jl_task_t *ct, jl_abi_t from_abi, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *invoke); +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *target, bool target_specsig); +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_value_t *rettype_const); std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst); Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 7c73ac072251a..0ed452daa8b0a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -387,6 +387,16 @@ static inline void memassign_safe(int hasptr, char *dst, const jl_value_t *src, #define GC_OLD_MARKED (GC_OLD | GC_MARKED) // reachable and old #define GC_IN_IMAGE 4 +// data structures for runtime codegen +typedef struct _jl_abi_t { + jl_value_t *sigt; + jl_value_t *rt; + size_t nargs; + int specsig; // bool + // OpaqueClosure Methods override the first argument of their signature + int is_opaque_closure; +} jl_abi_t; + // useful constants extern jl_methtable_t *jl_method_table JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_method_t *jl_opaque_closure_method JL_GLOBALLY_ROOTED; @@ -1629,7 +1639,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals); JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data); -JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, +JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_abi_t from_abi, jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 0ef81357f8baf..3f61e9bceff2c 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -423,6 +423,8 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) return f; }; jl_callptr_t invoke = nullptr; + bool is_opaque_closure = false; + jl_abi_t from_abi = { sigt, declrt, nargs, specsig, is_opaque_closure }; if (codeinst != NULL) { jl_value_t *astrt = codeinst->rettype; if (astrt != (jl_value_t*)jl_bottom_type && @@ -436,23 +438,23 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &f, 1); if (invoke != nullptr) { if (invoke == jl_fptr_const_return_addr) { - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false)); + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false)); } else if (invoke == jl_fptr_args_addr) { assert(f); if (!specsig && jl_subtype(astrt, declrt)) return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, false)); + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, false)); } else if (specsigflags & 0b1) { assert(f); if (specsig && jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, true)); + return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, true)); } } } - f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false); + f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false); if (codeinst == nullptr) cfuncdata->unspecialized = f; return assign_fptr(f); From 66ec6ee4c79c2ad1e9dff7694a634c6fc3adbb4d Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Thu, 12 Jun 2025 15:28:55 +0200 Subject: [PATCH 403/662] Add compat note for OncePer types (#58716) Closes #58703 --- base/lock.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/base/lock.jl b/base/lock.jl index 5fca3a982f22d..461cb2b12a807 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -704,6 +704,9 @@ calls in the same process will return exactly the same value. This is useful in code that will be precompiled, as it allows setting up caches or other state which won't get serialized. +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest @@ -814,6 +817,9 @@ if that behavior is correct within your library's threading-safety design. See also: [`OncePerTask`](@ref). +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest @@ -942,6 +948,9 @@ exactly once per Task. All future calls in the same Task will return exactly the See also: [`task_local_storage`](@ref). +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest From 6121477efd40de0c38f2bc8db10f58662e7c8f4d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 12 Jun 2025 12:18:36 -0400 Subject: [PATCH 404/662] add support for storing just the inferred inlining_cost in CodeInstance (#58662) In preparation for giving us the option of not storing code that does not seem useful immediately, but which we previously kept around just in case inference or codegen (always_inline) ever needed to decide if it was worthwhile to inline later. And also for investigating issues like the recently closed #58449 to examine whether the code was removed for either heuristic or correctness reasons. --- Compiler/src/optimize.jl | 4 +++- src/codegen.cpp | 2 +- src/gc-stock.h | 9 -------- src/gf.c | 2 +- src/ircode.c | 46 +++++++++++++++++++++++++++++++++++----- src/julia.h | 7 ++++-- src/precompile_utils.c | 2 +- src/staticdata.c | 20 +++++++++++------ 8 files changed, 65 insertions(+), 27 deletions(-) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 3532b2043d76f..da4a17c5d6913 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -113,7 +113,9 @@ set_inlineable!(src::CodeInfo, val::Bool) = function inline_cost_clamp(x::Int) x > MAX_INLINE_COST && return MAX_INLINE_COST x < MIN_INLINE_COST && return MIN_INLINE_COST - return convert(InlineCostType, x) + x = ccall(:jl_encode_inlining_cost, UInt8, (InlineCostType,), x) + x = ccall(:jl_decode_inlining_cost, InlineCostType, (UInt8,), x) + return x end const SRC_FLAG_DECLARED_INLINE = 0x1 diff --git a/src/codegen.cpp b/src/codegen.cpp index 372939825c2b3..3ea65d46dcfb3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -9900,7 +9900,7 @@ void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t &pa src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_method_t *def = mi->def.method; - if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + if (src && jl_is_string((jl_value_t*)src) && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); if (src && jl_is_code_info(src) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) { jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints diff --git a/src/gc-stock.h b/src/gc-stock.h index d478ee1366da0..8e27893697f68 100644 --- a/src/gc-stock.h +++ b/src/gc-stock.h @@ -365,15 +365,6 @@ STATIC_INLINE jl_gc_pagemeta_t *pop_page_metadata_back(jl_gc_pagemeta_t **ppg) J return v; } -#ifdef __clang_gcanalyzer__ /* clang may not have __builtin_ffs */ -unsigned ffs_u32(uint32_t bitvec) JL_NOTSAFEPOINT; -#else -STATIC_INLINE unsigned ffs_u32(uint32_t bitvec) -{ - return __builtin_ffs(bitvec) - 1; -} -#endif - extern bigval_t *oldest_generation_of_bigvals; extern int64_t buffered_pages; extern int gc_first_tid; diff --git a/src/gf.c b/src/gf.c index 57dc2760eecac..afd0199a61518 100644 --- a/src/gf.c +++ b/src/gf.c @@ -357,7 +357,7 @@ static int emit_codeinst_and_edges(jl_code_instance_t *codeinst) JL_GC_PUSH1(&code); jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_method_t *def = mi->def.method; - if (jl_is_string(code) && jl_is_method(def)) + if (jl_is_method(def)) code = (jl_value_t*)jl_uncompress_ir(def, codeinst, (jl_value_t*)code); if (jl_is_code_info(code)) { jl_emit_codeinst_to_jit(codeinst, (jl_code_info_t*)code); diff --git a/src/ircode.c b/src/ircode.c index 9a94c4c62431a..65130e46edfe0 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -989,7 +989,7 @@ static int codelocs_nstmts(jl_string_t *cl) JL_NOTSAFEPOINT #define IR_DATASIZE_FLAGS sizeof(uint16_t) #define IR_DATASIZE_PURITY sizeof(uint16_t) -#define IR_DATASIZE_INLINING_COST sizeof(uint16_t) +#define IR_DATASIZE_INLINING_COST sizeof(uint8_t) #define IR_DATASIZE_NSLOTS sizeof(int32_t) typedef enum { ir_offset_flags = 0, @@ -1044,7 +1044,7 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) code->ssaflags); write_uint16(s.s, checked_size(flags.packed, IR_DATASIZE_FLAGS)); write_uint16(s.s, checked_size(code->purity.bits, IR_DATASIZE_PURITY)); - write_uint16(s.s, checked_size(code->inlining_cost, IR_DATASIZE_INLINING_COST)); + write_uint8(s.s, checked_size(jl_encode_inlining_cost(code->inlining_cost), IR_DATASIZE_INLINING_COST)); size_t nslots = jl_array_nrows(code->slotflags); assert(nslots >= m->nargs && nslots < INT32_MAX); // required by generated functions @@ -1109,6 +1109,8 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t { if (jl_is_code_info(data)) return (jl_code_info_t*)data; + if (!jl_is_string(data)) + return (jl_code_info_t*)jl_nothing; JL_TIMING(AST_UNCOMPRESS, AST_UNCOMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) assert(jl_is_method(m)); @@ -1139,7 +1141,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t code->nospecializeinfer = flags.bits.nospecializeinfer; code->isva = flags.bits.isva; code->purity.bits = read_uint16(s.s); - code->inlining_cost = read_uint16(s.s); + code->inlining_cost = jl_decode_inlining_cost(read_uint8(s.s)); size_t nslots = read_int32(s.s); code->slotflags = jl_alloc_array_1d(jl_array_uint8_type, nslots); @@ -1240,12 +1242,46 @@ JL_DLLEXPORT uint8_t jl_ir_flag_has_image_globalref(jl_string_t *data) return flags.bits.has_image_globalref; } -JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_string_t *data) +// create a compressed u16 value with range 0..3968, 3 bits exponent, 5 bits mantissa, implicit first digit, rounding up, full accuracy over 0..63 +JL_DLLEXPORT uint8_t jl_encode_inlining_cost(uint16_t inlining_cost) { + unsigned shift = 0; + unsigned mantissa; + if (inlining_cost <= 0x1f) { + mantissa = inlining_cost; + } + else { + while (inlining_cost >> 5 >> shift != 0) + shift++; + assert(1 <= shift && shift <= 11); + mantissa = (inlining_cost >> (shift - 1)) & 0x1f; + mantissa += (inlining_cost & ((1 << (shift - 1)) - 1)) != 0; // round up if trailing bits non-zero, overflowing into exp + } + unsigned r = (shift << 5) + mantissa; + if (r > 0xff) + r = 0xff; + return r; +} + +JL_DLLEXPORT uint16_t jl_decode_inlining_cost(uint8_t inlining_cost) +{ + unsigned shift = inlining_cost >> 5; + if (inlining_cost == 0xff) + return 0xffff; + else if (shift == 0) + return inlining_cost; + else + return (0x20 | (inlining_cost & 0x1f)) << (shift - 1); +} + +JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_value_t *data) +{ + if (jl_is_uint8(data)) + return jl_decode_inlining_cost(*(uint8_t*)data); if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining_cost; assert(jl_is_string(data)); - uint16_t res = jl_load_unaligned_i16(jl_string_data(data) + ir_offset_inlining_cost); + uint16_t res = jl_decode_inlining_cost(*(uint8_t*)(jl_string_data(data) + ir_offset_inlining_cost)); return res; } diff --git a/src/julia.h b/src/julia.h index 83c32aa5d85ec..8018928b8a154 100644 --- a/src/julia.h +++ b/src/julia.h @@ -438,10 +438,11 @@ typedef struct _jl_code_instance_t { jl_value_t *rettype_const; // inferred constant return value, or null // Inferred result. When part of the runtime cache, either - // - A jl_code_info_t (may be compressed) containing the inferred IR + // - A jl_code_info_t (may be compressed as a String) containing the inferred IR // - jl_nothing, indicating that inference was completed, but the result was // deleted to save space. - // - null, indicating that inference was not yet completed or did not succeed + // - UInt8, indicating that inference recorded the estimated inlining cost, but deleted the result to save space + // - NULL, indicating that inference was not yet completed or did not succeed _Atomic(jl_value_t *) inferred; _Atomic(jl_debuginfo_t *) debuginfo; // stored information about edges from this object (set once, with a happens-before both source and invoke) _Atomic(jl_svec_t *) edges; // forward edge info @@ -2311,6 +2312,8 @@ JL_DLLEXPORT jl_value_t *jl_uncompress_argname_n(jl_value_t *syms, size_t i); JL_DLLEXPORT struct jl_codeloc_t jl_uncompress1_codeloc(jl_value_t *cl, size_t pc) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_compress_codelocs(int32_t firstline, jl_value_t *codelocs, size_t nstmts); JL_DLLEXPORT jl_value_t *jl_uncompress_codelocs(jl_value_t *cl, size_t nstmts); +JL_DLLEXPORT uint8_t jl_encode_inlining_cost(uint16_t inlining_cost) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t jl_decode_inlining_cost(uint8_t inlining_cost) JL_NOTSAFEPOINT; JL_DLLEXPORT int jl_is_operator(const char *sym); JL_DLLEXPORT int jl_is_unary_operator(const char *sym); diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 86bb723443925..491f111ac4746 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -208,7 +208,7 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); if (inferred && (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL || inferred == jl_nothing || - ((jl_is_string(inferred) || jl_is_code_info(inferred)) && jl_ir_inlining_cost(inferred) == UINT16_MAX))) { + ((jl_is_string(inferred) || jl_is_code_info(inferred) || jl_is_uint8(inferred)) && jl_ir_inlining_cost(inferred) == UINT16_MAX))) { do_compile = 1; } else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { diff --git a/src/staticdata.c b/src/staticdata.c index 3df8e7414420c..5c4bf99883eb3 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -911,30 +911,36 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred); - if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) + if (inferred && inferred != jl_nothing && !jl_is_uint8(inferred)) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) jl_method_t *def = mi->def.method; if (jl_is_method(def)) { // don't delete toplevel code int is_relocatable = !s->incremental || jl_is_code_info(inferred) || (jl_is_string(inferred) && jl_string_len(inferred) > 0 && jl_string_data(inferred)[jl_string_len(inferred) - 1]); + int discard = 0; if (!is_relocatable) { - inferred = jl_nothing; + discard = 1; } else if (def->source == NULL) { // don't delete code from optimized opaque closures that can't be reconstructed (and builtins) } else if (jl_atomic_load_relaxed(&ci->max_world) != ~(size_t)0 || // delete all code that cannot run jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant - inferred = jl_nothing; + discard = 1; } else if (native_functions && // don't delete any code if making a ji file (ci->owner == jl_nothing) && // don't delete code for external interpreters !effects_foldable(jl_atomic_load_relaxed(&ci->ipo_purity_bits)) && // don't delete code we may want for irinterp jl_ir_inlining_cost(inferred) == UINT16_MAX) { // don't delete inlineable code // delete the code now: if we thought it was worth keeping, it would have been converted to object code - inferred = jl_nothing; + discard = 1; } - if (inferred == jl_nothing) { - record_field_change((jl_value_t**)&ci->inferred, jl_nothing); + if (discard) { + // keep only the inlining cost, so inference can later decide if it is worth getting the source back + if (jl_is_string(inferred) || jl_is_code_info(inferred)) + inferred = jl_box_uint8(jl_encode_inlining_cost(jl_ir_inlining_cost(inferred))); + else + inferred = jl_nothing; + record_field_change((jl_value_t**)&ci->inferred, inferred); } else if (s->incremental && jl_is_string(inferred)) { // New roots for external methods @@ -2704,7 +2710,7 @@ static void strip_specializations_(jl_method_instance_t *mi) jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); while (codeinst) { jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); - if (inferred && inferred != jl_nothing) { + if (inferred && inferred != jl_nothing && !jl_is_uint8(inferred)) { if (jl_options.strip_ir) { record_field_change((jl_value_t**)&codeinst->inferred, jl_nothing); } From 468c057875cf23712cfb2c1f56468b9814726281 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Thu, 12 Jun 2025 18:44:56 -0300 Subject: [PATCH 405/662] GC Always Full Flag (#58713) This used to be an environment variable (`JULIA_GC_NO_GENERATIONAL`) that could only be enabled if Julia was built with `WITH_GC_DEBUG_ENV`. People benchmarking GC may want to disable generational behavior in non-debug builds as well. This PR introduces `gc-sweep-always-full` as a hidden command-line-option. It also moves the docstrings for `--hard-heap-limit` and `--heap-target-increment` into `opts_hidden`, as it should have been done in the original PR. --- base/options.jl | 1 + doc/src/manual/environment-variables.md | 11 ----------- src/gc-debug.c | 5 +---- src/gc-stock.c | 2 +- src/gc-stock.h | 3 --- src/jloptions.c | 26 +++++++++++++++++-------- src/jloptions.h | 1 + test/gc.jl | 9 +++++++++ 8 files changed, 31 insertions(+), 27 deletions(-) diff --git a/base/options.jl b/base/options.jl index e3865e076d5bb..bf36c3ba0527d 100644 --- a/base/options.jl +++ b/base/options.jl @@ -66,6 +66,7 @@ struct JLOptions trim::Int8 task_metrics::Int8 timeout_for_safepoint_straggler_s::Int16 + gc_sweep_always_full::Int8 end # This runs early in the sysimage != is not defined yet diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index ff505db1f11f2..8cbb86188febd 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -530,17 +530,6 @@ Allows you to enable or disable zones for a specific Julia run. For instance, setting the variable to `+GC,-INFERENCE` will enable the `GC` zones and disable the `INFERENCE` zones. See [Dynamically Enabling and Disabling Zones](@ref). -### [`JULIA_GC_NO_GENERATIONAL`](@id JULIA_GC_NO_GENERATIONAL) - -If set to anything besides `0`, then the Julia garbage collector never performs -"quick sweeps" of memory. - -!!! note - - This environment variable only has an effect if Julia was compiled with - garbage-collection debugging (that is, if `WITH_GC_DEBUG_ENV` is set to `1` - in the build configuration). - ### [`JULIA_GC_WAIT_FOR_DEBUGGER`](@id JULIA_GC_WAIT_FOR_DEBUGGER) If set to anything besides `0`, then the Julia garbage collector will wait for diff --git a/src/gc-debug.c b/src/gc-debug.c index 6e51064035b7b..fa588a2ca886a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -892,10 +892,7 @@ void gc_heuristics_summary( void jl_gc_debug_init(void) { #ifdef GC_DEBUG_ENV - char *env = getenv("JULIA_GC_NO_GENERATIONAL"); - if (env && strcmp(env, "0") != 0) - jl_gc_debug_env.always_full = 1; - env = getenv("JULIA_GC_WAIT_FOR_DEBUGGER"); + char *env = getenv("JULIA_GC_WAIT_FOR_DEBUGGER"); jl_gc_debug_env.wait_for_debugger = env && strcmp(env, "0") != 0; gc_debug_alloc_init(&jl_gc_debug_env.pool, "POOL"); gc_debug_alloc_init(&jl_gc_debug_env.other, "OTHER"); diff --git a/src/gc-stock.c b/src/gc-stock.c index d7491f8d6bcf1..278c53a685dc3 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3155,7 +3155,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS // If the live data outgrows the suggested max_total_memory // we keep going with minimum intervals and full gcs until // we either free some space or get an OOM error. - if (gc_sweep_always_full) { + if (jl_options.gc_sweep_always_full) { sweep_full = 1; gc_count_full_sweep_reason(FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL); } diff --git a/src/gc-stock.h b/src/gc-stock.h index 8e27893697f68..d9286ec2090bb 100644 --- a/src/gc-stock.h +++ b/src/gc-stock.h @@ -44,7 +44,6 @@ typedef struct { } jl_alloc_num_t; typedef struct { - int always_full; int wait_for_debugger; jl_alloc_num_t pool; jl_alloc_num_t other; @@ -647,14 +646,12 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_markqueue_t *mq, int off #ifdef GC_DEBUG_ENV JL_DLLEXPORT extern jl_gc_debug_env_t jl_gc_debug_env; -#define gc_sweep_always_full jl_gc_debug_env.always_full int jl_gc_debug_check_other(void); int gc_debug_check_pool(void); void jl_gc_debug_print(void); void gc_scrub_record_task(jl_task_t *ta) JL_NOTSAFEPOINT; void gc_scrub(void); #else -#define gc_sweep_always_full 0 static inline int jl_gc_debug_check_other(void) { return 0; diff --git a/src/jloptions.c b/src/jloptions.c index ca52d3c7f311c..0f0c6060bfc79 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -159,6 +159,7 @@ JL_DLLEXPORT void jl_init_options(void) JL_TRIM_NO, // trim 0, // task_metrics -1, // timeout_for_safepoint_straggler_s + 0, // gc_sweep_always_full }; jl_options_initialized = 1; } @@ -293,14 +294,6 @@ static const char opts[] = " number of bytes, optionally in units of: B, K (kibibytes),\n" " M (mebibytes), G (gibibytes), T (tebibytes), or % (percentage\n" " of physical memory).\n\n" - " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever go above this\n" - " limit, we will abort. The value may be specified as a\n" - " number of bytes, optionally in units of: B, K (kibibytes),\n" - " M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" - " --heap-target-increment=[] Set an upper bound on how much the heap target\n" - " can increase between consecutive collections. The value may be\n" - " specified as a number of bytes, optionally in units of: B,\n" - " K (kibibytes), M (mebibytes), G (gibibytes) or T (tebibytes).\n\n" ; static const char opts_hidden[] = @@ -344,6 +337,18 @@ static const char opts_hidden[] = " and can throw errors. With unsafe-warn warnings will be\n" " printed for dynamic call sites that might lead to such\n" " errors. In safe mode compile-time errors are given instead.\n" + " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever\n" + " go above this limit, we will abort. The value\n" + " may be specified as a number of bytes,\n" + " optionally in units of: B, K (kibibytes),\n" + " M (mebibytes), G (gibibytes) or T (tebibytes).\n" + " --heap-target-increment=[] Set an upper bound on how much the heap\n" + " target can increase between consecutive\n" + " collections. The value may be specified as\n" + " a number of bytes, optionally in units of:\n" + " B, K (kibibytes), M (mebibytes), G (gibibytes)\n" + " or T (tebibytes).\n" + " --gc-sweep-always-full Makes the GC always do a full sweep of the heap\n" ; JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) @@ -394,6 +399,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_heap_size_hint, opt_hard_heap_limit, opt_heap_target_increment, + opt_gc_sweep_always_full, opt_gc_threads, opt_permalloc_pkgimg, opt_trim, @@ -467,6 +473,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, { "hard-heap-limit", required_argument, 0, opt_hard_heap_limit }, { "heap-target-increment", required_argument, 0, opt_heap_target_increment }, + { "gc-sweep-always-full", no_argument, 0, opt_gc_sweep_always_full }, { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -1027,6 +1034,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: --timeout-for-safepoint-straggler=; seconds must be an integer between 1 and %d", INT16_MAX); jl_options.timeout_for_safepoint_straggler_s = (int16_t)timeout; break; + case opt_gc_sweep_always_full: + jl_options.gc_sweep_always_full = 1; + break; case opt_trim: if (optarg == NULL || !strcmp(optarg,"safe")) jl_options.trim = JL_TRIM_SAFE; diff --git a/src/jloptions.h b/src/jloptions.h index 35cc8c3e13375..2a1733a54d59a 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -70,6 +70,7 @@ typedef struct { int8_t trim; int8_t task_metrics; int16_t timeout_for_safepoint_straggler_s; + int8_t gc_sweep_always_full; } jl_options_t; #endif diff --git a/test/gc.jl b/test/gc.jl index 3e9f03ef40d92..7d4a3655a2438 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -82,6 +82,15 @@ end @testset "Full GC reasons" begin full_sweep_reasons_test() end + +@testset "GC Always Full" begin + prog = "using Test;\n + for _ in 1:10; GC.gc(); end;\n + reasons = Base.full_sweep_reasons();\n + @test reasons[:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL] >= 10;" + cmd = `$(Base.julia_cmd()) --depwarn=error --startup-file=no --gc-sweep-always-full -e $prog` + @test success(cmd) +end end @testset "Base.GC docstrings" begin From 7e8e9bb12e2ee3f60b5a5c97b15116784958f5d6 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 12 Jun 2025 19:53:39 -0400 Subject: [PATCH 406/662] CI: `PrAssignee.yml`: Don't post a comment (#58721) --- .github/workflows/PrAssignee.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml index 553a49351d924..074e7a5528ce8 100644 --- a/.github/workflows/PrAssignee.yml +++ b/.github/workflows/PrAssignee.yml @@ -202,16 +202,6 @@ jobs: // console.log('ERROR: Failed to add add prReviewLabel'); // } - // Post a comment - const commentBody = `Hello! I am a bot.\n\nThank you for your pull request!\n\nI have assigned \`@${selectedAssignee}\` to this pull request.\n\n\`@${selectedAssignee}\` can either choose to review this pull request themselves, or they can choose to find someone else to review this pull request.\n\nNote: If you are a Julia committer, please make sure that your organization membership is public.` - console.log('Attempting to post bot comment on the PR...'); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - body: commentBody, - }); - // Exit with error if any problems were encountered earlier if (weDidEncounterError) { const msg = 'ERROR: Encountered at least one problem while running the script'; From 9841acee7be5f11171221631d563f5161a1726b3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 12 Jun 2025 22:45:51 -0400 Subject: [PATCH 407/662] deps: PCRE2: Update to 10.45.0 (#58709) --- deps/checksums/pcre | 76 +++++++++++------------ deps/jlutilities/documenter/Manifest.toml | 2 +- deps/pcre.version | 2 +- stdlib/Manifest.toml | 2 +- stdlib/PCRE2_jll/Project.toml | 2 +- stdlib/PCRE2_jll/test/runtests.jl | 2 +- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/deps/checksums/pcre b/deps/checksums/pcre index 9e290c914baec..dd22c2b91576c 100644 --- a/deps/checksums/pcre +++ b/deps/checksums/pcre @@ -1,38 +1,38 @@ -PCRE2.v10.44.0+1.aarch64-apple-darwin.tar.gz/md5/14de26cfc0f6ff7635fac39e81e81a27 -PCRE2.v10.44.0+1.aarch64-apple-darwin.tar.gz/sha512/45079ecca5f4966a32895fcc63585f1dd60f306dc1cb5c098d42452fcff67f7f6b405c200a15747af4680151bb6a6374832a0119b8ddd743d2ed13d0beaef7c9 -PCRE2.v10.44.0+1.aarch64-linux-gnu.tar.gz/md5/3cf179ed36d37bff698ab81cf3d5797b -PCRE2.v10.44.0+1.aarch64-linux-gnu.tar.gz/sha512/db93e5a5c0c46b5536ed49515682d9bfe1d23f6ba8ae2468289ec8f2160140f39f5606a3c7095f45251f3663d8ccf2d6d7e5e8b1efb21c39bbf9a13b6ec60ef9 -PCRE2.v10.44.0+1.aarch64-linux-musl.tar.gz/md5/02baa415218f581a5ceeb7bf7fc0a090 -PCRE2.v10.44.0+1.aarch64-linux-musl.tar.gz/sha512/1685f37ed8f465ecc2f738fdf65d20bb1806934ff2c50194882282fb6c3900121c61c39210e4c0b89847493bfc3e15bb7b9136b0d968103b47c8662a78b412fe -PCRE2.v10.44.0+1.aarch64-unknown-freebsd.tar.gz/md5/4de065ea59ab4f622b46079df1d9d941 -PCRE2.v10.44.0+1.aarch64-unknown-freebsd.tar.gz/sha512/aa6df9edfb690d155a8b5a9390db7ca11622ac0020174cf070a33a075801bfe43bd4c80b8e28017989a8b7374d39897cdcf72ab0e1962e3e234239975f7ac0b4 -PCRE2.v10.44.0+1.armv6l-linux-gnueabihf.tar.gz/md5/f8a0907fbb20a06507fce849db098c4f -PCRE2.v10.44.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/3f5bcc1742380a31683a81740d55e198d7ec8d8ea5a13d6d0556d6603e4fadbf0dc648093c44e36dd6d3793c52a5e3dae6f2f459c73e3d3b5a005f3395d26772 -PCRE2.v10.44.0+1.armv6l-linux-musleabihf.tar.gz/md5/8854c24183441aa6fd21989c00888904 -PCRE2.v10.44.0+1.armv6l-linux-musleabihf.tar.gz/sha512/a74d9378f071dc4cb021e5171d66cd4ac5de3b348e993fc90d824ce5d2f554f7c8af7af55ec31d874d302aaba7d542b6505cc5963e53656c28026a06a53ed48b -PCRE2.v10.44.0+1.armv7l-linux-gnueabihf.tar.gz/md5/04960309ee7cf69a53e280878d5880ef -PCRE2.v10.44.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/a1644daf036daa3799368598427c87c23bcfdddac55a0d06adca08a2e9d617c893285855af562101b05129d0ed0d84d22f5a8a1703316ecd09aa1752b8330eef -PCRE2.v10.44.0+1.armv7l-linux-musleabihf.tar.gz/md5/1335defc6090be76c509840633f7cdfb -PCRE2.v10.44.0+1.armv7l-linux-musleabihf.tar.gz/sha512/9595052eeae4da413b930b14d7e89359a29220cd9e908325e0b7788c8f4a2feb2134e78a0d8f56007787f0fefadc9de31750db6104bbdd048fa50e1d785c2a8c -PCRE2.v10.44.0+1.i686-linux-gnu.tar.gz/md5/e2d6be1d19566c965c2afeb995aba52f -PCRE2.v10.44.0+1.i686-linux-gnu.tar.gz/sha512/4a9d981bb6aa9150b670db7c5d4d188c8391fcb2a16bc710ede7a84bf7ec546fc5fd9096a339720579d25b6dcb5674b2b5b28e9664e5ef589b1a5044ce38b6a7 -PCRE2.v10.44.0+1.i686-linux-musl.tar.gz/md5/23cf857bd3daea4f094fcec48a7712dc -PCRE2.v10.44.0+1.i686-linux-musl.tar.gz/sha512/534f0cfab0cd60db9498eff387f7280a8baaf893a98dd2e7a737e68ba6473ed8236e9da85116eefb9812ec5323c705a00fcaff010b1900f752de8bdff65ef3ad -PCRE2.v10.44.0+1.i686-w64-mingw32.tar.gz/md5/3d05764df2305f16e4ffab60031ad40c -PCRE2.v10.44.0+1.i686-w64-mingw32.tar.gz/sha512/3e21cc6b71849c1a361373de30567990dba13dfd8812e7a7b5e2734b572bf1d45aeb730289d329975e76932c4c40e476824be2ab8e80a40fb7a7e2f46159235a -PCRE2.v10.44.0+1.powerpc64le-linux-gnu.tar.gz/md5/596d7c29d1417ed8959ea3ae3b4df453 -PCRE2.v10.44.0+1.powerpc64le-linux-gnu.tar.gz/sha512/89e03bfd6890150e2c8dddc4e7d024f2e09421c25a3d0fef3b5cd7f6bab7d6402ec1e82b02ecb5d26d01dfa2fb6068d050513894c374b7f2244c8fcbf00d69e2 -PCRE2.v10.44.0+1.riscv64-linux-gnu.tar.gz/md5/8330a431f4da1d20cffdb64d2c270dfb -PCRE2.v10.44.0+1.riscv64-linux-gnu.tar.gz/sha512/a836d0b9feefd9ffd50cf29db72ab704e6ae442939322526e2a5613973eabc8e543c5546ce507b0c5f9e6f1ce324978aeb6e99f8833eb60fc90e74139e47c6d2 -PCRE2.v10.44.0+1.x86_64-apple-darwin.tar.gz/md5/18f13c78ff6388c601bd36788e526b31 -PCRE2.v10.44.0+1.x86_64-apple-darwin.tar.gz/sha512/7b43a289f54064fc3c292de98173ec91cde2e49402c99c7848cbdc0e6d90a23a86d41f521e3986fcc8d941ee070d09e29ddc89a4e23009b8e9333e577ae4a09c -PCRE2.v10.44.0+1.x86_64-linux-gnu.tar.gz/md5/9f45feca0955f81ceb898208b9c74e15 -PCRE2.v10.44.0+1.x86_64-linux-gnu.tar.gz/sha512/eac215838306f7b5adb2166c3f620a69ed52fbd752ef3673a887507963a826c305d9b078dbb5236dc9a45eaca0d34f77325aab41703745701a077c84822ec0d0 -PCRE2.v10.44.0+1.x86_64-linux-musl.tar.gz/md5/79f092c6e8e971027ac6c1f0987376fb -PCRE2.v10.44.0+1.x86_64-linux-musl.tar.gz/sha512/2c5655b0f719a7d442c89f1040f2973b03f8becd855a0cfd6c0a985a07b25de351a84e3b9daaebd952b62628db0d937de08a8d05ee4bcace7e72d6b5ce6b8435 -PCRE2.v10.44.0+1.x86_64-unknown-freebsd.tar.gz/md5/a0bc32a099a584d453458a76c892fe47 -PCRE2.v10.44.0+1.x86_64-unknown-freebsd.tar.gz/sha512/6649c1b9e9569a9decccf6ebaa61d44acdb9069208ec796777d8e70a908210f775be2142053f6a5762ebaa321e297f6d8b51db99629766bc702c498b5f772492 -PCRE2.v10.44.0+1.x86_64-w64-mingw32.tar.gz/md5/eeffb6164fba08b0d5c7f50afa081475 -PCRE2.v10.44.0+1.x86_64-w64-mingw32.tar.gz/sha512/f06db992a2070a88559c15224972aeb098d4291a4325970fc0fbbb7cdd539f4a2fd4f90c0de90a34fe454da6c38290f9e0c7fdf2fe8c441f687fe4491d652adc -pcre2-10.44.tar.bz2/md5/9d1fe11e2e919c7b395e3e8f0a5c3eec -pcre2-10.44.tar.bz2/sha512/ee91cc10a2962bc7818b03d368df3dd31f42ea9a7260ae51483ea8cd331b7431e36e63256b0adc213cc6d6741e7c90414fd420622308c0ae3fcb5dd878591be2 +PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/md5/896991347ac3198a2cc84a1c39b8bf2e +PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/sha512/92ad56e63f7efe6511092826a994cb5d1ac3cfb20f86db557042bb734df8786d9f2f592c7cfea695955494696274916bea5af90116fd549ece49dd142d2de5f5 +PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/md5/dceb9622c7845494635226627c5d9809 +PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/sha512/d2688bf19ccc7f0567e9d9926be1f95b3e5aa8c85800fc40aae6feda81eda4e59bbff03f8a8f08cd21ac14956c44e7b6077ef5b8eba61fb0fa3541144ba23f47 +PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/md5/5caf1b5a65e01ee25272b0125168c6aa +PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/sha512/e0bf9422cce467373df64cba82d411768a2bb7e1b03b28c16267f49a56cf259450b34a47ecfa18051ba12a6b1bf2126a28df18508c25e0e09a35040502bb95c8 +PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/md5/5e1fc216d3051103911b7958aeb9ca8f +PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/sha512/de892eae2c3f815f676cc05771a54e84c2f6fc9b8cf6994b6953ee19eec7160950c80a6a3ff2d9974c4d42a23fccd0d24cd0fc569c87263f3b9dd05e86a66527 +PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/md5/d00c7585e21c9c35288c9ad1ab86f6db +PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/4fba211d5cb526fc313d62b6eeaa49d5daa471f507a4649cb6331686b762c951217e54017ebc745214811ca7a26a621e850d0092f9e88f1bbf9e1dbb451776c9 +PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/md5/67e2cf530ea4f54065e8218da5735de5 +PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/sha512/5097fcf75be76e62f95f066c65a2e87f33bf8897cde9951889349dd2f0b62b609dafbe5f4b0039d3acfb79eb5a18606bc5f10e591ff0d8520c37e32b487bf7cf +PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/md5/87b587f70b5fea9a1623f836d810c73c +PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/07f00365f81d6e27dc3aab0ee9f5c47ec612a09e271baa9f703e40d6a5394e631c01cc171ced248f24dcf561d9ee95e54d879d4458db9809755887a799e0b97b +PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/md5/efdb80dd89b059f50e425e5788873422 +PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/sha512/9fcd05831e9581b8ef5c85bbf13d90ec11268a60b474237b3679c71d804176d7816750aa65b1d02652ab599814f61f3023fa4c57e4e0bc29a9667b928f906b0a +PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/md5/6d79e55c8e3461400eee5a280d1a7bf2 +PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/sha512/a500f6fbf3952497539b6dd1d3c01fe4ca713eec838de3162854183b559100077f43537021224121abbdd72836dd0ed599a7ef7d2728848bf32c5af344d8deaf +PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/md5/f820b790aba3c4dcbf0dd43c3570f94f +PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/sha512/b9daa0df01701b1c4b97bdeaab113226ee085f4af49298ca93f4bab29e94781c2cc10442342a0205f52f089937efe01900f4ed75360cb8174f70aeec48e864e9 +PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/md5/fd5bd53baab8ce75026722fc88a5f804 +PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/sha512/117bf87ced28c2c85e5a90e83af14bd75a314bcb0628a71a86d7103a8a098f43be168b6093bd6c0439eeec940ff7327e20a5cfcc57060b9a6383090b03e947b5 +PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/md5/b6d67f0a9710b6531ac5c5aa32559a24 +PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/sha512/a287b1ced53d8fac0f6970d140dcbac7677cde241382f4d33617d4b0370ab5b65d498218962891a46b6a9e80ed7b2bd361675d263b4606fdac336606cc80d36f +PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/md5/bd06447be24fe3b9eaa8258664684d6d +PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/sha512/a8c5aa1011daf811c9431122a7221e1ed98e1a593861f21f8ce06593428d6b024633b7fc1b31c89b06a69f03e0da7c99ea8c6abc4ab8c9c47ffef786fcab0de4 +PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/md5/f4cafdaaa4cbb224d35337fce37a5fa0 +PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/sha512/6f891e3fbcc0a3e9c906d645cf98cc15eb224addb31d1a31770e41aa5fd1d84b8979f8f8f76c9ed9b2a34e97e137f5404a7d3d49a1750c1e633cf9f03415b580 +PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/md5/26a732774abff1d4c301640f513f4e8c +PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/sha512/ee89bfedd7a79dbe85c83460bb0b3834606af3f950d26223676733417ee4028d9b5a9fe3f2851f42c55b3a4c800ee48e25afdf7ae7bdbe7df021c38cffa7c7fc +PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/md5/e4b0e47adc80128317697edab7312130 +PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/sha512/521cfe714c1ff1604e86b24935060b12fbb3ab62af9b0c59121f0c26627bc24f5d612f4758739e068f3f7974c3890043c55c6f28e500c9de6e5550705c45c982 +PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/md5/c6655a1f349c51ac898878836b72fa1a +PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/sha512/69b48b9b6872f0c6b61a8a3ae81ba103fb49887c392497e5a387de888b35fc45c716b99c972670ee5edf0255db361051f265e30ea8f509b790ab9edd50bcb6db +PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/md5/cb5c7a9a56e1f2209b2e227c598336c4 +PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/sha512/ff5a982d90089316ccc1e6a9dc2c17cc61b348df0db16aaa4bb7cf1787701acf31a65ffcf51be48abc8cb4436afe94984086fead84580b3de3509d111ba71992 +pcre2-10.45.tar.bz2/md5/f71abbe1b5adf25cd9af5d26ef223b66 +pcre2-10.45.tar.bz2/sha512/4c1f0cf793624516d7eeb15745d6c07c9f678dd2c2b349062c6b614e88bf42262972d133576e85140dee2a882984aaf2d688953fc9c69ec7105b2daaeae89845 diff --git a/deps/jlutilities/documenter/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml index e531512abef78..3b8acce93335b 100644 --- a/deps/jlutilities/documenter/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -184,7 +184,7 @@ version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] diff --git a/deps/pcre.version b/deps/pcre.version index 78245a5777a0c..6810ee36fe83f 100644 --- a/deps/pcre.version +++ b/deps/pcre.version @@ -3,4 +3,4 @@ PCRE_JLL_NAME := PCRE2 ## source build -PCRE_VER := 10.44 +PCRE_VER := 10.45 diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 54eb1ecd309ab..10ee965bbc1ea 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -173,7 +173,7 @@ version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] diff --git a/stdlib/PCRE2_jll/Project.toml b/stdlib/PCRE2_jll/Project.toml index 24ac196a3b8a9..1c6187e06be31 100644 --- a/stdlib/PCRE2_jll/Project.toml +++ b/stdlib/PCRE2_jll/Project.toml @@ -1,6 +1,6 @@ name = "PCRE2_jll" uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/PCRE2_jll/test/runtests.jl b/stdlib/PCRE2_jll/test/runtests.jl index 21df2ec430e0e..9040eb5b76af0 100644 --- a/stdlib/PCRE2_jll/test/runtests.jl +++ b/stdlib/PCRE2_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, PCRE2_jll vstr = zeros(UInt8, 32) @test ccall((:pcre2_config_8, libpcre2_8), Cint, (UInt32, Ref{UInt8}), 11, vstr) > 0 vn = VersionNumber(split(unsafe_string(pointer(vstr)), " ")[1]) - @test vn == v"10.44.0" + @test vn == v"10.45.0" end From 866c89014ffde70fd81699497275a5fb21b717c5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 13 Jun 2025 01:09:55 -0400 Subject: [PATCH 408/662] UnicodeData: Update to version 16 (#58710) --- deps/checksums/UnicodeData-13.0.0.txt/md5 | 1 - deps/checksums/UnicodeData-13.0.0.txt/sha512 | 1 - deps/checksums/UnicodeData-16.0.0.txt/md5 | 1 + deps/checksums/UnicodeData-16.0.0.txt/sha512 | 1 + doc/Makefile | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/UnicodeData-13.0.0.txt/md5 delete mode 100644 deps/checksums/UnicodeData-13.0.0.txt/sha512 create mode 100644 deps/checksums/UnicodeData-16.0.0.txt/md5 create mode 100644 deps/checksums/UnicodeData-16.0.0.txt/sha512 diff --git a/deps/checksums/UnicodeData-13.0.0.txt/md5 b/deps/checksums/UnicodeData-13.0.0.txt/md5 deleted file mode 100644 index 2b3ffc179ce01..0000000000000 --- a/deps/checksums/UnicodeData-13.0.0.txt/md5 +++ /dev/null @@ -1 +0,0 @@ -85879f1976cc8eb739ee5585a93938e2 diff --git a/deps/checksums/UnicodeData-13.0.0.txt/sha512 b/deps/checksums/UnicodeData-13.0.0.txt/sha512 deleted file mode 100644 index a93ba01e7ddda..0000000000000 --- a/deps/checksums/UnicodeData-13.0.0.txt/sha512 +++ /dev/null @@ -1 +0,0 @@ -1a4a662e2ab33469976bf5f91aa6933ed9b73f6d4179a2daffb349e1869d7d6cfa885b164e82d15dcdad7458cd451c81add58d875eb0c70de854589dc97b2055 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/md5 b/deps/checksums/UnicodeData-16.0.0.txt/md5 new file mode 100644 index 0000000000000..79ae5d27eff0e --- /dev/null +++ b/deps/checksums/UnicodeData-16.0.0.txt/md5 @@ -0,0 +1 @@ +f50a0495d2000b7d6dd979cb40e00ba2 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/sha512 b/deps/checksums/UnicodeData-16.0.0.txt/sha512 new file mode 100644 index 0000000000000..05b998b3724ba --- /dev/null +++ b/deps/checksums/UnicodeData-16.0.0.txt/sha512 @@ -0,0 +1 @@ +963e5a1e7a480873c6e66d53e9288232b5029942477a694a0bfafa7e994c55189cb9c2f8d00255de84b82b72ff6066932e5531e3664fb422eeef9c69ea25d80e diff --git a/doc/Makefile b/doc/Makefile index d076d9c01a31e..177eeac5e6a85 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -25,7 +25,7 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ texplatform=$(texplatform) revise=$(revise) stdlibdir=$(call cygpath_w,$(build_datarootdir)/julia/stdlib/$(VERSDIR)/) -UNICODE_DATA_VERSION=13.0.0 +UNICODE_DATA_VERSION=16.0.0 $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt: @mkdir -p "$(SRCCACHE)" $(JLDOWNLOAD) "$@" https://www.unicode.org/Public/$(UNICODE_DATA_VERSION)/ucd/UnicodeData.txt From 8e88820227f60b85cb686b35411013dc20fadb01 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Jun 2025 12:18:30 +0200 Subject: [PATCH 409/662] test/spawn: download busybox to a filename ending with .exe on Windows (#58646) --- test/spawn.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spawn.jl b/test/spawn.jl index bfb7c9a83ffb6..b20200d5ffa2b 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -25,7 +25,7 @@ busybox_hash_correct(file) = bytes2hex(open(SHA.sha256, file)) == "ed2f95da95552 function _tryonce_download_from_cache(desired_url::AbstractString) cache_url = "https://cache.julialang.org/$(desired_url)" - cache_output_filename = joinpath(mktempdir(), "busybox") + cache_output_filename = joinpath(mktempdir(), "busybox" * (Sys.iswindows() ? ".exe" : "")) cache_response = Downloads.request( cache_url; output = cache_output_filename, From ab49eb163ea63b3db35e36e557bbd6ff4c505654 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 13 Jun 2025 08:19:38 -0400 Subject: [PATCH 410/662] [NFC] codegen: Move target ABI calculation into `jl_jit_abi_converter` (#58718) Makes the API more directly reflect the operation here, which is essentially "Give me a specptr with ABI `abi` that invokes CodeInstance `codeinst`" This relieves the caller of the responsibility to peek into the CodeInstance and pull out all of the correct bits to determine the appropriate callee ABI. --- src/jitlayers.cpp | 42 +++++++++++++++++++++++++++++++++---- src/julia_internal.h | 3 +-- src/runtime_ccall.cpp | 49 ++++++++++++++----------------------------- 3 files changed, 55 insertions(+), 39 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 4d7f8e0480479..e8bb1f995c140 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -255,12 +255,46 @@ static void finish_params(Module *M, jl_codegen_params_t ¶ms, SmallVectorrettype, from_abi.rt)) + return specptr; // no adapter required + + target = specptr; + target_specsig = false; + } + else if (specsigflags & 0b1) { + assert(specptr != nullptr); + if (from_abi.specsig && jl_egal(mi->specTypes, from_abi.sigt) && jl_egal(codeinst->rettype, from_abi.rt)) + return specptr; // no adapter required + + target = specptr; + target_specsig = true; + } + } + } + orc::ThreadSafeModule result_m; std::string gf_thunk_name; { diff --git a/src/julia_internal.h b/src/julia_internal.h index 0ed452daa8b0a..60a3b99c18afc 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1639,8 +1639,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals); JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data); -JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_abi_t from_abi, - jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig); +JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst); // Special filenames used to refer to internal julia libraries diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 3f61e9bceff2c..73f4a4b5d4e3d 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -422,42 +422,25 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) JL_UNLOCK(&cfun_lock); return f; }; - jl_callptr_t invoke = nullptr; bool is_opaque_closure = false; jl_abi_t from_abi = { sigt, declrt, nargs, specsig, is_opaque_closure }; - if (codeinst != NULL) { - jl_value_t *astrt = codeinst->rettype; - if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function never returns since it is - // occasionally required by the C API (typically error callbacks) - // even though we're likely to encounter memory errors in that case - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); - } - uint8_t specsigflags; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &f, 1); - if (invoke != nullptr) { - if (invoke == jl_fptr_const_return_addr) { - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false)); - } - else if (invoke == jl_fptr_args_addr) { - assert(f); - if (!specsig && jl_subtype(astrt, declrt)) - return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, false)); - } - else if (specsigflags & 0b1) { - assert(f); - if (specsig && jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) - return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, true)); - } - } + if (codeinst == nullptr) { + // Generate an adapter to a dynamic dispatch + if (cfuncdata->unspecialized == nullptr) + cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, nullptr); + + return assign_fptr(cfuncdata->unspecialized); + } + + jl_value_t *astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is + // occasionally required by the C API (typically error callbacks) + // even though we're likely to encounter memory errors in that case + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); } - f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false); - if (codeinst == nullptr) - cfuncdata->unspecialized = f; - return assign_fptr(f); + return assign_fptr(jl_jit_abi_converter(ct, from_abi, codeinst)); } void jl_init_runtime_ccall(void) From b636acbce59b38c10de7be2f76776886070714be Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 13 Jun 2025 21:12:40 -0400 Subject: [PATCH 411/662] curl: Update to 8.14.1 (#58708) --- deps/checksums/curl | 76 +++++++++++------------ deps/curl.version | 2 +- deps/jlutilities/documenter/Manifest.toml | 2 +- stdlib/LibCURL_jll/Project.toml | 2 +- stdlib/Manifest.toml | 2 +- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/deps/checksums/curl b/deps/checksums/curl index d85f7796bb0a9..04938f1bfd37e 100644 --- a/deps/checksums/curl +++ b/deps/checksums/curl @@ -1,38 +1,38 @@ -LibCURL.v8.12.1+1.aarch64-apple-darwin.tar.gz/md5/a6a5a1a360d210e3e03f1815efc97260 -LibCURL.v8.12.1+1.aarch64-apple-darwin.tar.gz/sha512/8493cc358fff3123fa6a8e6bc42a72c5219a259d568a2e1ff3c216c7bbd4c10a4057bfe2d2f7c571532d51ea876450c97c0f03f901ede4c7d668916265631f80 -LibCURL.v8.12.1+1.aarch64-linux-gnu.tar.gz/md5/7ebe2338198069b12716df084cd07d9b -LibCURL.v8.12.1+1.aarch64-linux-gnu.tar.gz/sha512/4b4ca81dcf0098ad8ce4f9190c559dba9d92ef330e0d18c50356e13280a6c71dfdaaed0de762b216380af22338d762aba4cd7039e95f3ee84b2bb02d55dbb49d -LibCURL.v8.12.1+1.aarch64-linux-musl.tar.gz/md5/9ff411aa7296780c3bbd2369597c898b -LibCURL.v8.12.1+1.aarch64-linux-musl.tar.gz/sha512/d3a4ec96254f5a8816d92492ea1dff9f6d3e3dbb5c6db06472dac41b38a7671c84959e934676857262751e89ae9d0e17ba567f8e4fab3a190c9a0f4393c190aa -LibCURL.v8.12.1+1.aarch64-unknown-freebsd.tar.gz/md5/7baca39040ac317f18b48d851beadf3b -LibCURL.v8.12.1+1.aarch64-unknown-freebsd.tar.gz/sha512/23158572ecb7378e92fd60977f73c9bca31a95683be3c820f0e2f60b2289125e726e8066f7b32e5159ddb4c76400774d41c299e6c8db5468cc6e3ba98ca3c1eb -LibCURL.v8.12.1+1.armv6l-linux-gnueabihf.tar.gz/md5/3a3f0ce770e66ec4afa313ccd747bd8f -LibCURL.v8.12.1+1.armv6l-linux-gnueabihf.tar.gz/sha512/68d62659448e79add970982065cbcf0f18c82c52ed837d83b954ac241268a82624532f0f291a8bafe0aafb4a700f180ad2c57c02dc2edc2c00da626befd0da92 -LibCURL.v8.12.1+1.armv6l-linux-musleabihf.tar.gz/md5/d7d77c98169e0c8067215c4f063df040 -LibCURL.v8.12.1+1.armv6l-linux-musleabihf.tar.gz/sha512/ff53a1cc55f6d1eac8688ca9c7dddf19949157fdde8a36221747b2e97b0bce2595862213dc35cdae211ead6b27372599c1eef1ebdfd4161f4b8b7be23f6534b3 -LibCURL.v8.12.1+1.armv7l-linux-gnueabihf.tar.gz/md5/2226abe022064c9f7464297a0a19f334 -LibCURL.v8.12.1+1.armv7l-linux-gnueabihf.tar.gz/sha512/f49feb8e27b732fed0f7ae5c8e00d6787f93ce450e063cec62300f6f42e6ecd6b59a2fa148e1d9fd9ff142689fc1f8fca8c9df21baea95301859b8177cffd7d6 -LibCURL.v8.12.1+1.armv7l-linux-musleabihf.tar.gz/md5/9d82cf23338ce84dcc03201f73ecc300 -LibCURL.v8.12.1+1.armv7l-linux-musleabihf.tar.gz/sha512/302fba7ac037d7ba2d849fc6d5da0877f5876871453837afcccc6e59c944f7dbafe0c271ec55062437578bb572bc0fe34a2e7a2599250c85d42f82dfb6d81d7b -LibCURL.v8.12.1+1.i686-linux-gnu.tar.gz/md5/a6bd8eaca6d45b0ceacfeefa3dbb6dbc -LibCURL.v8.12.1+1.i686-linux-gnu.tar.gz/sha512/dbbe92fd917e6388fa0e9454c19a71de842c349fc80fb530f2ead6c1e9b6af46d56723a0d8d4f451dd88e8bebc0d06be06eca90afc4bff5c3a23c84fbf36acb1 -LibCURL.v8.12.1+1.i686-linux-musl.tar.gz/md5/895552ab1aa470d87534a239b4f62aa1 -LibCURL.v8.12.1+1.i686-linux-musl.tar.gz/sha512/af05a032cec927adba166093af876fde691f15ff89c3e1978e4aa07f3589aeca495620fae201e8067bf17d776aaa5985ae4893e2875b52d6fb7e412e12b11a59 -LibCURL.v8.12.1+1.i686-w64-mingw32.tar.gz/md5/c10aa57b3fe2c8d6591813cb609908e1 -LibCURL.v8.12.1+1.i686-w64-mingw32.tar.gz/sha512/dcb943a2a405d6989aa90d8de3d3a3c259c12932be7707915d49e2016ea6c66b58e2b4d505bb171ffd6f59b1524a2d34c676c3fe61117082179c309bf64be61d -LibCURL.v8.12.1+1.powerpc64le-linux-gnu.tar.gz/md5/8233682e6d1a00449cb15f41434dfe04 -LibCURL.v8.12.1+1.powerpc64le-linux-gnu.tar.gz/sha512/3bb32f53ae2188b9373ee364394a73c497138ac986bd642e393ff7140adc533adc518e489a1b5255dafed7d16ec3e7c06bd4825e27ee473ffdbe2a0f28562586 -LibCURL.v8.12.1+1.riscv64-linux-gnu.tar.gz/md5/4d49a478301a8ede00dcced9d94775b4 -LibCURL.v8.12.1+1.riscv64-linux-gnu.tar.gz/sha512/69dade7cf6b628cd33571ca07ce22c7394f25b472a2d4d2df20a4f0c13e84c9e1bf53600209444b900cb401ee8ff2649023350c7bf9f14bc78bf4aec8ddad9cb -LibCURL.v8.12.1+1.x86_64-apple-darwin.tar.gz/md5/e6de55f6cdef86653ac96c2d884aea28 -LibCURL.v8.12.1+1.x86_64-apple-darwin.tar.gz/sha512/0193b54b153d9673c182fd534cef7abbf03987073aacec55d1852dc29d696ab3005b7619b9c1abb6f8879f5856b66df7ca7841908c7372684f8cc9b32c311473 -LibCURL.v8.12.1+1.x86_64-linux-gnu.tar.gz/md5/c6f439d4e0b92ee342d2badf009d9e8e -LibCURL.v8.12.1+1.x86_64-linux-gnu.tar.gz/sha512/cc9415d69732f9174d64f7903a3fc43a270ad007016adf80e10314182fd8147dcf4eb7cd0df907a2b9ff9cc82016ed905d700c66fd6a8b8089b8e27106d9ccb1 -LibCURL.v8.12.1+1.x86_64-linux-musl.tar.gz/md5/cb3ea4385e3799004266ea77d02632d3 -LibCURL.v8.12.1+1.x86_64-linux-musl.tar.gz/sha512/a52c7f6f5a4c9d9423e1e0e9bf563791f369bd979e5dce22b0de32d4f339dd42ab2f3f21cd0c933d4c7841fb7b244a56849936136841c2f5d7995b3a92a811de -LibCURL.v8.12.1+1.x86_64-unknown-freebsd.tar.gz/md5/0532f49c6b32859cf39ab3bcb5646d02 -LibCURL.v8.12.1+1.x86_64-unknown-freebsd.tar.gz/sha512/dea1a8de19cf74b8f95811397b8d615f64610b00678ad93482db765df0547c080de03211cb9e50eeaf1bab425119fe170ee70bde1ca58c185e2cde34fbd1d945 -LibCURL.v8.12.1+1.x86_64-w64-mingw32.tar.gz/md5/311dc58c0e5969f2d547a129e86f97ed -LibCURL.v8.12.1+1.x86_64-w64-mingw32.tar.gz/sha512/caa39cb4da20de98df2c28a2038763e332729894c1cbef3cb33df12351294559f5cb0c7f405ffec2bfb9bc27bcb4c3400916745f4ac09c0c94be6b9775f47021 -curl-8.12.1.tar.bz2/md5/c53f2409d83c7afa138cae7efed5ac4a -curl-8.12.1.tar.bz2/sha512/6a30f6eda94c03d58a287b4bfa0af39faddb13a3c3d1f86bb874c3e7147d76ce1dc7323741e28ccd8dc48b59a19d444c3b92512f80b3b8a4cf209d5b89109a4e +LibCURL.v8.14.1+1.aarch64-apple-darwin.tar.gz/md5/77465587a033e919ce347c9df3909b23 +LibCURL.v8.14.1+1.aarch64-apple-darwin.tar.gz/sha512/04a6bb04f6fc190423349787a1b345923f52b140099a3c62fdd077b3145a22a20ea06808c488712d686bc6d51fe86caea3b13cd9acde36822fec7249905ff16d +LibCURL.v8.14.1+1.aarch64-linux-gnu.tar.gz/md5/a5b78f4d06821de8c57abba8fbfc1c23 +LibCURL.v8.14.1+1.aarch64-linux-gnu.tar.gz/sha512/a09f94bd8556d63b7850b8ac29f8272c7437cc32c8440200d7044a4eb89bba8752d74ad26d3da0433ab07565dfefb1edb058ef13a1949546994321ea88e9e515 +LibCURL.v8.14.1+1.aarch64-linux-musl.tar.gz/md5/09e1b11301296cddb9039e0247ba0cde +LibCURL.v8.14.1+1.aarch64-linux-musl.tar.gz/sha512/380159eec2b638a190d284a02fdb6c60e925f4f0268ea4e2ae7272fffe19bfff5de93d9a71f3830147ece69417bbd2f7faca229cb1a4b4df4f1828e3f8a0ff53 +LibCURL.v8.14.1+1.aarch64-unknown-freebsd.tar.gz/md5/f872ba79808c0b24dfee579bf2fc7229 +LibCURL.v8.14.1+1.aarch64-unknown-freebsd.tar.gz/sha512/aa6ac50c0c3eaed768f228b2a4d320d44afe1bd34ee81273f976ca4027ccbafe996c2e349eab87718e962da0a18afa8b3c5d297e913d46ba09a8d68bba7dabf1 +LibCURL.v8.14.1+1.armv6l-linux-gnueabihf.tar.gz/md5/4e28def7a2d3bffc8d2adddbe670dd04 +LibCURL.v8.14.1+1.armv6l-linux-gnueabihf.tar.gz/sha512/444df0655e0d85cd35e20013e79787af17138dc16a292bf4557ebbd049c8f855449b9aa78eb45b0a00255e8320a4ad2b44a08c6161b7155467c9da95e8bb0a7d +LibCURL.v8.14.1+1.armv6l-linux-musleabihf.tar.gz/md5/4c037de41ff577fe217792e5d7c7f9a0 +LibCURL.v8.14.1+1.armv6l-linux-musleabihf.tar.gz/sha512/65122b75188cc7dfb8dd97c27398f614d2e3149c73e95bd9c60cdad2eb1c95a3c3dd7558a02a5d24b0e129c8c3a315efe45dd49f5f219a525dbeea0b4ab9b93f +LibCURL.v8.14.1+1.armv7l-linux-gnueabihf.tar.gz/md5/f7dedafd8eb09c7bf9cbac4c0adae37a +LibCURL.v8.14.1+1.armv7l-linux-gnueabihf.tar.gz/sha512/7c8187ce79de364488b7d8606638f7cfc1094f9a1d5a26dc71dc5a1a6135834641fe459b08f71999556d5ff2dfd1820ff3f1371c2b5fa94b8ea5985744044527 +LibCURL.v8.14.1+1.armv7l-linux-musleabihf.tar.gz/md5/6ede02ecc42e088e60d5a6c8a0181fd9 +LibCURL.v8.14.1+1.armv7l-linux-musleabihf.tar.gz/sha512/e3ea55dd8035506b5c2cc11ea2dbf30533f3fdef017decdca35fafe19a09f9b945f821f722fa60651fb083204a8d0aab47bb2024544b4d81b5b647b7bf260e01 +LibCURL.v8.14.1+1.i686-linux-gnu.tar.gz/md5/b823b709fd3a26a2861962b8f538718b +LibCURL.v8.14.1+1.i686-linux-gnu.tar.gz/sha512/4d94062910ed94cdf013ba3ce546231e733a780078f106a42937e26400de65634991df440e27865368b4f1f395743629cfd88dd62dd27b90654f4766c0a5f468 +LibCURL.v8.14.1+1.i686-linux-musl.tar.gz/md5/a80641db6cc9731342c08e85835481cd +LibCURL.v8.14.1+1.i686-linux-musl.tar.gz/sha512/9983e83469b38f17645b4cfdf9c0005684fb35a29bf97b1be8c2a34bdeb1115cf029353811eef7a4c96c2c7da3e0c4e1ba8fa7c589b8452675232544f591cf7d +LibCURL.v8.14.1+1.i686-w64-mingw32.tar.gz/md5/f1703e82d19e3a9c9fe5068532a2f98e +LibCURL.v8.14.1+1.i686-w64-mingw32.tar.gz/sha512/03a98243ea31f04a05ad94b0ce378830ee486d313bc33e652f44fd11fa812d3bff31556484c66a1fe467815affb5f191e7e8212996ac0a7b89b46b7743db58e1 +LibCURL.v8.14.1+1.powerpc64le-linux-gnu.tar.gz/md5/324982f4b780aba414384c369580ec85 +LibCURL.v8.14.1+1.powerpc64le-linux-gnu.tar.gz/sha512/b6c0d05ce457e8d1c97098891982921c4282a1a9f2984ed82735fc4251cf12d230826d7e96bcbbaf073ae6cfa9357adfd51341abb937db3ffca57156eac2a0eb +LibCURL.v8.14.1+1.riscv64-linux-gnu.tar.gz/md5/363f5480fd9d59c232ae30e799e96090 +LibCURL.v8.14.1+1.riscv64-linux-gnu.tar.gz/sha512/3a821895af3d6ed2363a39057f691bec2f139c196fe430c8d4f6b6404ecdc4d5d563e02c451dea8304fe403b7c1b3b2fe5fb8a7376db594037e31f29bb07cfb6 +LibCURL.v8.14.1+1.x86_64-apple-darwin.tar.gz/md5/e369db8d77ea5234de6f2efad75c856a +LibCURL.v8.14.1+1.x86_64-apple-darwin.tar.gz/sha512/5de40ee8f893d36a91d547c70ecbb396e4a3bef63eaaa7dab37a9374ce20651a3b51b2a415d7803183a76f5828bb39e3df428e6f4bd913e735226bd7732e0f23 +LibCURL.v8.14.1+1.x86_64-linux-gnu.tar.gz/md5/7af620f052ca833f495f8a770685877d +LibCURL.v8.14.1+1.x86_64-linux-gnu.tar.gz/sha512/443dccf17f1946e12b53a50b1a65494b1101a7d61853f9357df49891c36e49eae6f309a49f860534bd359d1e258c43fdd3e1d1e2e81a00084ddd6a6a47ae11f0 +LibCURL.v8.14.1+1.x86_64-linux-musl.tar.gz/md5/84bd751a151c3eb27d64a49f3d6a86ca +LibCURL.v8.14.1+1.x86_64-linux-musl.tar.gz/sha512/2c2b52bb5eaf980acd1ae0db7b8b2c0b7f88dbd9f4e2dde7e2f06052492479fafe7ac17e54e255dccf7f9ef5b4ef095ad54a4ae614a670d2efad713cac83bbb5 +LibCURL.v8.14.1+1.x86_64-unknown-freebsd.tar.gz/md5/0575d2d4b334bc5bae99c04b6534cff2 +LibCURL.v8.14.1+1.x86_64-unknown-freebsd.tar.gz/sha512/3aed29b4a762d300247a9a5a6b90393c8d8d41c895a334060a296983c8d8fdb47a2c4f59f8f3467aacb88f2034b0969b5bfabace97a619e26f5194d9f8baba4d +LibCURL.v8.14.1+1.x86_64-w64-mingw32.tar.gz/md5/03391a09da53b96531f95f764ebf5a49 +LibCURL.v8.14.1+1.x86_64-w64-mingw32.tar.gz/sha512/231f76cdf61cda36e4aa94bb8d4a5cb2607bde11cad172e516dd8c8689f36628eb01629c36bfc89be4c578866723d7e42257027ebb5be8dc16edc3d242deb86d +curl-8.14.1.tar.bz2/md5/bf683fcf55bed4c5bf66ca57ab99a96b +curl-8.14.1.tar.bz2/sha512/95fd0fffb354e60bc2a7251ce53122dc41d5b0aae00863654cae65f2e9698f02ff67d60b367f1a9dd556bfa77e3bac1e5e985efe00aaecfb99e1ea68148f3344 diff --git a/deps/curl.version b/deps/curl.version index 0a435fca5b7da..e7a22071c7566 100644 --- a/deps/curl.version +++ b/deps/curl.version @@ -3,4 +3,4 @@ CURL_JLL_NAME := LibCURL ## source build -CURL_VER := 8.12.1 +CURL_VER := 8.14.1 diff --git a/deps/jlutilities/documenter/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml index 3b8acce93335b..594ef1b3d17f9 100644 --- a/deps/jlutilities/documenter/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -116,7 +116,7 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.14.1+1" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] diff --git a/stdlib/LibCURL_jll/Project.toml b/stdlib/LibCURL_jll/Project.toml index 5ee99517624cb..0debbc077d594 100644 --- a/stdlib/LibCURL_jll/Project.toml +++ b/stdlib/LibCURL_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibCURL_jll" uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.14.1+1" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 10ee965bbc1ea..81a638cba72de 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -93,7 +93,7 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.14.1+1" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] From 895a981efc60f27c3e7a5cf1fee00f0001b00c83 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 13 Jun 2025 21:15:31 -0400 Subject: [PATCH 412/662] AGENTS.md: Add note about using failfast (#58696) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 38a978617a966..0e0773c60427b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,7 @@ corresponding test to ensure that the test is still passing with your changes. - If you are adding a new test, add it to an existing test file. Do not create a new test file unless explicitly instructed. - Write one comment at the top of the test to explain what is being tested. Otherwise keep comments minimal. +- Use the environment variable `JULIA_TEST_FAILFAST=1` to make tests fail fast. ### External dependencies From a23ce4bcbbb06be413169424cbffee0718b8a431 Mon Sep 17 00:00:00 2001 From: nilesh646 Date: Sat, 14 Jun 2025 20:35:49 +0530 Subject: [PATCH 413/662] Add tests for eltype(::Type{<:AbstractString}) (#58728) --- test/strings/basic.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index b18e2830e06d8..60d24280efe6b 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -1463,3 +1463,12 @@ if Sys.iswindows() @test Base.cwstring(str_3) == UInt16[0x0061, 0x0723, 0xd808, 0xdc00, 0x0000] end end + + +@testset "eltype for AbstractString subtypes" begin + @test eltype(String) == Char + @test eltype(SubString{String}) == Char + + u = b"hello" + @test eltype(u) === UInt8 +end From 24b3273c190243fa95e24bec8884ece6284cad91 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Mon, 16 Jun 2025 23:33:13 +0530 Subject: [PATCH 414/662] Define `in` for `CartesianIndex` ranges (#58616) Currently, this hits a fallback method that assumes that division is defined for the elements of the range. After this, the following works: ```julia julia> r = StepRangeLen(CartesianIndex(1), CartesianIndex(1), 3); julia> r[1] in r true julia> CartesianIndex(0) in r false ``` --- base/multidimensional.jl | 18 ++++++++++++++++++ test/cartesian.jl | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 7add7b9e74205..fd87c7afcf80f 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -189,6 +189,24 @@ module IteratorsMD step(r), ", ", length(r), ")") end + Base.in(x::CartesianIndex, r::AbstractRange{<:CartesianIndex}) = false + function Base.in(x::CartesianIndex{N}, r::AbstractRange{CartesianIndex{N}}) where {N} + isempty(r) && return false + f, st, l = first(r), step(r), last(r) + # The n-th element of the range is a CartesianIndex + # whose elements are the n-th along each dimension + # Find the first dimension along which the index is changing, + # so that n may be uniquely determined + for i in 1:N + iszero(st[i]) && continue + n = findfirst(==(x[i]), f[i]:st[i]:l[i]) + isnothing(n) && return false + return r[n] == x + end + # if the step is zero, the elements are identical, so compare with the first + return x == f + end + # Iteration const OrdinalRangeInt = OrdinalRange{Int, Int} """ diff --git a/test/cartesian.jl b/test/cartesian.jl index f9a0c8f976956..6097d4ca3770a 100644 --- a/test/cartesian.jl +++ b/test/cartesian.jl @@ -588,3 +588,42 @@ end @test I[begin] == I[1] @test I[end] == I[2] end + +@testset "in for a CartesianIndex StepRangeLen" begin + @testset for l in [0, 1, 4], r in Any[ + StepRangeLen(CartesianIndex(), CartesianIndex(), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(0), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(1), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(4), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(-4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, 4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, -4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, 2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, 2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, -2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, -2), l), + StepRangeLen(CartesianIndex(-1, 2, 0), CartesianIndex(0, 0, 0), l), + StepRangeLen(CartesianIndex(-1, 2, 0), CartesianIndex(0, 0, -2), l), + ] + + if length(r) == 0 + @test !(first(r) in r) + @test !(last(r) in r) + end + for x in r + @test x in r + if step(r) != oneunit(x) + @test !((x + oneunit(x)) in r) + end + end + @test !(CartesianIndex(ntuple(x->0, ndims(r))) in r) + @test !(CartesianIndex(ntuple(x->typemax(Int), ndims(r))) in r) + @test !(CartesianIndex(ntuple(x->typemin(Int), ndims(r))) in r) + if ndims(r) > 1 + @test !(CartesianIndex(ntuple(x->0, ndims(r)-1)...) in r) + end + end +end From 89dfb683be39ce8a89ddcd80d766ccbcdfcf6d04 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 17 Jun 2025 11:25:47 +0900 Subject: [PATCH 415/662] inference: improve `isdefined_tfunc` accuracy for `MustAlias` (#58743) --- Compiler/src/tfuncs.jl | 3 +++ Compiler/test/inference.jl | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index 9e07567a39adc..e42afb065687c 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -405,6 +405,9 @@ end return isdefined_tfunc(𝕃, arg1, sym) end @nospecs function isdefined_tfunc(𝕃::AbstractLattice, arg1, sym) + if arg1 isa MustAlias + arg1 = widenmustalias(arg1) + end arg1t = arg1 isa Const ? typeof(arg1.val) : isconstType(arg1) ? typeof(arg1.parameters[1]) : widenconst(arg1) a1 = unwrap_unionall(arg1t) if isa(a1, DataType) && !isabstracttype(a1) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 45f67129039f4..21d0f079b91d7 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -2313,12 +2313,6 @@ let 𝕃ᵢ = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance) @test ifelse_tfunc(MustAlias(2, AliasableField{Any}, 1, Int), Int, Int) === Union{} end -@testset "issue #56913: `BoundsError` in type inference" begin - R = UnitRange{Int} - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) -end - maybeget_mustalias_tmerge(x::AliasableField) = x.f maybeget_mustalias_tmerge(x) = x @test Base.return_types((Union{Nothing,AliasableField{Any}},); interp=MustAliasInterpreter()) do x @@ -2593,6 +2587,19 @@ end |> only === Compiler.InterMustAlias return 0 end == Integer +# `isdefined` accuracy for `MustAlias` +@test Base.infer_return_type((Any,); interp=MustAliasInterpreter()) do x + xx = Ref{Any}(x) + xxx = Some{Any}(xx) + Val(isdefined(xxx.value, :x)) +end == Val{true} + +@testset "issue #56913: `BoundsError` in type inference" begin + R = UnitRange{Int} + @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) + @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) +end + function f25579(g) h = g[] t = (h === nothing) From 36a4616fc218479b09607b1c15e61a36f12d13ad Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 17 Jun 2025 15:07:28 +0530 Subject: [PATCH 416/662] Increment state conditionally in `CartesianIndices` iteration (#58742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/JuliaLang/julia/issues/53430 ```julia julia> a = rand(100,100); b = similar(a); av = view(a, axes(a)...); bv = view(b, axes(b)...); bv2 = view(b, UnitRange.(axes(b))...); julia> @btime copyto!($bv2, $av); # slow, indices are UnitRanges 12.352 μs (0 allocations: 0 bytes) # master, v"1.13.0-DEV.745" 1.662 μs (0 allocations: 0 bytes) # this PR julia> @btime copyto!($bv, $av); # reference 1.733 μs (0 allocations: 0 bytes) ``` The performances become comparable after this PR. I've also renamed the second `I` to `Itail`, as the two variables represent different quantities. --- base/multidimensional.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index fd87c7afcf80f..edf71927661ca 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -472,12 +472,12 @@ module IteratorsMD end @inline function __inc(state::Tuple{Int,Int,Vararg{Int}}, indices::Tuple{OrdinalRangeInt,OrdinalRangeInt,Vararg{OrdinalRangeInt}}) rng = indices[1] - I = state[1] + step(rng) if state[1] != last(rng) + I = state[1] + step(rng) return true, (I, tail(state)...) end - valid, I = __inc(tail(state), tail(indices)) - return valid, (first(rng), I...) + valid, Itail = __inc(tail(state), tail(indices)) + return valid, (first(rng), Itail...) end # 0-d cartesian ranges are special-cased to iterate once and only once From 45f58349a37183eba68314bc6aeff7869e710b0d Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 17 Jun 2025 09:36:43 -0400 Subject: [PATCH 417/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Distributed=20stdlib=20from=2051e5297=20to=203679026=20(#58748)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Distributed URL: https://github.com/JuliaLang/Distributed.jl Stdlib branch: master Julia branch: master Old commit: 51e5297 New commit: 3679026 Julia version: 1.13.0-DEV Distributed version: 1.11.0(Does not match) Bump invoked by: @DilumAluthge Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Distributed.jl/compare/51e52978481835413d15b589919aba80dd85f890...3679026d7b510befdedfa8c6497e3cb032f9cea1 ``` $ git log --oneline 51e5297..3679026 3679026 Merge pull request #137 from JuliaLang/dpa/dont-use-link-local 875cd5a Rewrite the code to be a bit more explicit 2a6ee53 Non-link-local IP4 > non-link-local IP6 > link-local IP4 > link-local IP6 c0e9eb4 Factor functionality out into separate `choose_bind_addr()` function 86cbb8a Add explanation 0b7288c Worker: Bind to the first non-link-local IPv4 address ff8689a Merge pull request #131 from JuliaLang/spawnat-docs ba3c843 Document that `@spawnat :any` doesn't do load-balancing ``` Co-authored-by: DilumAluthge <5619885+DilumAluthge@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/Distributed.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 create mode 100644 deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 delete mode 100644 deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 delete mode 100644 deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 new file mode 100644 index 0000000000000..ca3d9730181f4 --- /dev/null +++ b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 @@ -0,0 +1 @@ +fdf2e62fcaed6aa5ad69bca405329675 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 new file mode 100644 index 0000000000000..186dfb34221b6 --- /dev/null +++ b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 @@ -0,0 +1 @@ +2361fc4ccad83139cf728f14fa38466f075fbf93dddfac533af8f5c0c35d6b86511881b7b97a95ee59f858ba9a33428d3514ac3bd605b745b002a673acfc3190 diff --git a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 b/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 deleted file mode 100644 index cdf885890db7c..0000000000000 --- a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -bf358a22ed4aa0d57b8672ef81fb2b44 diff --git a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 b/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 deleted file mode 100644 index 171fd2fdbf512..0000000000000 --- a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -399ab073d8c8cd1404e2e89ce2753486c782aa18646d56d8508c0a929c6a312e8bac2c791eddc01966f99cec1af34e56bbdccd8d75196b26f0b1bbb1fa8bd9ac diff --git a/stdlib/Distributed.version b/stdlib/Distributed.version index be52c0d684b50..99b5238a55491 100644 --- a/stdlib/Distributed.version +++ b/stdlib/Distributed.version @@ -1,4 +1,4 @@ DISTRIBUTED_BRANCH = master -DISTRIBUTED_SHA1 = 51e52978481835413d15b589919aba80dd85f890 +DISTRIBUTED_SHA1 = 3679026d7b510befdedfa8c6497e3cb032f9cea1 DISTRIBUTED_GIT_URL := https://github.com/JuliaLang/Distributed.jl DISTRIBUTED_TAR_URL = https://api.github.com/repos/JuliaLang/Distributed.jl/tarball/$1 From d88369d4978d6e0721b622080d63a1b9c60ee28f Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 17 Jun 2025 15:56:31 +0200 Subject: [PATCH 418/662] devdocs: contributing: fix headings (#58749) In particular, it seems like Documenter takes the level-one heading to define the page title. So the page titles were missing in the TOC before this change. --- doc/src/devdocs/contributing/code-changes.md | 14 ++++++++------ doc/src/devdocs/contributing/documentation.md | 10 +++++----- doc/src/devdocs/contributing/formatting.md | 6 +++--- doc/src/devdocs/contributing/git-workflow.md | 4 ++-- doc/src/devdocs/contributing/patch-releases.md | 3 +-- doc/src/devdocs/contributing/tests.md | 2 +- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/src/devdocs/contributing/code-changes.md b/doc/src/devdocs/contributing/code-changes.md index 14dbd65b1b5ce..41bb03f01409b 100644 --- a/doc/src/devdocs/contributing/code-changes.md +++ b/doc/src/devdocs/contributing/code-changes.md @@ -1,4 +1,6 @@ -### Contributing to core functionality or base libraries +# Code changes + +## Contributing to core functionality or base libraries *By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* @@ -25,7 +27,7 @@ Add new code to Julia's base libraries as follows (this is the "basic" approach; Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git). -#### Running specific tests +### Running specific tests There are `make` targets for running specific tests: @@ -35,7 +37,7 @@ You can also use the `runtests.jl` script, e.g. to run `test/bitarray.jl` and `t ./usr/bin/julia test/runtests.jl bitarray math -#### Modifying base more efficiently with Revise.jl +### Modifying base more efficiently with Revise.jl [Revise](https://github.com/timholy/Revise.jl) is a package that tracks changes in source files and automatically updates function @@ -74,7 +76,7 @@ system image before running the corresponding test. This can be useful as a shor on the command line (since tests aren't always designed to be run outside the runtest harness). -### Contributing to the standard library +## Contributing to the standard library The standard library (stdlib) packages are baked into the Julia system image. When running the ordinary test workflow on the stdlib packages, the system image @@ -92,11 +94,11 @@ not override the package. Be sure to change the UUID value back before making the pull request. -#### News-worthy changes +### News-worthy changes For new functionality and other substantial changes, add a brief summary to `NEWS.md`. The news item should cross reference the pull request (PR) parenthetically, in the form `([#pr])`. To add the PR reference number, first create the PR, then push an additional commit updating `NEWS.md` with the PR reference number. We periodically run `./julia doc/NEWS-update.jl` from the julia directory to update the cross-reference links, but this should not be done in a typical PR in order to avoid conflicting commits. -#### Annotations for new features, deprecations and behavior changes +### Annotations for new features, deprecations and behavior changes API additions and deprecations, and minor behavior changes are allowed in minor version releases. For documented features that are part of the public API, a compatibility note should be added into diff --git a/doc/src/devdocs/contributing/documentation.md b/doc/src/devdocs/contributing/documentation.md index 3c7b7de6bd0c1..e12ac1d482063 100644 --- a/doc/src/devdocs/contributing/documentation.md +++ b/doc/src/devdocs/contributing/documentation.md @@ -1,4 +1,4 @@ -### Improving documentation +# Improving documentation *By contributing documentation to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* @@ -16,7 +16,7 @@ from Julia's root directory. This will rebuild the Julia system image, then inst Below are outlined the three most common types of documentation changes and the steps required to perform them. Please note that the following instructions do not cover the full range of features provided by Documenter.jl. Refer to [Documenter's documentation](https://juliadocs.github.io/Documenter.jl/stable) if you encounter anything that is not covered by the sections below. -#### Modifying files in `doc/src/` +## Modifying files in `doc/src/` Most of the source text for the Julia Manual is located in `doc/src/`. To update or add new text to any one of the existing files the following steps should be followed: @@ -33,7 +33,7 @@ To add a **new file** to `doc/src/` rather than updating a file replace step `1` 1. add the file to the appropriate subdirectory in `doc/src/` and also add the file path to the `PAGES` vector in `doc/make.jl`. -#### Modifying an existing docstring in `base/` +## Modifying an existing docstring in `base/` All docstrings are written inline above the methods or types they are associated with and can be found by clicking on the `source` link that appears below each docstring in the HTML file. The steps needed to make a change to an existing docstring are listed below: @@ -43,7 +43,7 @@ All docstrings are written inline above the methods or types they are associated 4. check the output in `doc/_build/html/` to make sure the changes are correct; 5. commit your changes and open a pull request. -#### Adding a new docstring to `base/` +## Adding a new docstring to `base/` The steps required to add a new docstring are listed below: @@ -71,7 +71,7 @@ The steps required to add a new docstring are listed below: 6. check the output in `doc/_build/html` to make sure the changes are correct; 7. commit your changes and open a pull request. -#### Doctests +## Doctests Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`. diff --git a/doc/src/devdocs/contributing/formatting.md b/doc/src/devdocs/contributing/formatting.md index e26bf865876f7..24459884223d2 100644 --- a/doc/src/devdocs/contributing/formatting.md +++ b/doc/src/devdocs/contributing/formatting.md @@ -1,6 +1,6 @@ -### Code Formatting Guidelines +# Code Formatting Guidelines -#### General Formatting Guidelines for Julia code contributions +## General Formatting Guidelines for Julia code contributions - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). - use whitespace to make the code more readable @@ -11,7 +11,7 @@ Unicode equivalents whenever possible - in docstrings refer to the language as "Julia" and the executable as "`julia`" -#### General Formatting Guidelines For C code contributions +## General Formatting Guidelines For C code contributions - 4 spaces per indentation level, no tabs - space between `if` and `(` (`if (x) ...`) diff --git a/doc/src/devdocs/contributing/git-workflow.md b/doc/src/devdocs/contributing/git-workflow.md index 85fe9fa5e63fd..2142eed69632a 100644 --- a/doc/src/devdocs/contributing/git-workflow.md +++ b/doc/src/devdocs/contributing/git-workflow.md @@ -1,6 +1,6 @@ # Git workflow recommendations -### Git Recommendations For Pull Requests +## Git Recommendations For Pull Requests - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes. - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes. @@ -14,6 +14,6 @@ - To remove whitespace relative to the `master` branch, run `git rebase --whitespace=fix master`. -#### Git Recommendations For Pull Request Reviewers +### Git Recommendations For Pull Request Reviewers - When merging, we generally like `squash+merge`. Unless it is the rare case of a PR with carefully staged individual commits that you want in the history separately, in which case `merge` is acceptable, but usually prefer `squash+merge`. diff --git a/doc/src/devdocs/contributing/patch-releases.md b/doc/src/devdocs/contributing/patch-releases.md index dcbef6056fcce..515c2cfd35225 100644 --- a/doc/src/devdocs/contributing/patch-releases.md +++ b/doc/src/devdocs/contributing/patch-releases.md @@ -1,5 +1,4 @@ - -### Contributing to patch releases +# Contributing to patch releases The process of [creating a patch release](https://docs.julialang.org/en/v1/devdocs/build/distributing/#Point-releasing-101) is roughly as follows: diff --git a/doc/src/devdocs/contributing/tests.md b/doc/src/devdocs/contributing/tests.md index 7df203b96ffb6..c1f25da5f4a7c 100644 --- a/doc/src/devdocs/contributing/tests.md +++ b/doc/src/devdocs/contributing/tests.md @@ -1,4 +1,4 @@ -### Writing tests +# Writing tests There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it. From aa10603c188669d9f3b4be4fa744eeab32b975af Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 17 Jun 2025 12:36:26 -0300 Subject: [PATCH 419/662] Work around LLVM JITLink stack overflow issue. (#58579) The JITLinker recurses for every symbol in the list so limit the size of the list This is kind of ugly. Also 1000 might be too large, we don't want to go too small because that wastes memory and 1000 was fine locally for the things I tested. Fixes https://github.com/JuliaLang/julia/issues/58229 --- src/jitlayers.cpp | 13 +++++++++++-- test/cmdlineargs.jl | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index e8bb1f995c140..9ea98fed68db3 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -723,8 +723,17 @@ static void jl_compile_codeinst_now(jl_code_instance_t *codeinst) if (!decls.specFunctionObject.empty()) NewDefs.push_back(decls.specFunctionObject); } - auto Addrs = jl_ExecutionEngine->findSymbols(NewDefs); - + // Split batches to avoid stack overflow in the JIT linker. + // FIXME: Patch ORCJITs InPlaceTaskDispatcher to not recurse on task dispatches but + // push the tasks to a queue to be drained later. This avoids the stackoverflow caused by recursion + // in the linker when compiling a large number of functions at once. + SmallVector Addrs; + for (size_t i = 0; i < NewDefs.size(); i += 1000) { + auto end = std::min(i + 1000, NewDefs.size()); + SmallVector batch(NewDefs.begin() + i, NewDefs.begin() + end); + auto AddrsBatch = jl_ExecutionEngine->findSymbols(batch); + Addrs.append(AddrsBatch); + } size_t nextaddr = 0; for (auto &this_code : linkready) { auto it = invokenames.find(this_code); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 5aee455ae75b4..5a131eca3001f 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1340,3 +1340,6 @@ end end end end + +# https://github.com/JuliaLang/julia/issues/58229 Recursion in jitlinking with inline=no +@test success(`$(Base.julia_cmd()) --inline=no -e 'Base.compilecache(Base.identify_package("Pkg"))'`) From 787e0d9bf62b76386a1853b961312bb3cd45cf32 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 18 Jun 2025 03:45:02 +0900 Subject: [PATCH 420/662] bump Compiler.jl version to 0.1.1 (#58744) As the latest version of BaseCompiler.jl will be bumped to v0.1.1 after JuliaRegistries/General#132990. --- Compiler/LICENSE.md | 2 +- Compiler/Project.toml | 2 +- Compiler/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Compiler/LICENSE.md b/Compiler/LICENSE.md index 028a39923ef04..dbbcd7506fc1e 100644 --- a/Compiler/LICENSE.md +++ b/Compiler/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2009-2024: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors +Copyright (c) 2009-2025: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Compiler/Project.toml b/Compiler/Project.toml index f44b45a810708..1a0cdf4abca39 100644 --- a/Compiler/Project.toml +++ b/Compiler/Project.toml @@ -1,6 +1,6 @@ name = "Compiler" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.1.0" +version = "0.1.1" [compat] julia = "1.10" diff --git a/Compiler/README.md b/Compiler/README.md index 5e58152519678..ae5aaa3f60792 100644 --- a/Compiler/README.md +++ b/Compiler/README.md @@ -19,10 +19,10 @@ Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" Compiler = "0.1" ``` -With the setup above, [the special placeholder version (v0.1.0)](https://github.com/JuliaLang/BaseCompiler.jl) +With the setup above, [the special placeholder version (v0.1)](https://github.com/JuliaLang/BaseCompiler.jl) will be installed by default.[^1] -[^1]: Currently, only version v0.1.0 is registered in the [General](https://github.com/JuliaRegistries/General) registry. +[^1]: Currently, only version v0.1 is registered in the [General](https://github.com/JuliaRegistries/General) registry. If needed, you can switch to a custom implementation of the `Compiler` module by running ```julia-repl From 76df602cfd9368e0979ab96e6f6527b49606babf Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 18 Jun 2025 19:35:07 +0900 Subject: [PATCH 421/662] REPL: fix typo and potential `UndefVarError` (#58761) Detected by the new LS diagnostics:) --- stdlib/REPL/src/REPL.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 545383e1a82d7..916cc5cec8b57 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1866,7 +1866,7 @@ function create_global_out!(mod) end return out end - return getglobal(mod, Out) + return getglobal(mod, :Out) end function capture_result(n::Ref{Int}, @nospecialize(x)) From 2fbf5d856cfd605c9ab599b9840d2099273fd743 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 18 Jun 2025 20:39:07 +0900 Subject: [PATCH 422/662] fix fallback code path in `take!(::IOBuffer)` method (#58762) JET told me that the `data` local variable was inparticular is undefined at this point. After reviewing this code, I think this code path is unreachable actually since `bytesavailable(io::IOBuffer)` returns `0` when `io` has been closed. So it's probably better to make it clear. --- base/iobuffer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index b2e51618a8835..dd3b757c0e3f0 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -771,7 +771,7 @@ function take!(io::IOBuffer) elseif io.writable data = wrap(Array, memoryref(io.data, io.ptr), nbytes) else - data = read!(io, data) + error("Unreachable IOBuffer state") end end if io.writable From 0a9051bbe1300cef736de0dd1a1c0ae43b629c1d Mon Sep 17 00:00:00 2001 From: Chris Garling Date: Wed, 18 Jun 2025 11:06:02 -0400 Subject: [PATCH 423/662] Fix multi-threading docs typo (#58770) --- doc/src/manual/multi-threading.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index 401169e9c2132..56911b1f04df6 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -212,7 +212,7 @@ Note that [`Threads.@threads`](@ref) does not have an optional reduction paramet ### Using `@threads` without data-races -The concept of a data-race is elaborated on in ["Communication and data races between threads"](@ref man-communication-and-data-races). For now, just known that a data race can result in incorrect results and dangerous errors. +The concept of a data-race is elaborated on in ["Communication and data races between threads"](@ref man-communication-and-data-races). For now, just know that a data race can result in incorrect results and dangerous errors. Lets say we want to make the function `sum_single` below multithreaded. ```julia-repl From 78653490de2d56a4ab4e79911f7a033c7a923a25 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Wed, 18 Jun 2025 20:40:10 +0200 Subject: [PATCH 424/662] help bounds checking to be eliminated for `getindex(::Memory, ::Int)` (#58754) Second try for PR #58741. This moves the `getindex(::Memory, ::Int)` bounds check to Julia, which is how it's already done for `getindex(::Array, ::Int)`, so I guess it's correct. Also deduplicate the bounds checking code while at it. --- base/essentials.jl | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index 607ce44b39361..fec8ee89f9128 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -377,13 +377,26 @@ macro _nospecializeinfer_meta() return Expr(:meta, :nospecializeinfer) end +function _checkbounds_array(::Type{Bool}, A::Union{Array, GenericMemory}, i::Int) + @inline + ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) +end +function _checkbounds_array(A::Union{Array, GenericMemory}, i::Int) + @inline + _checkbounds_array(Bool, A, i) || throw_boundserror(A, (i,)) +end + default_access_order(a::GenericMemory{:not_atomic}) = :not_atomic default_access_order(a::GenericMemory{:atomic}) = :monotonic default_access_order(a::GenericMemoryRef{:not_atomic}) = :not_atomic default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic -getindex(A::GenericMemory, i::Int) = (@_noub_if_noinbounds_meta; - memoryrefget(memoryrefnew(memoryrefnew(A), i, @_boundscheck), default_access_order(A), false)) +function getindex(A::GenericMemory, i::Int) + @_noub_if_noinbounds_meta + (@_boundscheck) && _checkbounds_array(A, i) + memoryrefget(memoryrefnew(memoryrefnew(A), i, false), default_access_order(A), false) +end + getindex(A::GenericMemoryRef) = memoryrefget(A, default_access_order(A), @_boundscheck) """ @@ -949,13 +962,13 @@ end # linear indexing function getindex(A::Array, i::Int) @_noub_if_noinbounds_meta - @boundscheck ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) || throw_boundserror(A, (i,)) + @boundscheck _checkbounds_array(A, i) memoryrefget(memoryrefnew(getfield(A, :ref), i, false), :not_atomic, false) end # simple Array{Any} operations needed for bootstrap function setindex!(A::Array{Any}, @nospecialize(x), i::Int) @_noub_if_noinbounds_meta - @boundscheck ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) || throw_boundserror(A, (i,)) + @boundscheck _checkbounds_array(A, i) memoryrefset!(memoryrefnew(getfield(A, :ref), i, false), x, :not_atomic, false) return A end From a812f03d866e3050216f6d18ccc291179f86e8bf Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Wed, 18 Jun 2025 22:15:42 +0200 Subject: [PATCH 425/662] Define textwidth for overlong chars (#58602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, this would error. There is no guarantee of how terminals render overlong encodings. Some terminals does not print them at all, and some print "�". Here, we set a textwidth of 1, conservatively. Refs #58593 --- base/strings/unicode.jl | 16 +++++++--------- test/strings/util.jl | 2 ++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index d8f3a2b6ad66b..520226aacd3cb 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -6,7 +6,7 @@ module Unicode import Base: show, ==, hash, string, Symbol, isless, length, eltype, convert, isvalid, ismalformed, isoverlong, iterate, AnnotatedString, AnnotatedChar, annotated_chartransform, - @assume_effects, annotations + @assume_effects, annotations, is_overlong_enc # whether codepoints are valid Unicode scalar values, i.e. 0-0xd7ff, 0xe000-0x10ffff @@ -262,17 +262,15 @@ julia> textwidth('⛵') 2 ``` """ -function textwidth(c::AbstractChar) - ismalformed(c) && return 1 - i = codepoint(c) - i < 0x7f && return Int(i >= 0x20) # ASCII fast path - Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), i)) -end +textwidth(c::AbstractChar) = textwidth(Char(c)::Char) function textwidth(c::Char) - b = bswap(reinterpret(UInt32, c)) # from isascii(c) + u = reinterpret(UInt32, c) + b = bswap(u) # from isascii(c) b < 0x7f && return Int(b >= 0x20) # ASCII fast path - ismalformed(c) && return 1 + # We can't know a priori how terminals will render invalid UTF8 chars, + # so we conservatively decide a width of 1. + (ismalformed(c) || is_overlong_enc(u)) && return 1 Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), c)) end diff --git a/test/strings/util.jl b/test/strings/util.jl index bb87881bbaa1d..9ced27ee3f8d0 100644 --- a/test/strings/util.jl +++ b/test/strings/util.jl @@ -8,6 +8,8 @@ SubStr(s) = SubString("abc$(s)de", firstindex(s) + 3, lastindex(s) + 3) @test textwidth(c^3) == w*3 @test w == @invoke textwidth(c::AbstractChar) end + @test textwidth('\xc0\xa0') == 1 # overlong + @test textwidth('\xf0\x80\x80') == 1 # malformed for i in 0x00:0x7f # test all ASCII chars (which have fast path) w = Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), i)) c = Char(i) From 422d05d1f8c185ad636deb0ab181aa41e3d424ea Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 18 Jun 2025 20:18:09 -0400 Subject: [PATCH 426/662] Add MethodError hints for functions in other modules (#58715) When a MethodError occurs, check if functions with the same name exist in other modules (particularly those of the argument types). This helps users discover that they may need to import a function or ensure multiple functions are the same generic function. - For Base functions: suggests importing (e.g., "You may have intended to import Base.length") - For other modules: suggests they may be intended as the same generic function - Shows all matches from relevant modules in sorted order - Uses modulesof! to properly handle all type structures including unions Fixes #58682 --- base/errorshow.jl | 34 +++++++++++++++++++++++++++------- test/errorshow.jl | 14 ++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 64601ea35f548..624461a31c641 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -327,13 +327,33 @@ function showerror(io::IO, ex::MethodError) if ft <: AbstractArray print(io, "\nIn case you're trying to index into the array, use square brackets [] instead of parentheses ().") end - # Check for local functions that shadow methods in Base - let name = ft.name.singletonname - if f_is_function && isdefined(Base, name) - basef = getfield(Base, name) - if basef !== f && hasmethod(basef, arg_types) - print(io, "\nYou may have intended to import ") - show_unquoted(io, Expr(:., :Base, QuoteNode(name))) + # Check for functions with the same name in other modules + if f_is_function && ex.world != typemax(UInt) + let name = ft.name.singletonname + modules_to_check = Set{Module}() + push!(modules_to_check, Base) + for T in san_arg_types_param + modulesof!(modules_to_check, T) + end + + # Check all modules (sorted for consistency) + sorted_modules = sort!(collect(modules_to_check), by=nameof) + for mod in sorted_modules + if isdefined(mod, name) + candidate = getfield(mod, name) + if candidate !== f && hasmethod(candidate, arg_types; world=ex.world) + if mod === Base + print(io, "\nYou may have intended to import ") + show_unquoted(io, Expr(:., :Base, QuoteNode(name))) + else + print(io, "\nThe definition in ") + show_unquoted(io, mod) + print(io, " may have intended to extend ") + f_module = parentmodule(ft) + show_unquoted(io, Expr(:., f_module, QuoteNode(name))) + end + end + end end end end diff --git a/test/errorshow.jl b/test/errorshow.jl index e37d22718b491..42dc5ad31cb34 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -1019,6 +1019,20 @@ for (func,str) in ((TestMethodShadow.:+,":+"), (TestMethodShadow.:(==),":(==)"), @test occursin("You may have intended to import Base.$str", sprint(Base.showerror, ex)) end +# Test hint for functions in modules of argument types (issue #58682) +module TestModuleHint + struct Bar end + length(x::Bar) = 42 +end +let ex = try + # Call Base.length on TestModuleHint.Bar - should suggest importing TestModuleHint.length + length(TestModuleHint.Bar()) + catch e + e + end::MethodError + @test occursin("may have intended to extend", sprint(Base.showerror, ex)) +end + # Test that implementation detail of include() is hidden from the user by default let bt = try @noinline include("testhelpers/include_error.jl") From 3bb7518b041e79d2abe6b9d6386c273301436c14 Mon Sep 17 00:00:00 2001 From: abhro <5664668+abhro@users.noreply.github.com> Date: Thu, 19 Jun 2025 05:47:56 -0400 Subject: [PATCH 427/662] Fix markdown bullet list in variables-and-scoping.md (#58771) --- doc/src/manual/variables-and-scoping.md | 29 ++++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/src/manual/variables-and-scoping.md b/doc/src/manual/variables-and-scoping.md index ab0dbdb845d8e..0a9180ecd9479 100644 --- a/doc/src/manual/variables-and-scoping.md +++ b/doc/src/manual/variables-and-scoping.md @@ -733,21 +733,24 @@ object (such as an array), and that object may still be modified. Additionally w to assign a value to a variable that is declared constant the following scenarios are possible: * Attempting to replace a constant without the const `keyword` is disallowed: -```jldoctest -julia> const x = 1.0 -1.0 -julia> x = 1 -ERROR: invalid assignment to constant x. This redefinition may be permitted using the `const` keyword. -``` -* All other defefinitions of constants are permitted, but may cause significant re-compilation: -```jldoctest -julia> const y = 1.0 -1.0 + ```jldoctest + julia> const x = 1.0 + 1.0 -julia> const y = 2.0 -2.0 -``` + julia> x = 1 + ERROR: invalid assignment to constant x. This redefinition may be permitted using the `const` keyword. + ``` + +* All other definitions of constants are permitted, but may cause significant re-compilation: + + ```jldoctest + julia> const y = 1.0 + 1.0 + + julia> const y = 2.0 + 2.0 + ``` !!! compat "Julia 1.12" Prior to julia 1.12, redefinition of constants was poorly supported. It was restricted to From 4962d2d5c0c2b96150e577c088213f51d1b34421 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 19 Jun 2025 08:43:05 -0400 Subject: [PATCH 428/662] CONTRIBUTING.md: Ask folks to disclose AI-written PRs (#58666) --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dbd456d35e8d..36ec53c6a181d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,8 @@ A useful bug report filed as a GitHub issue provides information about how to re * Review discussions on the [Julia Discourse forum](https://discourse.julialang.org). +* If your pull request contains substantial contributions from a generative AI tool, please disclose so with details, and review all changes before opening. + * Relax and have fun! ### Guidance for specific changes From a60d238cc74dd470b0a3f4fb36e3ca8c8518fadd Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 19 Jun 2025 13:51:47 -0400 Subject: [PATCH 429/662] Convert julia-repl blocks to jldoctest format (#58594) Convert appropriate julia-repl code blocks to jldoctest format to enable automatic testing. In addition, this introduces a new `nodoctest = "reason"` pattern to annotate code blocks that are deliberate not doctested, so future readers will know not to try. Many code blocks are converted, in particular: - Manual pages: arrays.md, asynchronous-programming.md, functions.md, integers-and-floating-point-numbers.md, metaprogramming.md, multi-threading.md, performance-tips.md, variables.md, variables-and-scoping.md - Base documentation: abstractarray.jl, bitarray.jl, expr.jl, file.jl, float.jl, iddict.jl, path.jl, scopedvalues.md, sort.md - Standard library: Dates/conversions.jl, Random/RNGs.jl, Sockets/addrinfo.jl Key changes: - Add filters for non-deterministic output (timing, paths, memory addresses) - Add setup/teardown for filesystem operations - Fix parentmodule(M) usage in expr.jl for doctest compatibility - Document double escaping requirement for regex filters in docstrings - Update AGENTS.md with test running instructions Note: Some julia-repl blocks were intentionally left unchanged when they demonstrate language internals subject to change or contain non-deterministic output that cannot be properly filtered. Refs #56921 --------- Co-authored-by: Keno Fischer Co-authored-by: Claude --- AGENTS.md | 21 +++++-- base/abstractarray.jl | 2 +- base/bitarray.jl | 2 +- base/expr.jl | 4 +- base/file.jl | 6 +- base/float.jl | 2 +- base/iddict.jl | 2 +- base/path.jl | 2 +- doc/make.jl | 1 - doc/src/base/scopedvalues.md | 57 +++++++++++------ doc/src/base/sort.md | 10 +-- doc/src/devdocs/contributing/jldoctests.md | 28 ++++++++- doc/src/manual/arrays.md | 63 ++++++++++--------- doc/src/manual/asynchronous-programming.md | 4 +- doc/src/manual/functions.md | 8 ++- .../integers-and-floating-point-numbers.md | 6 +- doc/src/manual/metaprogramming.md | 2 +- doc/src/manual/methods.md | 8 ++- doc/src/manual/multi-threading.md | 2 +- doc/src/manual/performance-tips.md | 8 ++- doc/src/manual/strings.md | 9 ++- doc/src/manual/variables-and-scoping.md | 10 +-- doc/src/manual/variables.md | 28 +++++---- stdlib/Dates/src/conversions.jl | 2 +- stdlib/Random/src/RNGs.jl | 2 +- stdlib/Sockets/src/addrinfo.jl | 2 +- 26 files changed, 183 insertions(+), 108 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0e0773c60427b..f8ba930f5a866 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,10 +38,23 @@ The test system supports doing this automatically (see below). ### Doctests -If you have changed doctests (i.e. any `jldoctest` code block), you should run the doctests before -preparing a PR. See `doc/README.md` for how to do this. For instructions on writing doctests, see -`doc/src/devdocs/contributing/jldoctests.md`. Note that the doctests may take up to 15 minutes. -Do not terminate the doctests before completion. If you are ChatGPT, you may have to increase yield_timeout_ms. +#### Writing doctests + +If you are asked to write new doctests, first review `doc/src/devdocs/contributing/jldoctests.md` +for best practices. + +#### Verifying doctests +If you have changed any `jldoctest` code blocks you should take +the following steps to verify your work: +- Review `doc/src/devdocs/contributing/jldoctests.md`. In particular, determine + if any of the changed doctests require filters, labels or setup code. +- Run the doctests to verify that your change works: + - To run doctest with the pre-built juliaup: `make -C doc doctest=true revise=true JULIA_EXECUTABLE=$HOME/.juliaup/bin/julia` + - To run doctest with in-trr julia (preferred): `make -C doc doctest=true revise=true`. Do not pass any other options. + - IMPORTANT: The doctests may take up to 15 minutes. Do NOT terminate the doctests before completion. Do NOT use a timeout for doctests. + - If you are ChatGPT, you may have to increase yield_timeout_ms. + +Follow these steps for EVERY change you make in a doctest. ### Test changes diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d026f8082cfcd..8f55e6a56eba8 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -796,7 +796,7 @@ julia> similar(1:10, 1, 4) Conversely, `similar(trues(10,10), 2)` returns an uninitialized `BitVector` with two elements since `BitArray`s are both mutable and can support 1-dimensional arrays: -```julia-repl +```jldoctest; filter = r"[01]" julia> similar(trues(10,10), 2) 2-element BitVector: 0 diff --git a/base/bitarray.jl b/base/bitarray.jl index e47cf82dd603f..5a3469fa7c7a2 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -53,7 +53,7 @@ Construct an undef [`BitArray`](@ref) with the given dimensions. Behaves identically to the [`Array`](@ref) constructor. See [`undef`](@ref). # Examples -```julia-repl +```jldoctest; filter = r"[01]" julia> BitArray(undef, 2, 2) 2×2 BitMatrix: 0 0 diff --git a/base/expr.jl b/base/expr.jl index bf9e2ef2bf92c..b5f7bd8188d50 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -144,7 +144,7 @@ There are differences between `@macroexpand` and [`macroexpand`](@ref). expands with respect to the module in which it is called. This is best seen in the following example: -```julia-repl +```jldoctest julia> module M macro m() 1 @@ -152,7 +152,7 @@ julia> module M function f() (@macroexpand(@m), macroexpand(M, :(@m)), - macroexpand(Main, :(@m)) + macroexpand(parentmodule(M), :(@m)) ) end end diff --git a/base/file.jl b/base/file.jl index 7fbb08c4b6fe6..24a21396721d3 100644 --- a/base/file.jl +++ b/base/file.jl @@ -164,7 +164,7 @@ required intermediate directories. Return `path`. # Examples -```julia-repl +```jldoctest; setup = :(curdir = pwd(); testdir = mktempdir(); cd(testdir)), teardown = :(cd(curdir); rm(testdir, recursive=true)), filter = r"^\\".*testingdir\\"\$" julia> mkdir("testingdir") "testingdir" @@ -516,7 +516,7 @@ If the file does not exist a new file is created. Return `path`. # Examples -```julia-repl +```jldoctest; setup = :(curdir = pwd(); testdir = mktempdir(); cd(testdir)), teardown = :(cd(curdir); rm(testdir, recursive=true)), filter = r"[\\d\\.]+e[\\+\\-]?\\d+" julia> write("my_little_file", 2); julia> mtime("my_little_file") @@ -1134,7 +1134,7 @@ for (path, dirs, files) in walkdir(".") end ``` -```julia-repl +```jldoctest; setup = :(prevdir = pwd(); tmpdir = mktempdir(); cd(tmpdir)), teardown = :(cd(prevdir); rm(tmpdir, recursive=true)) julia> mkpath("my/test/dir"); julia> itr = walkdir("my"); diff --git a/base/float.jl b/base/float.jl index 79d4a26ef930e..ba380102f8751 100644 --- a/base/float.jl +++ b/base/float.jl @@ -83,7 +83,7 @@ julia> NaN == NaN, isequal(NaN, NaN), isnan(NaN) !!! note Always use [`isnan`](@ref) or [`isequal`](@ref) for checking for `NaN`. Using `x === NaN` may give unexpected results: - ```julia-repl + ```jldoctest julia> reinterpret(UInt32, NaN32) 0x7fc00000 diff --git a/base/iddict.jl b/base/iddict.jl index f1632e93427a8..b12fbfcca107f 100644 --- a/base/iddict.jl +++ b/base/iddict.jl @@ -12,7 +12,7 @@ the same, so they get overwritten. The `IdDict` hashes by object-id, and thus preserves the 3 different keys. # Examples -```julia-repl +```jldoctest; filter = r" \\S+ +=> \\S+" => " KEY => VALUE" julia> Dict(true => "yes", 1 => "no", 1.0 => "maybe") Dict{Real, String} with 1 entry: 1.0 => "maybe" diff --git a/base/path.jl b/base/path.jl index 0e67dc4e95db6..a1221e0bdc844 100644 --- a/base/path.jl +++ b/base/path.jl @@ -633,7 +633,7 @@ the [Freedesktop File URI spec](https://www.freedesktop.org/wiki/Specifications/ julia> uripath("/home/user/example file.jl") # On a unix machine "file:///home/user/example%20file.jl" -juila> uripath("C:\\Users\\user\\example file.jl") # On a windows machine +julia> uripath("C:\\Users\\user\\example file.jl") # On a windows machine "file:///C:/Users/user/example%20file.jl" ``` """ diff --git a/doc/make.jl b/doc/make.jl index da78a8420fa2c..bd74316d9fd7e 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -318,7 +318,6 @@ if use_revise end function maybe_revise(ex) use_revise || return ex - STDLIB_DIR = Sys.STDLIB STDLIBS = filter!(x -> isfile(joinpath(STDLIB_DIR, x, "src", "$(x).jl")), readdir(STDLIB_DIR)) return quote $ex diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 6ad553429bb1f..d35129642b29b 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -27,38 +27,57 @@ Let's first look at an example of **lexical** scope. A `let` statement begins a new lexical scope within which the outer definition of `x` is shadowed by it's inner definition. -```julia +```jldoctest +julia> x = 1 +1 + +julia> let x = 5 + @show x + end; +x = 5 + +julia> @show x; x = 1 -let x = 5 - @show x # 5 -end -@show x # 1 ``` In the following example, since Julia uses lexical scope, the variable `x` in the body of `f` refers to the `x` defined in the global scope, and entering a `let` scope does not change the value `f` observes. -```julia +```jldoctest +julia> x = 1 +1 + +julia> f() = @show x +f (generic function with 1 method) + +julia> let x = 5 + f() + end; +x = 1 + +julia> f(); x = 1 -f() = @show x -let x = 5 - f() # 1 -end -f() # 1 ``` Now using a `ScopedValue` we can use **dynamic** scoping. -```julia -using Base.ScopedValues +```jldoctest +julia> using Base.ScopedValues -x = ScopedValue(1) -f() = @show x[] -with(x=>5) do - f() # 5 -end -f() # 1 +julia> x = ScopedValue(1) +ScopedValue{Int64}(1) + +julia> f() = @show x[] +f (generic function with 1 method) + +julia> with(x=>5) do + f() + end; +x[] = 5 + +julia> f(); +x[] = 1 ``` Note that the observed value of the `ScopedValue` is dependent on the execution diff --git a/doc/src/base/sort.md b/doc/src/base/sort.md index cef080c5f8995..cde0d571424a0 100644 --- a/doc/src/base/sort.md +++ b/doc/src/base/sort.md @@ -39,8 +39,8 @@ julia> a Instead of directly sorting an array, you can compute a permutation of the array's indices that puts the array into sorted order: -```julia-repl -julia> v = randn(5) +```jldoctest sort_example +julia> v = [0.297288, 0.382396, -0.597634, -0.0104452, -0.839027] 5-element Vector{Float64}: 0.297288 0.382396 @@ -67,7 +67,7 @@ julia> v[p] Arrays can be sorted according to an arbitrary transformation of their values: -```julia-repl +```jldoctest sort_example julia> sort(v, by=abs) 5-element Vector{Float64}: -0.0104452 @@ -79,7 +79,7 @@ julia> sort(v, by=abs) Or in reverse order by a transformation: -```julia-repl +```jldoctest sort_example julia> sort(v, by=abs, rev=true) 5-element Vector{Float64}: -0.839027 @@ -91,7 +91,7 @@ julia> sort(v, by=abs, rev=true) If needed, the sorting algorithm can be chosen: -```julia-repl +```jldoctest sort_example julia> sort(v, alg=InsertionSort) 5-element Vector{Float64}: -0.839027 diff --git a/doc/src/devdocs/contributing/jldoctests.md b/doc/src/devdocs/contributing/jldoctests.md index bf989062b1305..6631390cf9ca3 100644 --- a/doc/src/devdocs/contributing/jldoctests.md +++ b/doc/src/devdocs/contributing/jldoctests.md @@ -5,8 +5,17 @@ This page describes how to write and maintain `jldoctest` blocks in the document ## Filters Use `filter =` whenever output contains text that might vary across runs. -The documentation relies on several recurring patterns: +The following are common situations where this may happen: + +- The output contains arrays with undefined memory (e.g. from `undef` or `similar`) +- The output contains random numbers +- The output contains timing information +- The output contains file system paths + + +### Common filter sequences +The documentation relies on several recurring patterns: - `r"int.jl:\\d+"` — remove line numbers from introspection macros. - `r"Stacktrace:(\\n \\[0-9]+\\].*)*"` — hide stack traces when illustrating errors. @@ -29,6 +38,12 @@ If none of these match your situation, craft a regular expression that removes the varying text. Using filters keeps doctests stable across platforms and Julia versions. +!!! note "Double escaping in docstrings" + When writing regex filters inside docstrings, remember to double escape + backslashes. For example, use `r"[\\d\\.]+"` instead of `r"[\d\.]+"`. + This is necessary because the docstring itself processes escape sequences + before the regex is created. + ## Setup code Small setup expressions may be placed inline using the `setup =` option: @@ -56,6 +71,17 @@ DocTestSetup = nothing ``` ```` +### Teardown code + +If you need teardown code (e.g. to delete created temporary files or to reset +the current directory), you can use the `teardown =` option: + +```` +```jldoctest; setup = :(oldpath = pwd(); cd(mktempdir())), teardown = :(cd(oldpath)) +... +``` +```` + ## Maintaining state between snippets Related doctest blocks can share state by giving them the same label after the diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index ee9421ba1730b..6fa27ba75c90f 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -368,26 +368,26 @@ of the variable ranges `rx`, `ry`, etc. and each `F(x,y,...)` evaluation returns The following example computes a weighted average of the current element and its left and right neighbor along a 1-d grid: -```julia-repl -julia> x = rand(8) -8-element Vector{Float64}: - 0.843025 - 0.869052 - 0.365105 - 0.699456 - 0.977653 - 0.994953 - 0.41084 - 0.809411 +```jldoctest +julia> x = [4, 8, 2, 6, 10, 10, 2, 8] +8-element Vector{Int64}: + 4 + 8 + 2 + 6 + 10 + 10 + 2 + 8 julia> [ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ] 6-element Vector{Float64}: - 0.736559 - 0.57468 - 0.685417 - 0.912429 - 0.8446 - 0.656511 + 5.5 + 4.5 + 6.0 + 9.0 + 8.0 + 5.5 ``` The resulting array type depends on the types of the computed elements just like [array literals](@ref man-array-literals) do. In order to control the @@ -413,9 +413,12 @@ julia> sum(1/n^2 for n=1:1000) When writing a generator expression with multiple dimensions inside an argument list, parentheses are needed to separate the generator from subsequent arguments: -```julia-repl +```jldoctest julia> map(tuple, 1/(i+j) for i=1:2, j=1:2, [1:4;]) -ERROR: syntax: invalid iteration specification +ERROR: ParseError: +# Error @ none:1:44 +map(tuple, 1/(i+j) for i=1:2, j=1:2, [1:4;]) +# └ ── invalid iteration spec: expected one of `=` `in` or `∈` ``` All comma-separated expressions after `for` are interpreted as ranges. Adding parentheses lets @@ -1036,33 +1039,33 @@ It is sometimes useful to perform element-by-element binary operations on arrays sizes, such as adding a vector to each column of a matrix. An inefficient way to do this would be to replicate the vector to the size of the matrix: -```julia-repl -julia> a = rand(2, 1); A = rand(2, 3); +```jldoctest broadcast_example +julia> a = [0.2, 0.5]; A = [1.0 1.6 1.05; 1.07 1.36 1.18]; julia> repeat(a, 1, 3) + A 2×3 Matrix{Float64}: - 1.20813 1.82068 1.25387 - 1.56851 1.86401 1.67846 + 1.2 1.8 1.25 + 1.57 1.86 1.68 ``` This is wasteful when dimensions get large, so Julia provides [`broadcast`](@ref), which expands singleton dimensions in array arguments to match the corresponding dimension in the other array without using extra memory, and applies the given function elementwise: -```julia-repl +```jldoctest broadcast_example julia> broadcast(+, a, A) 2×3 Matrix{Float64}: - 1.20813 1.82068 1.25387 - 1.56851 1.86401 1.67846 + 1.2 1.8 1.25 + 1.57 1.86 1.68 -julia> b = rand(1,2) +julia> b = [0.9 0.1] 1×2 Matrix{Float64}: - 0.867535 0.00457906 + 0.9 0.1 julia> broadcast(+, a, b) 2×2 Matrix{Float64}: - 1.71056 0.847604 - 1.73659 0.873631 + 1.1 0.3 + 1.4 0.6 ``` [Dotted operators](@ref man-dot-operators) such as `.+` and `.*` are equivalent diff --git a/doc/src/manual/asynchronous-programming.md b/doc/src/manual/asynchronous-programming.md index d1d095c48b2ff..ccdffda9a1acc 100644 --- a/doc/src/manual/asynchronous-programming.md +++ b/doc/src/manual/asynchronous-programming.md @@ -203,7 +203,7 @@ A channel can be visualized as a pipe, i.e., it has a write end and a read end : freely via [`take!`](@ref) and [`put!`](@ref) calls. [`close`](@ref) closes a [`Channel`](@ref). On a closed [`Channel`](@ref), [`put!`](@ref) will fail. For example: - ```julia-repl + ```jldoctest channel_example julia> c = Channel(2); julia> put!(c, 1) # `put!` on an open channel succeeds @@ -220,7 +220,7 @@ A channel can be visualized as a pipe, i.e., it has a write end and a read end : * [`take!`](@ref) and [`fetch`](@ref) (which retrieves but does not remove the value) on a closed channel successfully return any existing values until it is emptied. Continuing the above example: - ```julia-repl + ```jldoctest channel_example julia> fetch(c) # Any number of `fetch` calls succeed. 1 diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index fd97b44cdc345..a83d16ca8c9b7 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -65,18 +65,22 @@ a function will be visible to the caller. (This is the same behavior found in Sc Python, Ruby and Perl, among other dynamic languages.) For example, in the function -```julia +```jldoctest argpassing; output = false function f(x, y) x[1] = 42 # mutates x y = 7 + y # new binding for y, no mutation return y end + +# output + +f (generic function with 1 method) ``` The statement `x[1] = 42` *mutates* the object `x`, and hence this change *will* be visible in the array passed by the caller for this argument. On the other hand, the assignment `y = 7 + y` changes the *binding* ("name") `y` to refer to a new value `7 + y`, rather than mutating the *original* object referred to by `y`, and hence does *not* change the corresponding argument passed by the caller. This can be seen if we call `f(x, y)`: -```julia-repl +```jldoctest argpassing julia> a = [4, 5, 6] 3-element Vector{Int64}: 4 diff --git a/doc/src/manual/integers-and-floating-point-numbers.md b/doc/src/manual/integers-and-floating-point-numbers.md index 0139a33f4854b..845d42e33abfb 100644 --- a/doc/src/manual/integers-and-floating-point-numbers.md +++ b/doc/src/manual/integers-and-floating-point-numbers.md @@ -59,7 +59,7 @@ julia> 1234 The default type for an integer literal depends on whether the target system has a 32-bit architecture or a 64-bit architecture: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> typeof(1) Int32 @@ -72,7 +72,7 @@ Int64 The Julia internal variable [`Sys.WORD_SIZE`](@ref) indicates whether the target system is 32-bit or 64-bit: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> Sys.WORD_SIZE 32 @@ -85,7 +85,7 @@ julia> Sys.WORD_SIZE Julia also defines the types `Int` and `UInt`, which are aliases for the system's signed and unsigned native integer types respectively: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> Int Int32 diff --git a/doc/src/manual/metaprogramming.md b/doc/src/manual/metaprogramming.md index 9ed579594571a..ee252d8c9fa2e 100644 --- a/doc/src/manual/metaprogramming.md +++ b/doc/src/manual/metaprogramming.md @@ -711,7 +711,7 @@ user to optionally specify their own error message, instead of just printing the Just like in functions with a variable number of arguments ([Varargs Functions](@ref)), this is specified with an ellipses following the last argument: -```jldoctest assert2 +```julia-repl assert2 julia> macro assert(ex, msgs...) msg_body = isempty(msgs) ? ex : msgs[1] msg = string(msg_body) diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 7538649f2b8df..cebd9dbb6061b 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -288,13 +288,17 @@ Such specializations are *not* listed by `methods`, as this doesn't create new ` For example, if you create a method -``` +```jldoctest mysum_example; output = false mysum(x::Real, y::Real) = x + y + +# output + +mysum (generic function with 1 method) ``` you've given the function `mysum` one new method (possibly its only method), and that method takes any pair of `Real` number inputs. But if you then execute -```julia-repl +```jldoctest mysum_example julia> mysum(1, 2) 3 diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index 56911b1f04df6..bdf642b777963 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -47,7 +47,7 @@ $ julia --threads 4 Let's verify there are 4 threads at our disposal. -```julia-repl +```jldoctest; filter = r"[0-9]+" julia> Threads.nthreads() 4 ``` diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index d506ac9946ba6..b08b71f65db05 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -58,14 +58,16 @@ Passing arguments to functions is better style. It leads to more reusable code a In the following REPL session: -```julia-repl +```jldoctest julia> x = 1.0 +1.0 ``` is equivalent to: -```julia-repl +```jldoctest julia> global x = 1.0 +1.0 ``` so all the performance issues discussed previously apply. @@ -1697,7 +1699,7 @@ in generated code by using Julia's [`code_native`](@ref) function. Note that `@fastmath` also assumes that `NaN`s will not occur during the computation, which can lead to surprising behavior: -```julia-repl +```jldoctest julia> f(x) = isnan(x); julia> f(NaN) diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index 57431d07c0aa5..2948c8aad0335 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -482,7 +482,7 @@ The resulting string may contain different characters than the input strings, and its number of characters may be lower than sum of numbers of characters of the concatenated strings, e.g.: -```julia-repl +```jldoctest julia> a, b = "\xe2\x88", "\x80" ("\xe2\x88", "\x80") @@ -992,7 +992,7 @@ The `r"..."` literal is constructed without interpolation and unescaping (except quotation mark `"` which still has to be escaped). Here is an example showing the difference from standard string literals: -```julia-repl +```jldoctest julia> x = 10 10 @@ -1006,7 +1006,10 @@ julia> r"\x" r"\x" julia> "\x" -ERROR: syntax: invalid escape sequence +ERROR: ParseError: +# Error @ none:1:2 +"\x" +#└┘ ── invalid hex escape sequence ``` Triple-quoted regex strings, of the form `r"""..."""`, are also supported (and may be convenient diff --git a/doc/src/manual/variables-and-scoping.md b/doc/src/manual/variables-and-scoping.md index 0a9180ecd9479..9756b4ec5bd87 100644 --- a/doc/src/manual/variables-and-scoping.md +++ b/doc/src/manual/variables-and-scoping.md @@ -28,7 +28,7 @@ a global variable by the same name is allowed or not. !!! tip "A Common Confusion" If you run into an unexpectedly undefined variable, - ```julia + ```julia ; nodoctest = "Pseudocode" # Print the numbers 1 through 5 i = 0 while i < 5 @@ -40,7 +40,7 @@ a global variable by the same name is allowed or not. a simple fix is to change all global variable definitions into local definitions by wrapping the code in a `let` block or `function`. - ```julia + ```julia ; nodoctest = "Pseudocode" # Print the numbers 1 through 5 let i = 0 while i < 5 @@ -370,7 +370,7 @@ scope rule applies and `x` is created as local to the `for` loop and therefore g undefined after the loop executes. Next, let's consider the body of `sum_to_def` extracted into global scope, fixing its argument to `n = 10` -```julia +```julia ; nodoctest = "Specifically shows scope differences" s = 0 for i = 1:10 t = s + i @@ -472,7 +472,7 @@ years were confused about this behavior and complained that it was complicated a explain and understand. Fair point. Second, and arguably worse, is that it's bad for programming "at scale." When you see a small piece of code in one place like this, it's quite clear what's going on: -```julia +```julia ; nodoctest="Expects global file scope" s = 0 for i = 1:10 s += i @@ -483,7 +483,7 @@ Obviously the intention is to modify the existing global variable `s`. What else However, not all real world code is so short or so clear. We found that code like the following often occurs in the wild: -```julia +```julia ; nodoctest="Expects global file scope" x = 123 # much later diff --git a/doc/src/manual/variables.md b/doc/src/manual/variables.md index 4c3e98ca57281..074a7207698d1 100644 --- a/doc/src/manual/variables.md +++ b/doc/src/manual/variables.md @@ -3,21 +3,17 @@ A variable, in Julia, is a name associated (or bound) to a value. It's useful when you want to store a value (that you obtained after some math, for example) for later use. For example: -```julia-repl -# Assign the value 10 to the variable x -julia> x = 10 +```jldoctest +julia> x = 10 # Assign the value 10 to the variable x 10 -# Doing math with x's value -julia> x + 1 +julia> x + 1 # Doing math with x's value 11 -# Reassign x's value -julia> x = 1 + 1 +julia> x = 1 + 1 # Reassign x's value 2 -# You can assign values of other types, like strings of text -julia> x = "Hello World!" +julia> x = "Hello World!" # You can assign values of other types, like strings of text "Hello World!" ``` @@ -125,7 +121,7 @@ it from `+ ᵃx` where `ᵃx` is the variable name. A particular class of variable names is one that contains only underscores. These identifiers are write-only. I.e. they can only be assigned values, which are immediately discarded, and their values cannot be used in any way. -```julia-repl +```jldoctest julia> x, ___ = size([2 2; 1 1]) (2, 2) @@ -138,12 +134,18 @@ ERROR: syntax: all-underscore identifiers are write-only and their values cannot The only explicitly disallowed names for variables are the names of the built-in [Keywords](@ref Keywords): -```julia-repl +```jldoctest julia> else = false -ERROR: syntax: unexpected "else" +ERROR: ParseError: +# Error @ none:1:1 +else = false +└──┘ ── invalid identifier julia> try = "No" -ERROR: syntax: unexpected "=" +ERROR: ParseError: +# Error @ none:1:1 +try = "No" +└────────┘ ── try without catch or finally ``` Some Unicode characters are considered to be equivalent in identifiers. diff --git a/stdlib/Dates/src/conversions.jl b/stdlib/Dates/src/conversions.jl index edb521d31f831..a68b2a85e9c87 100644 --- a/stdlib/Dates/src/conversions.jl +++ b/stdlib/Dates/src/conversions.jl @@ -85,7 +85,7 @@ Return a `DateTime` corresponding to the user's system time as UTC/GMT. For other time zones, see the TimeZones.jl package. # Examples -```julia +```jldoctest; filter = r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}" => "2023-01-04T10:52:24.864" julia> now(UTC) 2023-01-04T10:52:24.864 ``` diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 3e8cb00f7787f..bd3df8e54f194 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -196,7 +196,7 @@ If `rng` is not specified, it defaults to seeding the state of the shared task-local generator. # Examples -```julia-repl +```jldoctest; filter = r"(true|false)" julia> Random.seed!(1234); julia> x1 = rand(2) diff --git a/stdlib/Sockets/src/addrinfo.jl b/stdlib/Sockets/src/addrinfo.jl index db03cdb7c1dc8..8e12a41f08e3b 100644 --- a/stdlib/Sockets/src/addrinfo.jl +++ b/stdlib/Sockets/src/addrinfo.jl @@ -129,7 +129,7 @@ Uses the operating system's underlying getaddrinfo implementation, which may do a DNS lookup. # Examples -```jldoctest +```jldoctest; filter = r"(ip\\"::1\\"|ERROR: DNSError:.*|Stacktrace:(\\n \\[0-9]+\\].*)*)" julia> getaddrinfo("localhost", IPv6) ip"::1" From 58e20a1999a6b2e774ba914fd12b8f5812062822 Mon Sep 17 00:00:00 2001 From: cschen Date: Thu, 19 Jun 2025 20:17:12 +0200 Subject: [PATCH 430/662] adds the `nth` function for iterables (#56580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hi, I've turned the open ended issue #54454 into an actual PR. Tangentially related to #10092 ? This PR introduces the `nth(itr, n)` function to iterators to give a `getindex` type of behaviour. I've tried my best to optimize as much as possible by specializing on different types of iterators. In the spirit of iterators any OOB access returns `nothing`. (edit: instead of throwing an error, i.e. `first(itr, n)` and `last(itr, n)`) here is the comparison of running the testsuite (~22 different iterators) using generic `nth` and specialized `nth`: ```julia @btime begin for (itr, n, _) in $testset _fallback_nth(itr, n) end end 117.750 μs (366 allocations: 17.88 KiB) @btime begin for (itr, n, _) in $testset nth(itr, n) end end 24.250 μs (341 allocations: 16.70 KiB) ``` --------- Co-authored-by: adienes <51664769+adienes@users.noreply.github.com> Co-authored-by: Steven G. Johnson Co-authored-by: Dilum Aluthge --- NEWS.md | 1 + base/iterators.jl | 84 +++++++++++++++++++++++++++++++++++++++++++++-- test/iterators.jl | 59 +++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1e4e24b67a4db..55b5a09d5b525 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ New language features --------------------- - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) + - New `nth` function to access the `n`-th element of a generic iterable. ([#56580]) - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. ([JuliaLang/JuliaSyntax.jl#525], [#57143]) diff --git a/base/iterators.jl b/base/iterators.jl index 4b956073f0c04..53d7b28316ee1 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -16,7 +16,7 @@ using .Base: (:), |, +, -, *, !==, !, ==, !=, <=, <, >, >=, =>, missing, any, _counttuple, eachindex, ntuple, zero, prod, reduce, in, firstindex, lastindex, tail, fieldtypes, min, max, minimum, zero, oneunit, promote, promote_shape, LazyString, - afoldl + afoldl, mod1 using .Core using Core: @doc @@ -32,7 +32,7 @@ import Base: getindex, setindex!, get, iterate, popfirst!, isdone, peek, intersect -export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition +export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition, nth public accumulate, filter, map, peel, reverse, Stateful """ @@ -991,6 +991,7 @@ end reverse(it::Cycle) = Cycle(reverse(it.xs)) last(it::Cycle) = last(it.xs) + # Repeated - repeat an object infinitely many times struct Repeated{O} @@ -1602,4 +1603,83 @@ end # be the same as the keys, so this is a valid optimization (see #51631) pairs(s::AbstractString) = IterableStatePairs(s) +""" + nth(itr, n::Integer) + +Get the `n`th element of an iterable collection. Throw a `BoundsError`[@ref] if not existing. +Will advance any `Stateful`[@ref] iterator. + +See also: [`first`](@ref), [`last`](@ref) + +# Examples +```jldoctest +julia> Iterators.nth(2:2:10, 4) +8 + +julia> Iterators.nth(reshape(1:30, (5,6)), 6) +6 + +julia> stateful = Iterators.Stateful(1:10); Iterators.nth(stateful, 7) +7 + +julia> first(stateful) +8 +``` +""" +nth(itr, n::Integer) = _nth(IteratorSize(itr), itr, n) +nth(itr::Cycle{I}, n::Integer) where I = _nth(IteratorSize(I), itr, n) +nth(itr::Flatten{Take{Repeated{O}}}, n::Integer) where O = _nth(IteratorSize(O), itr, n) +@propagate_inbounds nth(itr::AbstractArray, n::Integer) = itr[begin + n - 1] + +function _nth(::Union{HasShape, HasLength}, itr::Cycle{I}, n::Integer) where {I} + N = length(itr.xs) + N == 0 && throw(BoundsError(itr, n)) + + # prevents wrap around behaviour and inherit the error handling + return nth(itr.xs, n > 0 ? mod1(n, N) : n) +end + +# Flatten{Take{Repeated{O}}} is the actual type of an Iterators.cycle(iterable::O, m) iterator +function _nth(::Union{HasShape, HasLength}, itr::Flatten{Take{Repeated{O}}}, n::Integer) where {O} + cycles = itr.it.n + torepeat = itr.it.xs.x + k = length(torepeat) + (n > k*cycles || k == 0) && throw(BoundsError(itr, n)) + + # prevent wrap around behaviour and inherit the error handling + return nth(torepeat, n > 0 ? mod1(n, k) : n) +end + +function _nth(::IteratorSize, itr, n::Integer) + # unrolled version of `first(drop)` + n > 0 || throw(BoundsError(itr, n)) + y = iterate(itr) + for _ in 1:n-1 + y === nothing && break + y = iterate(itr, y[2]) + end + y === nothing && throw(BoundsError(itr, n)) + y[1] +end +""" + nth(n::Integer) + +Return a function that gets the `n`-th element from any iterator passed to it. +Equivalent to `Base.Fix2(nth, n)` or `itr -> nth(itr, n)`. + +See also: [`nth`](@ref), [`Base.Fix2`](@ref) +# Examples +```jldoctest +julia> fifth_element = Iterators.nth(5) +(::Base.Fix2{typeof(Base.Iterators.nth), Int64}) (generic function with 2 methods) + +julia> fifth_element(reshape(1:30, (5,6))) +5 + +julia> map(fifth_element, ("Willis", "Jovovich", "Oldman")) +('i', 'v', 'a') +``` +""" +nth(n::Integer) = Base.Fix2(nth, n) + end diff --git a/test/iterators.jl b/test/iterators.jl index a6ab4720c0d0c..cebabf9cc07fc 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -1139,6 +1139,65 @@ end end end +@testset "nth" begin + + Z = Array{Int,0}(undef) + Z[] = 17 + it_result_pairs = Dict( + (Z, 1) => 17, + (collect(1:100), 23) => 23, + (10:6:1000, 123) => 10 + 6 * 122, + ("∀ϵ>0", 3) => '>', + ((1, 3, 5, 10, 78), 2) => 3, + (reshape(1:30, (5, 6)), 21) => 21, + (3, 1) => 3, + (true, 1) => true, + ('x', 1) => 'x', + (4 => 5, 2) => 5, + (view(Z), 1) => 17, + (view(reshape(1:30, (5, 6)), 2:4, 2:6), 10) => 22, + ((x^2 for x in 1:10), 9) => 81, + (Iterators.Filter(isodd, 1:10), 3) => 5, + (Iterators.flatten((1:10, 50:60)), 15) => 54, + (pairs(50:60), 7) => 7 => 56, + (zip(1:10, 21:30, 51:60), 6) => (6, 26, 56), + (Iterators.product(1:3, 10:12), 3) => (3, 10), + (Iterators.repeated(3.14159, 5), 4) => 3.14159, + ((a=2, b=3, c=5, d=7, e=11), 4) => 7, + (Iterators.cycle(collect(1:100)), 9999) => 99, + (Iterators.cycle([1, 2, 3, 4, 5], 5), 25) => 5, + (Iterators.cycle("String", 10), 16) => 'i', + (Iterators.cycle(((),)), 1000) => () + ) + + + @testset "iter: $IT" for (IT, n) in keys(it_result_pairs) + @test it_result_pairs[(IT, n)] == nth(IT, n) + @test_throws BoundsError nth(IT, -42) + + IT isa Iterators.Cycle && continue # cycles are infinite so never OOB + @test_throws BoundsError nth(IT, 999999999) + end + + empty_cycle = Iterators.cycle([]) + @test_throws BoundsError nth(empty_cycle, 42) + + # test the size unknown branch for cycles + # only generate odd numbers so we know the actual length + # but the iterator is still SizeUnknown() + it_size_unknown = Iterators.filter(isodd, 1:2:10) + @test Base.IteratorSize(it_size_unknown) isa Base.SizeUnknown + @test length(collect(it_size_unknown)) == 5 + + cycle_size_unknown = Iterators.cycle(it_size_unknown) + finite_cycle_size_unknown = Iterators.cycle(it_size_unknown, 5) + @test nth(cycle_size_unknown, 2) == 3 + @test nth(cycle_size_unknown, 20) == 9 # mod1(20, 5) = 5, wraps 4 times + @test nth(finite_cycle_size_unknown, 2) == 3 + @test nth(finite_cycle_size_unknown, 20) == 9 + @test_throws BoundsError nth(finite_cycle_size_unknown, 30) # only wraps 5 times, max n is 5 * 5 = 25 +end + @testset "Iterators docstrings" begin @test isempty(Docs.undocumented_names(Iterators)) end From 098a4f8dd2f3c739b1f031dc2e387c023b1ae8b6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 19 Jun 2025 15:17:09 -0400 Subject: [PATCH 431/662] refine IR model queries (#58661) - `jl_isa_ast_node` was missing `enter`/`leave` nodes. - `Core.IR` exports mistakenly included a function `memoryref`. - `Base.IR`, and `quoted` were not public or documented. - Add julia function `isa_ast_node` to improve accuracy of `quoted`. - Change `==` on AST nodes to check egal equality of any constants in the IR / AST, and make hashing consistent with that change. This helpfully allows determining that `x + 1` and `x + 1.0` are not equivalent, exchangeable operations. If you need to compare any two objects for semantic equality, you may need to first wrap them with `x = Base.isa_ast_node(x) ? x : QuoteNode(x)` to resolve the ambiguity of whether the comparison is of the semantics or value. - Handle `undef` fields in Phi/PhiC node equality and hashing --- Compiler/src/ssair/slot2ssa.jl | 2 +- base/boot.jl | 7 +- base/essentials.jl | 2 +- base/expr.jl | 130 +++++++++++++++++++++++++++--- base/hashing.jl | 21 +++++ base/public.jl | 5 ++ base/timing.jl | 2 +- doc/src/devdocs/builtins.md | 2 + doc/src/manual/metaprogramming.md | 8 ++ src/ast.c | 13 +-- test/core.jl | 24 +++++- test/hashing.jl | 8 +- 12 files changed, 200 insertions(+), 24 deletions(-) diff --git a/Compiler/src/ssair/slot2ssa.jl b/Compiler/src/ssair/slot2ssa.jl index d9ea2539f0ffc..16a964b4d72f1 100644 --- a/Compiler/src/ssair/slot2ssa.jl +++ b/Compiler/src/ssair/slot2ssa.jl @@ -8,7 +8,7 @@ end SlotInfo() = SlotInfo(Int[], Int[], false) function scan_entry!(result::Vector{SlotInfo}, idx::Int, @nospecialize(stmt)) - # NewVarNodes count as defs for the purpose + # NewvarNodes count as defs for the purpose # of liveness analysis (i.e. they kill use chains) if isa(stmt, NewvarNode) result[slot_id(stmt.slot)].any_newvar = true diff --git a/base/boot.jl b/base/boot.jl index 02b1e83739335..f1ef8bd37a276 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -590,6 +590,7 @@ const undef = UndefInitializer() # empty vector constructor (self::Type{GenericMemory{kind,T,addrspace}})() where {T,kind,addrspace} = self(undef, 0) +# memoryref is simply convenience wrapper function around memoryrefnew memoryref(mem::GenericMemory) = memoryrefnew(mem) memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(memoryrefnew(mem), Int(i), @_boundscheck) memoryref(ref::GenericMemoryRef, i::Integer) = memoryrefnew(ref, Int(i), @_boundscheck) @@ -744,17 +745,19 @@ let end # module providing the IR object model +# excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode) +# any type beyond these is self-quoting (see also Base.is_ast_node) module IR export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode, NewvarNode, SSAValue, SlotNumber, Argument, PiNode, PhiNode, PhiCNode, UpsilonNode, DebugInfo, - Const, PartialStruct, InterConditional, EnterNode, memoryref + Const, PartialStruct, InterConditional, EnterNode using Core: CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode, NewvarNode, SSAValue, SlotNumber, Argument, PiNode, PhiNode, PhiCNode, UpsilonNode, DebugInfo, - Const, PartialStruct, InterConditional, EnterNode, memoryref + Const, PartialStruct, InterConditional, EnterNode end # module IR diff --git a/base/essentials.jl b/base/essentials.jl index fec8ee89f9128..820cce6839f13 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, memoryrefnew, memoryrefget, memoryrefset! +using Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, memoryref, memoryrefnew, memoryrefget, memoryrefset! const Callable = Union{Function,Type} diff --git a/base/expr.jl b/base/expr.jl index b5f7bd8188d50..b44c9336024e5 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -60,7 +60,8 @@ function copy(x::PhiCNode) return PhiCNode(new_values) end -# copy parts of an AST that the compiler mutates +# copy parts of an IR that the compiler mutates +# (this is not a general-purpose copy for an Expr AST) function copy_exprs(@nospecialize(x)) if isa(x, Expr) return copy(x) @@ -91,10 +92,86 @@ function copy(c::CodeInfo) return cnew end +function isequal_exprarg(@nospecialize(x), @nospecialize(y)) + x isa typeof(y) || return false + x === y && return true + # c.f. list of types in copy_expr also + if x isa Expr + x == (y::Expr) && return true + elseif x isa QuoteNode + x == (y::QuoteNode) && return true + elseif x isa PhiNode + x == (y::PhiNode) && return true + elseif x isa PhiCNode + x == (y::PhiCNode) && return true + elseif x isa CodeInfo + x == (y::CodeInfo) && return true + end + return false +end + + +function isequal_exprargs(x::Array{Any,1}, y::Array{Any,1}) + l = length(x) + l == length(y) || return false + for i = 1:l + if !isassigned(x, i) + # phi and phic values are permitted to be undef + isassigned(y, i) && return false + else + isassigned(y, i) || return false + isequal_exprarg(x[i], y[i]) || return false + end + end + return true +end + +# define == such that == inputs to parsing (including line numbers) yield == outputs from lowering (including all metadata) +# (aside from cases where parsing just returns a number, which are ambiguous here) +==(x::Expr, y::Expr) = x.head === y.head && isequal_exprargs(x.args, y.args) + +==(x::QuoteNode, y::QuoteNode) = isequal_exprarg(x.value, y.value) + +==(stmt1::Core.PhiNode, stmt2::Core.PhiNode) = isequal(stmt1.edges, stmt2.edges) && isequal_exprargs(stmt1.values, stmt2.values) + +==(stmt1::Core.PhiCNode, stmt2::Core.PhiCNode) = isequal_exprargs(stmt1.values, stmt2.values) + +function ==(stmt1::CodeInfo, stmt2::CodeInfo) + for i in 1:nfields(stmt1) + if !isdefined(stmt1, i) + isdefined(stmt2, i) && return false + else + isdefined(stmt2, i) || return false + f1 = getfield(stmt1, i) + f2 = getfield(stmt2, i) + f1 isa typeof(f2) || return false + if f1 isa Vector{Any} + # code or types vectors + isequal_exprargs(f1, f2::Vector{Any}) || return false + elseif f1 isa DebugInfo + f1 == f2::DebugInfo || return false + elseif f1 isa Vector + # misc data + l = length(f1) + l == length(f2::Vector) || return false + for i = 1:l + f1[i] === f2[i] || return false + end + else + # misc fields + f1 === f2 || return false + end + end + end + return true +end -==(x::Expr, y::Expr) = x.head === y.head && isequal(x.args, y.args) -==(x::QuoteNode, y::QuoteNode) = isequal(x.value, y.value) -==(stmt1::Core.PhiNode, stmt2::Core.PhiNode) = stmt1.edges == stmt2.edges && stmt1.values == stmt2.values +function ==(x::DebugInfo, y::DebugInfo) + for i in 1:nfields(x) + getfield(x, i) == getfield(y, i) || return false + end + return true +end """ macroexpand(m::Module, x; recursive=true) @@ -1662,14 +1739,45 @@ end is_meta_expr_head(head::Symbol) = head === :boundscheck || head === :meta || head === :loopinfo is_meta_expr(@nospecialize x) = isa(x, Expr) && is_meta_expr_head(x.head) -function is_self_quoting(@nospecialize(x)) - return isa(x,Number) || isa(x,AbstractString) || isa(x,Tuple) || isa(x,Type) || - isa(x,Char) || x === nothing || isa(x,Function) -end +""" + isa_ast_node(x) -function quoted(@nospecialize(x)) - return is_self_quoting(x) ? x : QuoteNode(x) -end +Return false if `x` is not interpreted specially by any of inference, lowering, +or codegen as either an AST or IR special form. +""" +function isa_ast_node(@nospecialize x) + # c.f. Core.IR module, augmented with AST types + return x isa NewvarNode || + x isa CodeInfo || + x isa LineNumberNode || + x isa GotoNode || + x isa GotoIfNot || + x isa EnterNode || + x isa ReturnNode || + x isa SSAValue || + x isa SlotNumber || + x isa Argument || + x isa QuoteNode || + x isa GlobalRef || + x isa Symbol || + x isa PiNode || + x isa PhiNode || + x isa PhiCNode || + x isa UpsilonNode || + x isa Expr +end + +is_self_quoting(@nospecialize(x)) = !isa_ast_node(x) + +""" + quoted(x) + +Return `x` made safe for inserting as a constant into IR. Note that this does +not make it safe for inserting into an AST, since eval will sometimes copy some +types of AST object inside, and even may sometimes evaluate and interpolate any +`\$` inside, depending on the context. +""" +quoted(@nospecialize(x)) = isa_ast_node(x) ? QuoteNode(x) : x # Implementation of generated functions function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) diff --git a/base/hashing.jl b/base/hashing.jl index 848b69fee7e23..1b323f2e9097e 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -232,11 +232,32 @@ end ## symbol & expression hashing ## if UInt === UInt64 + # conservatively hash using == equality of all of the data, even though == often uses === internally hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x83c7900696d26dc6)) hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x2c97bf8b3de87020) + hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x2c97bf8b3de87020)) + hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x2c97bf8b3de87020) else hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x469d72af)) hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x469d72af) + hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x469d72af)) + hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x469d72af) +end + +function hash(x::CodeInfo, h::UInt) + h ⊻= UInt === UInt64 ? 0x2c97bf8b3de87020 : 0x469d72af + for i in 1:nfields(x) + h = hash(isdefined(x, i) ? getfield(x, i) : missing, h) + end + return h +end + +function hash(x::DebugInfo, h::UInt) + h ⊻= UInt === UInt64 ? 0x2c97bf8b3de87020 : 0x469d72af + for i in 1:nfields(x) + h = hash(getfield(x, i), h) + end + return h end hash(x::Symbol) = objectid(x) diff --git a/base/public.jl b/base/public.jl index 3308c58ff1780..217d91b615848 100644 --- a/base/public.jl +++ b/base/public.jl @@ -68,6 +68,11 @@ public ispublic, remove_linenums!, +# AST handling + IR, + isa_ast_node, + quoted, + # Operators operator_associativity, operator_precedence, diff --git a/base/timing.jl b/base/timing.jl index 9e3a4cf128413..998103d1e78bc 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -495,7 +495,7 @@ function is_simply_call(@nospecialize ex) for a in ex.args a isa QuoteNode && continue a isa Symbol && continue - Base.is_self_quoting(a) && continue + isa_ast_node(a) || continue return false end return true diff --git a/doc/src/devdocs/builtins.md b/doc/src/devdocs/builtins.md index bb6b1ae0d82a2..2f1651b2b2518 100644 --- a/doc/src/devdocs/builtins.md +++ b/doc/src/devdocs/builtins.md @@ -41,4 +41,6 @@ Core.get_binding_type Core.IntrinsicFunction Core.Intrinsics Core.IR +Base.quoted +Base.isa_ast_node ``` diff --git a/doc/src/manual/metaprogramming.md b/doc/src/manual/metaprogramming.md index ee252d8c9fa2e..38d688f427078 100644 --- a/doc/src/manual/metaprogramming.md +++ b/doc/src/manual/metaprogramming.md @@ -363,6 +363,14 @@ QuoteNode `QuoteNode` can also be used for certain advanced metaprogramming tasks. +Note that while it does not support `$`, it also does not prevent it, nor does +it preserve the identity of the wrapped object: + +```jldoctest +julia> b = 2; eval(Expr(:quote, QuoteNode(Expr(:$, :b)))) +:($(QuoteNode(2))) +``` + ### Evaluating expressions Given an expression object, one can cause Julia to evaluate (execute) it at global scope using diff --git a/src/ast.c b/src/ast.c index fdb29349a0db6..04c87d220f409 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1047,14 +1047,15 @@ int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT // Utility function to return whether `e` is any of the special AST types or // will always evaluate to itself exactly unchanged. This corresponds to -// `is_self_quoting` in Core.Compiler utilities. -int jl_is_ast_node(jl_value_t *e) JL_NOTSAFEPOINT +// `isa_ast_node` in Core.Compiler utilities. +int jl_isa_ast_node(jl_value_t *e) JL_NOTSAFEPOINT { return jl_is_newvarnode(e) || jl_is_code_info(e) || jl_is_linenode(e) || jl_is_gotonode(e) || jl_is_gotoifnot(e) + || jl_is_enternode(e) || jl_is_returnnode(e) || jl_is_ssavalue(e) || jl_is_slotnumber(e) @@ -1069,9 +1070,10 @@ int jl_is_ast_node(jl_value_t *e) JL_NOTSAFEPOINT || jl_is_expr(e); } -static int is_self_quoting_expr(jl_expr_t *e) JL_NOTSAFEPOINT +static int is_self_escaping_expr(jl_expr_t *e) JL_NOTSAFEPOINT { return (e->head == jl_inert_sym || + e->head == jl_leave_sym || e->head == jl_core_sym || e->head == jl_line_sym || e->head == jl_lineinfo_sym || @@ -1089,12 +1091,13 @@ int need_esc_node(jl_value_t *e) JL_NOTSAFEPOINT || jl_is_ssavalue(e) || jl_is_slotnumber(e) || jl_is_argument(e) + || jl_is_enternode(e) || jl_is_quotenode(e)) return 0; if (jl_is_expr(e)) - return !is_self_quoting_expr((jl_expr_t*)e); + return !is_self_escaping_expr((jl_expr_t*)e); // note: jl_is_globalref(e) is not included here, since we care a little about about having a line number for it - return jl_is_ast_node(e); + return jl_isa_ast_node(e); } static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule, jl_module_t **ctx, jl_value_t **lineinfo, size_t world, int throw_load_error) diff --git a/test/core.jl b/test/core.jl index c84d68bafece9..4abe4f95a2e0b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4679,8 +4679,28 @@ end @test Macro_Yielding_Global_Assignment.x == 2 # issue #15718 -@test :(f($NaN)) == :(f($NaN)) -@test isequal(:(f($NaN)), :(f($NaN))) +function compare_test(x, y) + lx = Meta.lower(@__MODULE__, x) + ly = Meta.lower(@__MODULE__, y) + if isequal(x, y) + @test x == y + @test hash(x) == hash(y) + @test isequal(lx, ly) + @test lx == ly + @test hash(lx) == hash(ly) + true + else + @test x != y + @test !isequal(lx, ly) + @test lx != ly + false + end +end +@test compare_test(:(f($NaN)), :(f($NaN))) +@test !compare_test(:(1 + (1 * 1)), :(1 + (1 * 1.0))) +@test compare_test(:(1 + (1 * $NaN)), :(1 + (1 * $NaN))) +@test compare_test(QuoteNode(NaN), QuoteNode(NaN)) +@test !compare_test(QuoteNode(1), QuoteNode(1.0)) # PR #16011 Make sure dead code elimination doesn't delete push and pop # of metadata diff --git a/test/hashing.jl b/test/hashing.jl index 4e05b8a0102eb..1af73e45fc541 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -179,8 +179,14 @@ end @test hash([1,2]) == hash(view([1,2,3,4],1:2)) let a = QuoteNode(1), b = QuoteNode(1.0) - @test (hash(a)==hash(b)) == (a==b) + @test hash(a) == hash(b) + @test a != b end +let a = QuoteNode(:(1 + 2)), b = QuoteNode(:(1 + 2)) + @test hash(a) == hash(b) + @test a == b +end + let a = Expr(:block, Core.SlotNumber(1)), b = Expr(:block, Core.SlotNumber(1)), From 089810cd490d0f040d1ea5d748243c56307d6201 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 19 Jun 2025 15:17:27 -0400 Subject: [PATCH 432/662] fix showing types after removing using Core (#58773) PR #57357 changed the default using list, but only changed some of the places where the `show` code handled that. This led to duplicate (confusing) printing, since both Core. and Base. prefixes are dropped. Fix #58772 --- base/show.jl | 10 +++++----- test/show.jl | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/base/show.jl b/base/show.jl index 14518be0f153e..73a3a0fcad66f 100644 --- a/base/show.jl +++ b/base/show.jl @@ -538,7 +538,7 @@ function active_module() return invokelatest(active_module, active_repl)::Module end -module UsesCoreAndBaseOnly +module UsesBaseOnly end function show_function(io::IO, f::Function, compact::Bool, fallback::Function) @@ -550,8 +550,8 @@ function show_function(io::IO, f::Function, compact::Bool, fallback::Function) elseif isdefined(fname, :module) && isdefinedglobal(fname.module, fname.singletonname) && isconst(fname.module, fname.singletonname) && getglobal(fname.module, fname.singletonname) === f # this used to call the removed internal function `is_exported_from_stdlib`, which effectively - # just checked for exports from Core and Base. - mod = get(io, :module, UsesCoreAndBaseOnly) + # just checked for exports from Base. + mod = get(io, :module, UsesBaseOnly) if !(isvisible(fname.singletonname, fname.module, mod) || fname.module === mod) print(io, fname.module, ".") end @@ -825,7 +825,7 @@ function make_typealiases(@nospecialize(x::Type)) Any === x && return aliases, Union{} x <: Tuple && return aliases, Union{} mods = modulesof!(Set{Module}(), x) - Core in mods && push!(mods, Base) + replace!(mods, Core=>Base) vars = Dict{Symbol,TypeVar}() xenv = UnionAll[] each = Any[] @@ -843,7 +843,7 @@ function make_typealiases(@nospecialize(x::Type)) ti === Union{} && continue # make sure this alias wasn't from an unrelated part of the Union mod2 = modulesof!(Set{Module}(), alias) - mod in mod2 || (mod === Base && Core in mods) || continue + mod in mod2 || (mod === Base && Core in mod2) || continue env = env::SimpleVector applied = alias if !isempty(env) diff --git a/test/show.jl b/test/show.jl index 063deaefb376e..7922215ee789c 100644 --- a/test/show.jl +++ b/test/show.jl @@ -2488,6 +2488,7 @@ end @test string(Union{M37012.SimpleU, Nothing, T} where T) == "Union{Nothing, $(curmod_prefix)M37012.SimpleU, T} where T" @test string(Union{AbstractVector{T}, T} where T) == "Union{AbstractVector{T}, T} where T" @test string(Union{AbstractVector, T} where T) == "Union{AbstractVector, T} where T" +@test string(Union{Array, Memory}) == "Union{Array, Memory}" @test sprint(show, :(./)) == ":((./))" @test sprint(show, :((.|).(.&, b))) == ":((.|).((.&), b))" From d47d4ecdc03fc870e0994df6a4e6c95fd7c503db Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 20 Jun 2025 14:39:59 +0900 Subject: [PATCH 433/662] inform compiler about local variable definedness (#58778) JET's new analysis pass now detects local variables that may be undefined, which has revealed such issues in several functions within Base (JuliaLang/julia#58762). This commit addresses local variables whose definedness the compiler cannot properly determine, primarily in functions reachable from JET's test suite. No functional changes are made. --- base/logging/logging.jl | 10 +++++++--- base/partr.jl | 8 +++----- base/ryu/shortest.jl | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/base/logging/logging.jl b/base/logging/logging.jl index b43b7d9faa594..25f4dbe4902be 100644 --- a/base/logging/logging.jl +++ b/base/logging/logging.jl @@ -411,9 +411,13 @@ function logmsg_code(_module, file, line, level, message, exs...) end line = $(log_data._line) local msg, kwargs - $(logrecord) && $handle_message_nothrow( - logger, level, msg, _module, group, id, file, line; - kwargs...) + if $(logrecord) + @assert @isdefined(msg) "Assertion to tell the compiler about the definedness of this variable" + @assert @isdefined(kwargs) "Assertion to tell the compiler about the definedness of this variable" + $handle_message_nothrow( + logger, level, msg, _module, group, id, file, line; + kwargs...) + end end end end diff --git a/base/partr.jl b/base/partr.jl index 6053a584af5ba..d488330f0c87e 100644 --- a/base/partr.jl +++ b/base/partr.jl @@ -107,7 +107,6 @@ function multiq_sift_down(heap::taskheap, idx::Int32) end end - function multiq_size(tpid::Int8) nt = UInt32(Threads._nthreads_in_pool(tpid)) tp = tpid + 1 @@ -138,7 +137,6 @@ function multiq_size(tpid::Int8) return heap_p end - function multiq_insert(task::Task, priority::UInt16) tpid = ccall(:jl_get_task_threadpoolid, Int8, (Any,), task) @assert tpid > -1 @@ -171,10 +169,8 @@ function multiq_insert(task::Task, priority::UInt16) return true end - function multiq_deletemin() - local rn1, rn2 - local prio1, prio2 + local rn1::UInt32 tid = Threads.threadid() tp = ccall(:jl_threadpoolid, Int8, (Int16,), tid-1) + 1 @@ -208,6 +204,8 @@ function multiq_deletemin() end end + @assert @isdefined(rn1) "Assertion to tell the compiler about the definedness of this variable" + heap = tpheaps[rn1] task = heap.tasks[1] if ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) == 0 diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index 37024f18b4df1..b4ba8255e0b8a 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -196,6 +196,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. e10 = 0 if maxsignif !== nothing && b > maxsignif + roundup = false b_allzero = true # reduce to max significant digits while true From f855e4c86939f713f436b2df1b712767798eaea3 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Fri, 20 Jun 2025 14:46:59 +0200 Subject: [PATCH 434/662] better effects for `iterate` for `Memory` and `Array` (#58755) --- base/array.jl | 2 +- base/genericmemory.jl | 7 ++++++- test/abstractarray.jl | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 40ecb18c6e828..5c9b37f887ea8 100644 --- a/base/array.jl +++ b/base/array.jl @@ -904,7 +904,7 @@ end ## Iteration ## -iterate(A::Array, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing) +iterate(A::Array, i=1) = (@inline; _iterate_array(A, i)) ## Indexing: getindex ## diff --git a/base/genericmemory.jl b/base/genericmemory.jl index 2a33336c0aad6..1b45ba23bcded 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -225,7 +225,12 @@ Memory{T}(x::AbstractArray{S,1}) where {T,S} = copyto_axcheck!(Memory{T}(undef, ## Iteration ## -iterate(A::Memory, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing) +function _iterate_array(A::Union{Memory, Array}, i::Int) + @inline + (i - 1)%UInt < length(A)%UInt ? (A[i], i + 1) : nothing +end + +iterate(A::Memory, i=1) = (@inline; _iterate_array(A, i)) ## Indexing: getindex ## diff --git a/test/abstractarray.jl b/test/abstractarray.jl index bbac7812174da..ad821855e573a 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -2287,6 +2287,20 @@ end end end +@testset "effect inference for `iterate` for `Array` and for `Memory`" begin + for El ∈ (Float32, Real, Any) + for Arr ∈ (Memory{El}, Array{El, 0}, Vector{El}, Matrix{El}, Array{El, 3}) + effects = Base.infer_effects(iterate, Tuple{Arr, Int}) + @test Base.Compiler.is_effect_free(effects) + @test Base.Compiler.is_terminates(effects) + @test Base.Compiler.is_notaskstate(effects) + @test Base.Compiler.is_noub(effects) + @test Base.Compiler.is_nonoverlayed(effects) + @test Base.Compiler.is_nortcall(effects) + end + end +end + @testset "iterate for linear indexing" begin A = [1 2; 3 4] v = view(A, :) From 7b6065e9f25bca60f5ed21532a0163d760a7dbf8 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 20 Jun 2025 12:39:54 -0400 Subject: [PATCH 435/662] Test: Hide REPL internals in backtraces (#58732) --- stdlib/Test/src/Test.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 7ad1c35294be7..d31c1a00f2e53 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -222,7 +222,8 @@ struct Error <: Result end if test_type === :test_error || test_type === :nontest_error bt_str = try # try the latest world for this, since we might have eval'd new code for show - Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) + # Apply REPL backtrace scrubbing to hide REPL internals, similar to how REPL.jl handles it + Base.invokelatest(sprint, Base.show_exception_stack, Base.scrub_repl_backtrace(bt); context=stdout) catch ex "#=ERROR showing exception stack=# " * try From 1d677220708d721a37ba257df423fd3fdd99dfb3 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:02:09 -0700 Subject: [PATCH 436/662] Update docs for various type predicates (#58774) Makes the description for `isdispatchtuple` accurate, adds a docstring for `iskindtype` and `isconcretedispatch`, and adds notes to the docs for `isconcretetype` and `isabstracttype` explaining why they aren't antonyms. --- base/runtime_internals.jl | 98 +++++++++++++++++++++++++++++++++++++-- doc/src/base/base.md | 2 + 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 902ac516f1775..4347c67438577 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -851,10 +851,45 @@ end """ isdispatchtuple(T) -Determine whether type `T` is a tuple of concrete types, -meaning it could appear as a type signature in dispatch -and has no subtypes (or supertypes) which could appear in a call. +Determine whether type `T` is a [`Tuple`](@ref) that could appear as a type +signature in dispatch. For this to be true, every element of the tuple type +must be either: +- [concrete](@ref isconcretetype) but not a [kind type](@ref Base.iskindtype) +- a [`Type{U}`](@ref Type) with no free type variables in `U` + +!!! note + A dispatch tuple is relevant for method dispatch because it has no inhabited + subtypes. + + For example, `Tuple{Int, DataType}` is concrete, but is not a dispatch tuple + because `Tuple{Int, Type{Bool}}` is an inhabited subtype. + + `Tuple{Tuple{DataType}}` *is* a dispatch tuple because `Tuple{DataType}` is + concrete and not a kind; the subtype `Tuple{Tuple{Type{Int}}}` is not + inhabited. + If `T` is not a type, then return `false`. + +# Examples +```jldoctest +julia> isdispatchtuple(Int) +false + +julia> isdispatchtuple(Tuple{Int}) +true + +julia> isdispatchtuple(Tuple{Number}) +false + +julia> isdispatchtuple(Tuple{DataType}) +false + +julia> isdispatchtuple(Tuple{Type{Int}}) +true + +julia> isdispatchtuple(Tuple{Type}) +false +``` """ isdispatchtuple(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & 0x0004) == 0x0004) @@ -900,7 +935,40 @@ function isidentityfree(@nospecialize(t)) return false end +""" + Base.iskindtype(T) + +Determine whether `T` is a kind, that is, the type of a Julia type: +a [`DataType`](@ref), [`Union`](@ref), [`UnionAll`](@ref), +or [`Core.TypeofBottom`](@ref). + +All kinds are [concrete](@ref isconcretetype) because types are Julia values. +""" iskindtype(@nospecialize t) = (t === DataType || t === UnionAll || t === Union || t === typeof(Bottom)) + +""" + Base.isconcretedispatch(T) + +Returns true if `T` is a [concrete type](@ref isconcretetype) that could appear +as an element of a [dispatch tuple](@ref isdispatchtuple). + +See also: [`isdispatchtuple`](@ref). + +# Examples +```jldoctest +julia> Base.isconcretedispatch(Int) +true + +julia> Base.isconcretedispatch(Number) +false + +julia> Base.isconcretedispatch(DataType) +false + +julia> Base.isconcretedispatch(Type{Int}) +false +``` +""" isconcretedispatch(@nospecialize t) = isconcretetype(t) && !iskindtype(t) using Core: has_free_typevars @@ -923,6 +991,16 @@ Determine whether type `T` is a concrete type, meaning it could have direct inst Note that this is not the negation of `isabstracttype(T)`. If `T` is not a type, then return `false`. +!!! note + While concrete types are not [abstract](@ref isabstracttype) and + vice versa, types can be neither concrete nor abstract (for example, + `Vector` (a [`UnionAll`](@ref))). + +!!! note + `T` must be the exact type that would be returned from `typeof`. It is + possible for a type `U` to exist such that `T == U`, `isconcretetype(T)`, + but `!isconcretetype(U)`. + See also: [`isbits`](@ref), [`isabstracttype`](@ref), [`issingletontype`](@ref). # Examples @@ -933,6 +1011,9 @@ false julia> isconcretetype(Complex{Float32}) true +julia> isconcretetype(Vector) +false + julia> isconcretetype(Vector{Complex}) true @@ -944,6 +1025,9 @@ false julia> isconcretetype(Union{Int,String}) false + +julia> isconcretetype(Tuple{T} where T<:Int) +false ``` """ isconcretetype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & 0x0002) == 0x0002) @@ -953,9 +1037,15 @@ isconcretetype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & Determine whether type `T` was declared as an abstract type (i.e. using the `abstract type` syntax). -Note that this is not the negation of `isconcretetype(T)`. If `T` is not a type, then return `false`. +!!! note + While abstract types are not [concrete](@ref isconcretetype) and + vice versa, types can be neither concrete nor abstract (for example, + `Vector` (a [`UnionAll`](@ref))). + +See also: [`isconcretetype`](@ref). + # Examples ```jldoctest julia> isabstracttype(AbstractArray) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 2c34bcb8be9ab..f3eb62b3680d5 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -193,6 +193,7 @@ Base.typeintersect Base.promote_type Base.promote_rule Base.promote_typejoin +Base.iskindtype Base.isdispatchtuple ``` @@ -251,6 +252,7 @@ Base.instances Core.Any Core.Union Union{} +Core.TypeofBottom Core.UnionAll Core.Tuple Core.NTuple From 76d5b14c9c280c52b2c275e6cf449fe1ba7fc8d2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 20 Jun 2025 22:54:05 -0400 Subject: [PATCH 437/662] Test: show context when a let testset errors (#58727) --- stdlib/Test/src/Test.jl | 31 ++++++++++++++----- stdlib/Test/test/runtests.jl | 58 +++++++++++++++++++++++++++++++++++- test/runtests.jl | 4 +-- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index d31c1a00f2e53..8278fed697f7b 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -214,9 +214,10 @@ struct Error <: Result orig_expr::String value::String backtrace::String + context::Union{Nothing, String} source::LineNumberNode - function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode) + function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode, context::Union{Nothing, String}=nothing) if test_type === :test_error bt = scrub_exc_stack(bt, nothing, extract_file(source)) end @@ -249,8 +250,14 @@ struct Error <: Result string(orig_expr), value, bt_str, + context, source) end + + # Internal constructor for creating Error with pre-processed values (used by ContextTestSet) + function Error(test_type::Symbol, orig_expr::String, value::String, backtrace::String, context::Union{Nothing, String}, source::LineNumberNode) + return new(test_type, orig_expr, value, backtrace, context, source) + end end function Base.show(io::IO, t::Error) @@ -268,6 +275,9 @@ function Base.show(io::IO, t::Error) elseif t.test_type === :test_error println(io, " Test threw exception") println(io, " Expression: ", t.orig_expr) + if t.context !== nothing + println(io, " Context: ", t.context) + end # Capture error message and indent to match join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") elseif t.test_type === :test_unbroken @@ -752,13 +762,13 @@ function do_test(result::ExecutionResult, orig_expr) Fail(:test, orig_expr, result.data, value, nothing, result.source, false) else # If the result is non-Boolean, this counts as an Error - Error(:test_nonbool, orig_expr, value, nothing, result.source) + Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing) end else # The predicate couldn't be evaluated without throwing an # exception, so that is an Error and not a Fail @assert isa(result, Threw) - testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source) + testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source, nothing) end isa(testres, Pass) || trigger_test_failure_break(result) record(get_testset(), testres) @@ -771,11 +781,11 @@ function do_broken_test(result::ExecutionResult, orig_expr) value = result.value if isa(value, Bool) if value - testres = Error(:test_unbroken, orig_expr, value, nothing, result.source) + testres = Error(:test_unbroken, orig_expr, value, nothing, result.source, nothing) end else # If the result is non-Boolean, this counts as an Error - testres = Error(:test_nonbool, orig_expr, value, nothing, result.source) + testres = Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing) end end record(get_testset(), testres) @@ -1109,6 +1119,13 @@ function record(c::ContextTestSet, t::Fail) context = t.context === nothing ? context : string(t.context, "\n ", context) record(c.parent_ts, Fail(t.test_type, t.orig_expr, t.data, t.value, context, t.source, t.message_only)) end +function record(c::ContextTestSet, t::Error) + context = string(c.context_name, " = ", c.context) + context = t.context === nothing ? context : string(t.context, "\n ", context) + # Create a new Error with the same data but updated context using internal constructor + new_error = Error(t.test_type, t.orig_expr, t.value, t.backtrace, context, t.source) + record(c.parent_ts, new_error) +end #----------------------------------------------------------------------- @@ -1845,7 +1862,7 @@ function testset_beginend_call(args, tests, source) if is_failfast_error(err) get_testset_depth() > 1 ? rethrow() : failfast_print() else - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing)) end finally copy!(default_rng(), default_rng_orig) @@ -1933,7 +1950,7 @@ function testset_forloop(args, testloop, source) if is_failfast_error(err) get_testset_depth() > 1 ? rethrow() : failfast_print() else - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing)) end end end diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 4a8a25f0089d4..a5d2ca831d673 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -390,7 +390,7 @@ let retval_tests = @testset NoThrowTestSet begin ts = Test.DefaultTestSet("Mock for testing retval of record(::DefaultTestSet, ::T <: Result) methods") pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock")) @test Test.record(ts, pass_mock) isa Test.Pass - error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock")) + error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock"), nothing) @test Test.record(ts, error_mock; print_result=false) isa Test.Error fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false) @test Test.record(ts, fail_mock; print_result=false) isa Test.Fail @@ -1892,3 +1892,59 @@ end @test _escape_call(:((==).(x, y))) == (; func=Expr(:., esc(:(==))), args, kwargs, quoted_func=QuoteNode(Expr(:., :(==)))) end end + +@testset "Context display in @testset let blocks" begin + # Mock parent testset that just captures results + struct MockParentTestSet <: Test.AbstractTestSet + results::Vector{Any} + MockParentTestSet() = new([]) + end + Test.record(ts::MockParentTestSet, t) = (push!(ts.results, t); t) + Test.finish(ts::MockParentTestSet) = ts + + @testset "context shown when a context testset fails" begin + mock_parent1 = MockParentTestSet() + ctx_ts1 = Test.ContextTestSet(mock_parent1, :x, 42) + + fail_result = Test.Fail(:test, "x == 99", "42 == 99", "42", nothing, LineNumberNode(1, :test), false) + Test.record(ctx_ts1, fail_result) + + @test length(mock_parent1.results) == 1 + recorded_fail = mock_parent1.results[1] + @test recorded_fail isa Test.Fail + @test recorded_fail.context !== nothing + @test occursin("x = 42", recorded_fail.context) + end + + @testset "context shown when a context testset errors" begin + mock_parent2 = MockParentTestSet() + ctx_ts2 = Test.ContextTestSet(mock_parent2, :x, 42) + + # Use internal constructor to create Error with pre-processed values + error_result = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test)) + Test.record(ctx_ts2, error_result) + + @test length(mock_parent2.results) == 1 + recorded_error = mock_parent2.results[1] + @test recorded_error isa Test.Error + @test recorded_error.context !== nothing + @test occursin("x = 42", recorded_error.context) + + # Context shows up in string representation + error_str = sprint(show, recorded_error) + @test occursin("Context:", error_str) + @test occursin("x = 42", error_str) + + # Multiple variables context + mock_parent3 = MockParentTestSet() + ctx_ts3 = Test.ContextTestSet(mock_parent3, :(x, y), (42, "hello")) + + error_result2 = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test)) + Test.record(ctx_ts3, error_result2) + + recorded_error2 = mock_parent3.results[1] + @test recorded_error2 isa Test.Error + @test recorded_error2.context !== nothing + @test occursin("(x, y) = (42, \"hello\")", recorded_error2.context) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index bd56552377421..00653a55e4a28 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -414,7 +414,7 @@ cd(@__DIR__) do # deserialization errors or something similar. Record this testset as Errored. fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() @@ -423,7 +423,7 @@ cd(@__DIR__) do for test in all_tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1))) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() From 92bcc94af6c3818b66705c17676468de05061631 Mon Sep 17 00:00:00 2001 From: Alexis Montoison <35051714+amontoison@users.noreply.github.com> Date: Sat, 21 Jun 2025 06:52:28 +0000 Subject: [PATCH 438/662] [libblastrampoline_jll] Upgrade to v5.13.1 (#58775) ### Check list Version numbers: - [x] `deps/libblastrampoline.version`: `LIBNAME_VER`, `LIBNAME_BRANCH`, `LIBNAME_SHA1` and `LIBNAME_JLL_VER` - [x] `stdlib/libblastrampoline_jll/Project.toml`: `version` Checksum: - [x] `deps/checksums/libblastrampoline` --- deps/blastrampoline.version | 6 +- deps/checksums/blastrampoline | 76 +++++++++++------------ stdlib/libblastrampoline_jll/Project.toml | 4 +- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/deps/blastrampoline.version b/deps/blastrampoline.version index 1e4a75305a4dd..bb711cfcd67ec 100644 --- a/deps/blastrampoline.version +++ b/deps/blastrampoline.version @@ -4,6 +4,6 @@ BLASTRAMPOLINE_JLL_NAME := libblastrampoline ## source build -BLASTRAMPOLINE_VER := 5.12.0 -BLASTRAMPOLINE_BRANCH=v5.12.0 -BLASTRAMPOLINE_SHA1=b127bc8dd4758ffc064340fff2aef4ead552f386 +BLASTRAMPOLINE_VER := 5.13.1 +BLASTRAMPOLINE_BRANCH=v5.13.1 +BLASTRAMPOLINE_SHA1=f26278e83ddc9035ae7695da597f1a5b26a4c62b diff --git a/deps/checksums/blastrampoline b/deps/checksums/blastrampoline index 9e007f6055cf9..7870242560f34 100644 --- a/deps/checksums/blastrampoline +++ b/deps/checksums/blastrampoline @@ -1,38 +1,38 @@ -blastrampoline-b127bc8dd4758ffc064340fff2aef4ead552f386.tar.gz/md5/395f2035bcb52e886b55ac926a7bf183 -blastrampoline-b127bc8dd4758ffc064340fff2aef4ead552f386.tar.gz/sha512/9ae0fe2ca75dc0b2c784d5b7248caca29ed6d44258743ee2b32827032734757e9078dd6bcdf80a02b042deb5c7ca7b4e5be392be6700efde91427091fb53a03f -libblastrampoline.v5.12.0+0.aarch64-apple-darwin.tar.gz/md5/9a18b39bb575d0112834992043d302c0 -libblastrampoline.v5.12.0+0.aarch64-apple-darwin.tar.gz/sha512/4e406b155149414d3e4fd5db49ab56a87ed13577ebb399eaf8a251692c0b84e639c6e1a4eb20863e2638c31add0241ca916e57f91bb5a4aed07e2c56cc580870 -libblastrampoline.v5.12.0+0.aarch64-linux-gnu.tar.gz/md5/e100e93f0d6a104fc66c9f78a67150c5 -libblastrampoline.v5.12.0+0.aarch64-linux-gnu.tar.gz/sha512/f7e0c379e32d8163dbb4919b77e9637e1b16cf26618b9260222cf985bfab9ca3f36bebccd0e8360af68db925035c82127ba85d46b4a6578961dde6a049c7cf93 -libblastrampoline.v5.12.0+0.aarch64-linux-musl.tar.gz/md5/814a79e8cfe8744ca5a2a722f007fcaa -libblastrampoline.v5.12.0+0.aarch64-linux-musl.tar.gz/sha512/bc886b199500fc4245a95446d4c862fc636711e0875a9d5cf9aef661d819d00324adfd3e037d9c03e274be26034353d033fb041e7608ecef222e1d154f38337d -libblastrampoline.v5.12.0+0.aarch64-unknown-freebsd.tar.gz/md5/9b9a7fe0e45a73009bb9f8044f4a27a2 -libblastrampoline.v5.12.0+0.aarch64-unknown-freebsd.tar.gz/sha512/51d52afb13e326ef4750bdcad800aaf3db2c9e068b4c38bd148e312c63358b2228b81d23626d18b8983534a8a6f24df1b64b4e7121779d2535574ea907bd18ba -libblastrampoline.v5.12.0+0.armv6l-linux-gnueabihf.tar.gz/md5/1b6fd062d133b13e8efc63f08528fb51 -libblastrampoline.v5.12.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/78d525f425ee27068b94b94f89ef44a51ffac9f642ffe66e177434804e59b4ac3ba875190aceee386a8d740f7903e979e5b91f0973138d0fc7753061c6f5f26d -libblastrampoline.v5.12.0+0.armv6l-linux-musleabihf.tar.gz/md5/506be2b7669aa171efcc541388cb5444 -libblastrampoline.v5.12.0+0.armv6l-linux-musleabihf.tar.gz/sha512/2975136376c3f61b8f227676c4e1368d1847d85ff469dddbc0a330635eac77c00072c7544ae4aa9981d16a4ab04d494be54fc951b434a56fbf14003c42626579 -libblastrampoline.v5.12.0+0.armv7l-linux-gnueabihf.tar.gz/md5/99403eae880f52aa97884143e2ca7215 -libblastrampoline.v5.12.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/986dfcf5fe3ac731df3c71eb6b0bf3d7525511952d22cc9128ff35e6fcb330acf69e897aeb97920ebabd1ccccd1dd6ce9b6c16d0dbf661d39a103ce5b477462f -libblastrampoline.v5.12.0+0.armv7l-linux-musleabihf.tar.gz/md5/20adf8d2ef348f5362cb03e1a2780476 -libblastrampoline.v5.12.0+0.armv7l-linux-musleabihf.tar.gz/sha512/95068a3b5bcf17bd5f13373a2730a6508d3992f0aa83a91629527821cf038b9607327843cc44fb72730b63c01d3d70e2eb488eca8f48ed9444d7736f67745d02 -libblastrampoline.v5.12.0+0.i686-linux-gnu.tar.gz/md5/a56f833ad986fc3e9e64e5abdb16915f -libblastrampoline.v5.12.0+0.i686-linux-gnu.tar.gz/sha512/d478b4981dc17afb8aa8625fdbb23139f1c3edaa9aaa179e70d274984a056147b2e65e9f473b007733d094369f448823c33aa95fadd228016ecf9dfbf17f06bb -libblastrampoline.v5.12.0+0.i686-linux-musl.tar.gz/md5/8578119b3b3e84393e6324996e9506aa -libblastrampoline.v5.12.0+0.i686-linux-musl.tar.gz/sha512/b546de6687755ce43680f312008a23a8f9df422603098807f33e2ae969c9e9de0ca32a3319067d4f8fa1f782f21b6465638cd59e4c86fc6261fb4180f0ed116f -libblastrampoline.v5.12.0+0.i686-w64-mingw32.tar.gz/md5/b9e2800b8758d3fa0ac0597f738c399c -libblastrampoline.v5.12.0+0.i686-w64-mingw32.tar.gz/sha512/e0aa0ee2a750cfe702e0bd5861e352f97f433f67444dbc6e5814055fb32f571de318f640ac670c91bad233f8af85f0421daef71b7768a710de5b15febee28b27 -libblastrampoline.v5.12.0+0.powerpc64le-linux-gnu.tar.gz/md5/bab2048857c7c1ba4a6c3d540b9275c6 -libblastrampoline.v5.12.0+0.powerpc64le-linux-gnu.tar.gz/sha512/576026c970b19cc00480d7bb9439933c5bb432eec17def66b22f5c0dfd418bcf75bb10ccfc1b01fef48e8d504ebf953c5f6c63d504713315c43d9579ab5fa2e4 -libblastrampoline.v5.12.0+0.riscv64-linux-gnu.tar.gz/md5/f37e2849a948a8c8c8bfa6055e30909c -libblastrampoline.v5.12.0+0.riscv64-linux-gnu.tar.gz/sha512/89f30d52f1a1dcc0aa38b4b343534b7fadcff12d788f455172c043ea2511c03b2735fdacf8f794a6f62156cb5d82fb0e9e0edd04bb9c57a1ca3e680410456b17 -libblastrampoline.v5.12.0+0.x86_64-apple-darwin.tar.gz/md5/b07c42b602b91bf2229b1a5cfd8e37b3 -libblastrampoline.v5.12.0+0.x86_64-apple-darwin.tar.gz/sha512/ab064dff373826776f9b64a4a77e3418461d53d5119798a5e702967e4ac4f68c58cd8c3c0cc01bda3edeb613cf50b9d3171d9141c91ff9ef3a2c88a8e8f00a37 -libblastrampoline.v5.12.0+0.x86_64-linux-gnu.tar.gz/md5/c37b01242012e51e124711d5ad10cf97 -libblastrampoline.v5.12.0+0.x86_64-linux-gnu.tar.gz/sha512/3f9015bec4aaddc677cb3f3aebd432db8bad89b3f6e563634a37569afeb9fb0efa4f214166c984c2c1926831d5cd79fcd4d605d40675e0d1a7e494a76c066f02 -libblastrampoline.v5.12.0+0.x86_64-linux-musl.tar.gz/md5/c24e440a1757a45f087a2e1ac649fb45 -libblastrampoline.v5.12.0+0.x86_64-linux-musl.tar.gz/sha512/824b930d50df929fd22ead6dffad06593d2aad9fcb149f07f1c2f6d4b7b34911e89c2be5a1e9b8ad5ad8292ac29f9e5dbe6d7bb205d2b207432ade61ae5f8b68 -libblastrampoline.v5.12.0+0.x86_64-unknown-freebsd.tar.gz/md5/5721328a24473cefbb3e77ba85e46922 -libblastrampoline.v5.12.0+0.x86_64-unknown-freebsd.tar.gz/sha512/3537ea491828492f1cb68fa961dc5574b63a88b49abf19eb86f9d1a4544e1398fcd84d6338c6dcb9550ee3abcdcab0654f5cc2b85699c5ed5b3b31a1c35a199d -libblastrampoline.v5.12.0+0.x86_64-w64-mingw32.tar.gz/md5/450afb701cc2899c7c083bd3f3e580a0 -libblastrampoline.v5.12.0+0.x86_64-w64-mingw32.tar.gz/sha512/e4d1785a06b051a4f16edd7343021eed61ac45cf45d26b4e3ef1e54cfaadb44da2e74b7d854e31b05a733dbb3004f3e85644967316c4f41d1ad64400fed126f2 +blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/md5/855b7723a6e9eb8885876eb675d48329 +blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/sha512/29cbd060c8f5eb17ef486d0a10ee4b221eeceec3a2ab0f9f98f60880f3d19a2247d93ac0dc0d32ec568ef876acd30f6c0642aaf704757580c2e17884e425607f +libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/md5/d8dc0f092f86b379b2fb9da97382be70 +libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/sha512/d9fc0439565afaabe53f56f64c20aeddb846c991dafeafdef6c2369bd7a359c1a6b49cdf8d63eaae2730a336509854b5c306e630eb520445712efc4e41c0263e +libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/md5/c181e51a6ca4cde0da3d036d561e24dc +libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/sha512/fe4a86bb4c94ef86c2307adad528bb58d0508a33c194c64190fffe7902f5b915592567d9e0cc35414633c5ab9067def2fa20cf669a2f4309265744180a5ec51a +libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/md5/6f9eb8d73a0e61f3a2b97dba7105086e +libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/sha512/9c3db080155729a91b5dd47df91d3852539aefc331d4dc51167fccaf3b01e601b36911ec259c53e211fe192c108e839a1f14b837009fa4f7d88ed82d658f80ff +libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/md5/68f65db9da9938929d510eea3540335b +libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/sha512/2fc7b375a751f3bb201504e0417828602fe014a2c8626137779c09ca7264ac6d39d44db0d1d32e0dc506284f56b49e23791922b0cc1237021473fb505fbf06bd +libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a377fa4e5751fbeb3c42f319cb6341de +libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/9ddb1e2f4daab45d65b66dafc00df6ca7f788cb919cd6699c4aa0deca3e99a86d9ced10c3741610a6e480093d483e8a02c1d9165f91a7179632c1e2ae1abcfb7 +libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/md5/42c841baa05f80f17ea1b1d4f3405bef +libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/sha512/0c3ed42bd48f8f1ee9b1dc18faa7afa6e2fb27cffc59b9a420e29b5e6cdf8fb3bde36b82f3086075f8f7f329614aeb91ca5f173b1683e30e9530076f341ea2e0 +libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/md5/61e515ec1223c99705175a26e6fbaf87 +libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/92260dcc563ece74719f21921a7cb51266884ed01b50c97fa997b4a98737e900ec9eaa8012d2c4c67ab479c4080bd1cf2708612eaaaddbba28e4f9147f3708ea +libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/md5/d45816d705dd46572d85105567bc060e +libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/sha512/45cba07050b818cd85c67acdfc29515e1fe416eb4e0c219171f2c0c026f7412903c3a9367d48258259a16e89f36c1e8f9fa054e455759720f1c6c5e8e27be476 +libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/md5/c8d3fd5f314353133934396361857c92 +libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/sha512/a949b3c0655ad9d6f8d53fd8a3f0b4ab504046e49a373039defc94e832b7faf90c77520f3912c4d6db8b0829951d85b4fc2a4021b3d8bb2c399d1ad04ce77ab0 +libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/md5/a7bbd2233366d180ce8aa61fd3568c11 +libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/sha512/e78cbef5b3bcfa93a86e14eebf0d704a94ac7b1f5c7030706d1f4a960de888c42e3daddb65395c7102e08dfd444efbfb40273e58a5f1de199d44ad55fd3ae658 +libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/md5/4ca5cf3f855d04d3e8bdbd15321944ad +libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/sha512/33160caa000c6c44cedd594195e1f2efefb950459653ee12ad2be4cedf0b833874772512f1812948d753f075ee7b8fe5629e5f9bd753a3da7804c4a6e1b0e0a8 +libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/md5/8be947c20f7d35ec22247f9a11ccce43 +libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/sha512/56f4246f96d2f49b03f5e5f3b996660a48d50b3784f89df7cd1dc52bab6efea0c120a65015040041a51d18fc6f361f89486f77317d771fbf588a1ba7565d77a2 +libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/md5/85e1f70a3235097158b4884a58a58154 +libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/sha512/11f1f5c2a409dbdab11d6bc968610b5700e9b0cb95094e348fe43ddca5586eda47bda1c382fb1f4b5a15aa741a6fc2b31f58f9b08bfe46631b5471e864bc009b +libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/md5/c6756ca8b6778ce2a4a440f63355c32e +libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/sha512/895d9bba75a9a0861809dca48b3dae7b5ffc5d866a518729ffd52f70fa1742a41a4b8b4e03bb354cba12d9ad11a33f3f112fa69a30ab3f945a9dede0d59d92b3 +libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/md5/1326a406aa98b6045f7459d7fb237894 +libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/sha512/4965baa1de5532425ea57b8100e369cf44b55963340cd144c0359f845560f27a1bea1597e4c72ec541917f71aaff8a4863f47d01a095c2e761a68212bfb08d1e +libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/md5/5103983b7fecc7b87f495cd3b6c4d7a5 +libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/sha512/f3243d84a0a0a191abad9e3850c37be78892eb5905b63b47bfb3e5a4148e0dae672ee72d311c5c764ad0fffe57d39c10dfd2086466efd76b5030118941d36a00 +libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/md5/a001ecd07b5178ce724a4f78996dc43e +libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/sha512/508866d54a9a49df2ef7eaa5d807173016c6dfaec59c4c89d5b37cd3faa7384302d2d4d39aca1975d79a948414657b7ec048a3ebdf6bf5c938037aa89303013a +libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/md5/14fc4ec99e72e5bb646f5e6e8410fe01 +libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/sha512/b7d07218047917fe217736b3c97d2b0565f6c904cd9cf6de96e38c66552aeec13b3cde714775fce1eb5a230db0ec0f2822572de8f0e166cb042552a16beb2b79 diff --git a/stdlib/libblastrampoline_jll/Project.toml b/stdlib/libblastrampoline_jll/Project.toml index d1dde4c6074a7..ea4f645dc153c 100644 --- a/stdlib/libblastrampoline_jll/Project.toml +++ b/stdlib/libblastrampoline_jll/Project.toml @@ -1,13 +1,13 @@ name = "libblastrampoline_jll" uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.12.0+0" +version = "5.13.1+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] -julia = "1.12" +julia = "1.13" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From d887bd21ab6b750859405a51f60a60e34780fb3f Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sat, 21 Jun 2025 14:28:09 -0400 Subject: [PATCH 439/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=205577f68d6=20to=20e3d456127=20(#58781)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: master Julia branch: master Old commit: 5577f68d6 New commit: e3d456127 Julia version: 1.13.0-DEV Pkg version: 1.13.0 Bump invoked by: @KristofferC Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/5577f68d612139693282c037d070f515bf160d1b...e3d4561272fc029e9a5f940fe101ba4570fa875d ``` $ git log --oneline 5577f68d6..e3d456127 e3d456127 add update function to apps and fix a bug when adding an already installed app (#4263) cae9ce02a Fix historical stdlib fixup if `Pkg` is in the Manifest (#4264) a42046240 don't use tree hash from manifest if the path is set from sources (#4260) a94a6bcae fix dev taking when the app is already installed (#4259) 313fddccb Internals: Add fallback `Base.show(::IO, ::RegistryInstance)` method (#4251) ``` Co-authored-by: KristofferC <1282691+KristofferC@users.noreply.github.com> --- .../Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 | 1 - .../Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 | 1 - .../Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 | 1 + .../Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 create mode 100644 deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 diff --git a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 deleted file mode 100644 index 6724025f60360..0000000000000 --- a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -de84d77a95619c7127bf333a1a3e1c81 diff --git a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 b/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 deleted file mode 100644 index 17f80096c03eb..0000000000000 --- a/deps/checksums/Pkg-5577f68d612139693282c037d070f515bf160d1b.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7e904d8d984e7bcb48fb740d8bd35d3e26525dcbfd112b17e38d24ee6b72d4d0e96d07979f12004675097007b4bce58312c3822913c61c7fcf0441863d75c8de diff --git a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 new file mode 100644 index 0000000000000..bd80c0cbd8f59 --- /dev/null +++ b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 @@ -0,0 +1 @@ +3a5ec13a2d262404f9112a6a14ebd06d diff --git a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 new file mode 100644 index 0000000000000..1776b08ff133a --- /dev/null +++ b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 @@ -0,0 +1 @@ +fe9db6d3bbdb3d4a19e8ecf3833a1f315fdbe48d1437d2b2a12052183012da67380db0fc1842e0112007e4d80748eb314320d45526e9b8ab7fe873e565371605 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index def2c01b96494..19c3fcc398d62 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 5577f68d612139693282c037d070f515bf160d1b +PKG_SHA1 = e3d4561272fc029e9a5f940fe101ba4570fa875d PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From f61c640aaf218d4738a734ede86292c2a320dd9e Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:41:02 +0200 Subject: [PATCH 440/662] prevent unnecessary repeated squaring calculation (#58720) --- base/intfuncs.jl | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 4116f1631b724..925dafdd8f378 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -335,11 +335,6 @@ function invmod(n::T) where {T<:BitInteger} end # ^ for any x supporting * -function to_power_type(x::Number) - T = promote_type(typeof(x), typeof(x*x)) - convert(T, x) -end -to_power_type(x) = oftype(x*x, x) @noinline throw_domerr_powbysq(::Any, p) = throw(DomainError(p, LazyString( "Cannot raise an integer x to a negative power ", p, ".", "\nConvert input to float."))) @@ -355,12 +350,23 @@ to_power_type(x) = oftype(x*x, x) "or write float(x)^", p, " or Rational.(x)^", p, "."))) # The * keyword supports `*=checked_mul` for `checked_pow` @assume_effects :terminates_locally function power_by_squaring(x_, p::Integer; mul=*) - x = to_power_type(x_) + x_squared_ = x_ * x_ + x_squared_type = typeof(x_squared_) + T = if x_ isa Number + promote_type(typeof(x_), x_squared_type) + else + x_squared_type + end + x = convert(T, x_) + square_is_useful = mul === * if p == 1 return copy(x) elseif p == 0 return one(x) elseif p == 2 + if square_is_useful # avoid performing the same multiplication a second time when possible + return convert(T, x_squared_) + end return mul(x, x) elseif p < 0 isone(x) && return copy(x) @@ -369,6 +375,11 @@ to_power_type(x) = oftype(x*x, x) end t = trailing_zeros(p) + 1 p >>= t + if square_is_useful # avoid performing the same multiplication a second time when possible + if (t -= 1) > 0 + x = convert(T, x_squared_) + end + end while (t -= 1) > 0 x = mul(x, x) end From a595ea425438c20148ad9ec86828c366b92f69a5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Jun 2025 09:49:28 -0400 Subject: [PATCH 441/662] LibGit2: Update to 1.9.1 (#58731) --- base/binaryplatforms.jl | 5 +- deps/checksums/libgit2 | 76 +++++++++++------------ deps/jlutilities/documenter/Manifest.toml | 4 +- deps/jlutilities/revise/Manifest.toml | 4 +- deps/libgit2.mk | 14 +++-- deps/libgit2.version | 4 +- stdlib/LibGit2_jll/Project.toml | 4 +- stdlib/LibGit2_jll/src/LibGit2_jll.jl | 10 +-- stdlib/LibGit2_jll/test/runtests.jl | 2 +- stdlib/Manifest.toml | 6 +- test/stdlib_dependencies.jl | 13 ++++ 11 files changed, 81 insertions(+), 61 deletions(-) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 1d01fcf2d202f..c2f019c4d4eea 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -822,8 +822,9 @@ function parse_dl_name_version(path::String, os::String=_this_os_name()) local dlregex # Keep this up to date with _this_os_name if os == "windows" - # On Windows, libraries look like `libnettle-6.dll` - dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$"sa + # On Windows, libraries look like `libnettle-6.dll`. + # Stay case-insensitive, the suffix might be `.DLL`. + dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$"isa elseif os == "macos" # On OSX, libraries look like `libnettle.6.3.dylib` dlregex = r"^(.*?)((?:\.[\d]+)*)\.dylib$"sa diff --git a/deps/checksums/libgit2 b/deps/checksums/libgit2 index c1906c995cc73..2339d381a7079 100644 --- a/deps/checksums/libgit2 +++ b/deps/checksums/libgit2 @@ -1,38 +1,38 @@ -LibGit2.v1.9.0+0.aarch64-apple-darwin.tar.gz/md5/1e22c2cf3e6003addd9bf16026ac4a06 -LibGit2.v1.9.0+0.aarch64-apple-darwin.tar.gz/sha512/78d5e5d246534164e1d70cf69dea273bbb8386df24c13fc3c3571762df15f2714307e7ff4cae6f977eee9def121c94cfe33cfcd44a60905a8161d65d17565e90 -LibGit2.v1.9.0+0.aarch64-linux-gnu.tar.gz/md5/70bfe9da256442ea2c295a016a89d3b9 -LibGit2.v1.9.0+0.aarch64-linux-gnu.tar.gz/sha512/14916a5521aa1281b443e61beee2573bc55b76d88810a3bec8bdea677d95763da82f1a527975cdabcdaa213e69aa1640201a03656bdb505b886906795aad0c74 -LibGit2.v1.9.0+0.aarch64-linux-musl.tar.gz/md5/62f6e885de29a345cc5ee3e773c74471 -LibGit2.v1.9.0+0.aarch64-linux-musl.tar.gz/sha512/09e793209505ea954e608c609138b8865d8a1630340fa8ff032a55234bfb8277d2c3c31f26048ae4993bf8c3d8f165abd0b4ccd80526c61efca0807f634572df -LibGit2.v1.9.0+0.aarch64-unknown-freebsd.tar.gz/md5/6fcba6e43265aa7a1ea5bba85977d622 -LibGit2.v1.9.0+0.aarch64-unknown-freebsd.tar.gz/sha512/34b836d3c22436e74963141dbe1f9372cb7ee695ebb2054ee0af1353d4401e1dfb855e91341a1d06a24ce18d57caaa3aa1e2bc7063000fa4f9be40130eb6ff95 -LibGit2.v1.9.0+0.armv6l-linux-gnueabihf.tar.gz/md5/75ede2c2c7312adf06a2a9859cd6310f -LibGit2.v1.9.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/9de567bee3aad33eebac51ad5b57b4fefaa4b778ce8510b2524a55cd223bfaf3051fd48c8713741e799d1464b308469580716dcb847a6eb97fd632727ca22a7d -LibGit2.v1.9.0+0.armv6l-linux-musleabihf.tar.gz/md5/e5341f0c76c89273c465cb43cbf0f284 -LibGit2.v1.9.0+0.armv6l-linux-musleabihf.tar.gz/sha512/1029d47c82ce20223b1c108da77a1a32ef0b91b9645040c1d941e7abdd161011736a81f4ad25006b32d83d4c07c548fcf1c8a3326cf3cb91d56fd443e2e9ced7 -LibGit2.v1.9.0+0.armv7l-linux-gnueabihf.tar.gz/md5/03191a1c4ff1c1ae764092b26c941783 -LibGit2.v1.9.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/6bb113c722b550fb28fc84033a3a38565ed5305a7fa193eeb4949b979fcf4599b84c748f50dad2ad47481827138a6e405eaf727f719d219984a809088bbb2948 -LibGit2.v1.9.0+0.armv7l-linux-musleabihf.tar.gz/md5/1678d6e57aa887963b27917c884cbf36 -LibGit2.v1.9.0+0.armv7l-linux-musleabihf.tar.gz/sha512/52590e9ca4118e0dec70191353b2c76155363df77df6c0bb5741dfb3f333539a8ad75339796748a744c342b51c15869726cfe9bbf6ca78d524e7d2ccce4a4622 -LibGit2.v1.9.0+0.i686-linux-gnu.tar.gz/md5/3fc50746cb80e0455f8e7c7622cd433a -LibGit2.v1.9.0+0.i686-linux-gnu.tar.gz/sha512/20c97e1a816456267a16759378a5e968e6bca122d1e0dc7cc282cad2bf2a8e3929e90373752065d91dfb6688e39ac6db660d9bdbb3277f1b9cb04b5d3f46fd8c -LibGit2.v1.9.0+0.i686-linux-musl.tar.gz/md5/fadb5e051e3b21e68a61b2a3049f65c7 -LibGit2.v1.9.0+0.i686-linux-musl.tar.gz/sha512/369c8c64df89149e9ed600028c1ac96db24e7b2c1977146667b8aeba93aa7a3b4787a49734411448680654188ece33e740fa475108b80b876a5082edad722925 -LibGit2.v1.9.0+0.i686-w64-mingw32.tar.gz/md5/610da247e41070b73e71df7e41267846 -LibGit2.v1.9.0+0.i686-w64-mingw32.tar.gz/sha512/d5b61c885133e3002e48e0fc37ceed0bfeef070e8fc6b2d78ec5f3069ad80966ea5b3a2b3aeae1ca478e9a2f839309fd67c3a186ecf751f4642ff4cb4ca3cb38 -LibGit2.v1.9.0+0.powerpc64le-linux-gnu.tar.gz/md5/f05f5f07de55fd297c564b6cd4e54747 -LibGit2.v1.9.0+0.powerpc64le-linux-gnu.tar.gz/sha512/57b740ca3ef6b18994386d74f1cf39c97c1f58f5a63e749c1a0dcef8c43a915f13cc093a8e1d06cef1d1c60cf484ba0e38d20a96344df69dfc997daa63ee1137 -LibGit2.v1.9.0+0.riscv64-linux-gnu.tar.gz/md5/b043226b10e5cbbe4914be3392f5bf72 -LibGit2.v1.9.0+0.riscv64-linux-gnu.tar.gz/sha512/a580795dd9a7ee237cd1d51d55f5079588686b1adfe391a017de743946e1bd4e7d5e4f8b79a6f84f0ce165733ca1b67ea740d06fa18547c29616df2f73e3f289 -LibGit2.v1.9.0+0.x86_64-apple-darwin.tar.gz/md5/bad8607d4997ef82cd43edfc7579d0fb -LibGit2.v1.9.0+0.x86_64-apple-darwin.tar.gz/sha512/c7359d79949a6727973b1df2264b672bfcd1617b6d4c74d281ef70ac93bcadfe47f99f7a5d031eed36b65077668ba12f2b31bbe6d491542b6938816659070317 -LibGit2.v1.9.0+0.x86_64-linux-gnu.tar.gz/md5/21e5fd214a6358f643477973c22ec70c -LibGit2.v1.9.0+0.x86_64-linux-gnu.tar.gz/sha512/9e68cb6d25d85ad272fcb0d77deedce2daa9c62d7ce2fd7e9221647d021aa00e372f490ad29211d7ca2b5ddefb4addcc4733e25e3df038aaf26fe3cb269d8f56 -LibGit2.v1.9.0+0.x86_64-linux-musl.tar.gz/md5/e9ad320825b22ee378b33856ca266b12 -LibGit2.v1.9.0+0.x86_64-linux-musl.tar.gz/sha512/bd33b4d31a7622a0440bd0979ecc7bbdef7ba7a52bfc911f880c9430d57d2b9ea1c6c4e57697b5a2b63c2e00e07673b3dad6feac056a4f345ed6e3b0ef7aef77 -LibGit2.v1.9.0+0.x86_64-unknown-freebsd.tar.gz/md5/501c63c8810616e6764ff80c23fff0b5 -LibGit2.v1.9.0+0.x86_64-unknown-freebsd.tar.gz/sha512/109e5676899ba6992a68fcff6d7503f49cc3b748b4b0faffcf951f318f9730e242914b57a7848111e229642070fdbce29bc181cbc79ac2e794c6ef489bb27293 -LibGit2.v1.9.0+0.x86_64-w64-mingw32.tar.gz/md5/4e76fa8356407a7065b50298817ad462 -LibGit2.v1.9.0+0.x86_64-w64-mingw32.tar.gz/sha512/01204b29ff2f90a9204d2e91fb7d48a3b6bea008a77984e3e67423a04f630690073d648a7200168809999aa5885fa6035c5b099256724b0379229c257ef19b9f -libgit2-338e6fb681369ff0537719095e22ce9dc602dbf0.tar.gz/md5/0ce4a212921ef1752ea057a3be45e384 -libgit2-338e6fb681369ff0537719095e22ce9dc602dbf0.tar.gz/sha512/4eb018a85a59c6ac0514f09f19a40813d8a4bc5ea230bf54897aa2ef5f584796e8c680a27ac68985a3457e1ea1f554ba4af803b430d67a9065cf51ff317d7a89 +LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/md5/1281d4cfc44ab26054b83355a053c42b +LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/sha512/d4040d56c588333c222d66488944c2988a3ed1bb096ffe2bac67061b0ce20c44c5a803741fdc53a14a2523d0bd3fe5b3a6f247b0c366d0d3676c48bac44d3c05 +LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/md5/b2571cd9b20a307fc12c9d7979f42e81 +LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/sha512/52a2301f1311a8fb3c091b158e99d29267175156979779896d346cff5a5dac8b784f113793ca86bbd41f92c937c4eb64b8b8238df55ae95a6b1320cc3455b379 +LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/md5/b53fa9546706075f41cf12c39c4969ed +LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/sha512/9d818007520f1780eff4d0b6b2d8c7c1fc09763f36280b55d3795895dfa908d87db1699e9dc265162f831b9b1f03879cd187c020f8082c77eda57320eca14ff0 +LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/md5/5975e2abf13b3cefbc8cfd3d1d24a956 +LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/sha512/05337be9595c5d94816d168cf245289f2c7b2717c49ea000ba872b582c919b327be3a8cfa877eaf7c541a38897db3f57e11a8f49df9b9bfbf8f3110b39389e0e +LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a4de1b1a529f130d85a75b56f82551e8 +LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/cfdb3aef71f88405e7ec3c31df1f84617ad01dd227d0d6dde9a7dcd5e4a59590437f4ae90b91b63ab14a5716b366e554ed8d2e403eca2bcc9f4a3f7b813d3df3 +LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/md5/6a5deeaf7dc9f4352dc018df12c1ec61 +LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/sha512/cec900743c68c1b2580d21e51abe1852ae44ecadd682e48300d0b656ce955f71c4706c1462c35d301e331d79c37330bb4abface88709dee8b4154d1b65ea1217 +LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/md5/90d0cab30f58b6e1d75b006c12f3ce0f +LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/735174f87677795e313b8338a50f97ce438058386ff594ae91f7b8bed3085e8579c4da5e305e3e70821e30cbf6d878deb1c161b695b1f19f58bc924663bf2949 +LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/md5/56c495c7cd018e3942ee5879d0e397fe +LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/sha512/2ea7cafb587c2f2f693789f37c1962fdcd7d4815b064e676c876178f42f8dab400a91d48767bbfd1758646174bf060cad2978cd0cc74fa810955acb1e96b6ba0 +LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/md5/d8fd641059379d0a153aefc806adc12a +LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/sha512/bbe3b2903c981037d71d841a1376d744068a5f1ac9d476f833e3f31d7c22284d35262f04ce0fd885d63102c0bb68558e29ec2743adafa6dc618b13b061be402a +LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/md5/c7e31adcdb6735e4a658e8c75d5cf213 +LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/sha512/1f1ec4a3f1cbe14ff1b94ddb1d4c693b0fefec4bff51ab18920ed84eb12406d778af624baebf75a82c6411540cb62bbd3e00d975b38281a7ef51d7b7cfe255ff +LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/md5/d28b2507115bcbceab8fe2c1a3141e72 +LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/sha512/42d3ba8a06207dbe4a9862981f18a08c77fbf7eb0da047b2631d4cc46352e6403c8016fa6490b6732165c8286e341bd167f1d4953a77607102fceec276f37eed +LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/md5/ef21df7cd15d2eb71e9b1ed00d30a008 +LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/sha512/1b6b76d098d1b3bb48ece5c95b529e067b815aa95115f8df604a9169ed243d8b1b1c86598425f34193dce594dba19d7094ae95860ad7fed723d891df3cbf53c1 +LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/md5/52eb7e4c1057c704a5abf4a3526ff6f9 +LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/sha512/1d1378143ca1f1cb61335785b6d7db655d00da721ba5a5837c735b3dbd2a5be3207ee1e8bd2cfbcbd08e41f8aa61c028335affad7854cfe499b0210b5642c124 +LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/md5/a571908803d937204a84d0418c49284e +LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/sha512/bc3f19cc22889f7d082363274e6fd70487e0e5c1c0e156098d6f4f1350b78b327f8d7891f3c5c476628e85649a44ef0b2ec51899afa8721fbb26363fd52c1586 +LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/md5/9823d346d9cfebb7d14f537e7d4f66da +LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/sha512/5ac7a780861907554e3586ec2bd2109603af93ef3a6a3d8a9746af65013d3d745f00289dd24312b6b6bbe6154f0e66fd7925e4907873e712f3e765b960a36771 +LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/md5/d12787cdb2407c9668dd1c9fe5cef246 +LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/sha512/d7b0a08bc154639ca0c3f12ebf3b5d1725edc45a177ac27e91abb83add19c3e3b2c414c955477d9ace6ceeddb5e1f7257a56baccee8134812b5d47932162089c +LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/md5/0aecfa9999032119fe0ee6aee13a55c1 +LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/sha512/b2b42ff038abf32fb4e3c15d4b4760578410218685fcd2761c91efb366d5430e2dcad267f51c28205040b2714dbe91aaa060742ac66b77690744367de14484d7 +LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/md5/d610537c9abd49eec7d861c814226c2d +LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/sha512/5bea03406ef4d2bf89bd82fa6cdd8fa52fb3cc6efdc7e5016a1e3a33e60603be23f867630d7a5b14d2fe435fcade5229e10067a86945a5e46abaa20ce1f7d9dd +libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/md5/904b3287acfbd1098a447e50b957f76d +libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/sha512/e3f49faa709456d506993328177226932e3673fce3cf9ac0d0b0063206aa9ce0a6d780348c35f5ad28f59ae50f777c9f4fa54ec35af04582742cf7e36af09b56 diff --git a/deps/jlutilities/documenter/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml index 594ef1b3d17f9..55ff511a39daa 100644 --- a/deps/jlutilities/documenter/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -124,9 +124,9 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] diff --git a/deps/jlutilities/revise/Manifest.toml b/deps/jlutilities/revise/Manifest.toml index bfc8483f487e3..27ea36fc31de4 100644 --- a/deps/jlutilities/revise/Manifest.toml +++ b/deps/jlutilities/revise/Manifest.toml @@ -44,9 +44,9 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] diff --git a/deps/libgit2.mk b/deps/libgit2.mk index d051da9df6a35..a6affb3897110 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -13,7 +13,15 @@ ifeq ($(USE_SYSTEM_OPENSSL), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/openssl endif -LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DBUILD_CLI=OFF +ifeq ($(USE_SYSTEM_PCRE), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/pcre +endif + +ifeq ($(USE_SYSTEM_ZLIB), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/zlib +endif + +LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=OFF -DUSE_SSH=ON -DREGEX_BACKEND=pcre2 -DBUILD_CLI=OFF ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifeq ($(USE_SYSTEM_LIBSSH2), 0) @@ -42,10 +50,6 @@ ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) LIBGIT2_OPTS += -DUSE_HTTPS="OpenSSL" -DUSE_SHA1="CollisionDetection" -DCMAKE_INSTALL_RPATH="\$$ORIGIN" endif -# use the bundled distribution of libpcre. we should consider linking against the -# pcre2 library we're building anyway, but this is currently how Yggdrasil does it. -LIBGIT2_OPTS += -DREGEX_BACKEND="builtin" - LIBGIT2_SRC_PATH := $(SRCCACHE)/$(LIBGIT2_SRC_DIR) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: $(LIBGIT2_SRC_PATH)/source-extracted diff --git a/deps/libgit2.version b/deps/libgit2.version index ffba640e3b24e..71cea3aa9dd7b 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -3,8 +3,8 @@ LIBGIT2_JLL_NAME := LibGit2 ## source build -LIBGIT2_BRANCH=v1.9.0 -LIBGIT2_SHA1=338e6fb681369ff0537719095e22ce9dc602dbf0 +LIBGIT2_BRANCH=v1.9.1 +LIBGIT2_SHA1=0060d9cf5666f015b1067129bd874c6cc4c9c7ac ## Other deps # Specify the version of the Mozilla CA Certificate Store to obtain. diff --git a/stdlib/LibGit2_jll/Project.toml b/stdlib/LibGit2_jll/Project.toml index 1b78c6d6b4877..960e0e1d7c16b 100644 --- a/stdlib/LibGit2_jll/Project.toml +++ b/stdlib/LibGit2_jll/Project.toml @@ -1,10 +1,12 @@ name = "LibGit2_jll" uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [deps] OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" +PCRE2_jll = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" diff --git a/stdlib/LibGit2_jll/src/LibGit2_jll.jl b/stdlib/LibGit2_jll/src/LibGit2_jll.jl index 53663f1105220..350ed85a503f7 100644 --- a/stdlib/LibGit2_jll/src/LibGit2_jll.jl +++ b/stdlib/LibGit2_jll/src/LibGit2_jll.jl @@ -3,7 +3,7 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibGit2_jll.jl baremodule LibGit2_jll -using Base, Libdl, LibSSH2_jll +using Base, Libdl, LibSSH2_jll, PCRE2_jll, Zlib_jll if !(Sys.iswindows() || Sys.isapple()) using OpenSSL_jll end @@ -33,14 +33,14 @@ const libgit2 = LazyLibrary( end; dependencies = if Sys.iswindows() if Sys.WORD_SIZE == 32 - LazyLibrary[libssh2, libgcc_s] + LazyLibrary[libssh2, libgcc_s, libpcre2_8, libz] else - LazyLibrary[libssh2] + LazyLibrary[libssh2, libpcre2_8, libz] end elseif Sys.isfreebsd() || Sys.islinux() - LazyLibrary[libssh2, libssl, libcrypto] + LazyLibrary[libssh2, libssl, libcrypto, libpcre2_8, libz] else - LazyLibrary[libssh2] + LazyLibrary[libssh2, libpcre2_8, libz] end ) diff --git a/stdlib/LibGit2_jll/test/runtests.jl b/stdlib/LibGit2_jll/test/runtests.jl index 06edefe335a2f..a971c1e8aa402 100644 --- a/stdlib/LibGit2_jll/test/runtests.jl +++ b/stdlib/LibGit2_jll/test/runtests.jl @@ -7,5 +7,5 @@ using Test, Libdl, LibGit2_jll minor = Ref{Cint}(0) patch = Ref{Cint}(0) @test ccall((:git_libgit2_version, libgit2), Cint, (Ref{Cint}, Ref{Cint}, Ref{Cint}), major, minor, patch) == 0 - @test VersionNumber(major[], minor[], patch[]) == v"1.9.0" + @test VersionNumber(major[], minor[], patch[]) == v"1.9.1" end diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 81a638cba72de..894a672ae2d96 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -101,9 +101,9 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] @@ -118,7 +118,7 @@ version = "2.0.1+20" [[deps.LibUnwind_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "Zlib_jll"] uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" -version = "1.8.1+2" +version = "1.8.2+0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl index 9d0f6d7c05524..0af7d02f6e6a3 100644 --- a/test/stdlib_dependencies.jl +++ b/test/stdlib_dependencies.jl @@ -220,6 +220,19 @@ try missing_deps = setdiff(real_lib_deps, lazy_lib_deps) extraneous_deps = setdiff(lazy_lib_deps, real_lib_deps) + # The library name is `libpcre2-8`, with a dash in + # its name. That works fine on Unix. On Windows, a + # suffix starting with a dash denotes the + # library's soversion. So we think (on Windows) + # that this is the library `libpcre2`, soversion + # 8, and things don't match. + if Sys.iswindows() + if "libpcre2-8" in missing_deps && "libpcre2" in extraneous_deps + missing_deps = setdiff(missing_deps, ["libpcre2-8"]) + extraneous_deps = setdiff(extraneous_deps, ["libpcre2"]) + end + end + # We expect there to be no missing or extraneous deps deps_mismatch = !isempty(missing_deps) || !isempty(extraneous_deps) From 40c6d1b7c13c94d53b18bce14cecca7cae260e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Mon, 23 Jun 2025 23:30:04 +0100 Subject: [PATCH 442/662] Unify `_checkbounds_array` into `checkbounds` and use it in more places (#58785) Ref: https://github.com/JuliaLang/julia/pull/58755#discussion_r2158944282. --------- Co-authored-by: Matt Bauman Co-authored-by: Matt Bauman --- base/array.jl | 2 +- base/essentials.jl | 13 +++++++------ base/genericmemory.jl | 4 ++-- base/strings/basic.jl | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/base/array.jl b/base/array.jl index 5c9b37f887ea8..61a3c39ca2bb7 100644 --- a/base/array.jl +++ b/base/array.jl @@ -993,7 +993,7 @@ function setindex!(A::Array{T}, x, i::Int) where {T} end function _setindex!(A::Array{T}, x::T, i::Int) where {T} @_noub_if_noinbounds_meta - @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, (i,)) + @boundscheck checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) memoryrefset!(memoryrefnew(A.ref, i, false), x, :not_atomic, false) return A end diff --git a/base/essentials.jl b/base/essentials.jl index 820cce6839f13..e1bc648e7dbd0 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -377,13 +377,14 @@ macro _nospecializeinfer_meta() return Expr(:meta, :nospecializeinfer) end -function _checkbounds_array(::Type{Bool}, A::Union{Array, GenericMemory}, i::Int) +# These special checkbounds methods are defined early for bootstrapping +function checkbounds(::Type{Bool}, A::Union{Array, Memory}, i::Int) @inline ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) end -function _checkbounds_array(A::Union{Array, GenericMemory}, i::Int) +function checkbounds(A::Union{Array, GenericMemory}, i::Int) @inline - _checkbounds_array(Bool, A, i) || throw_boundserror(A, (i,)) + checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) end default_access_order(a::GenericMemory{:not_atomic}) = :not_atomic @@ -393,7 +394,7 @@ default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic function getindex(A::GenericMemory, i::Int) @_noub_if_noinbounds_meta - (@_boundscheck) && _checkbounds_array(A, i) + (@_boundscheck) && checkbounds(A, i) memoryrefget(memoryrefnew(memoryrefnew(A), i, false), default_access_order(A), false) end @@ -962,13 +963,13 @@ end # linear indexing function getindex(A::Array, i::Int) @_noub_if_noinbounds_meta - @boundscheck _checkbounds_array(A, i) + @boundscheck checkbounds(A, i) memoryrefget(memoryrefnew(getfield(A, :ref), i, false), :not_atomic, false) end # simple Array{Any} operations needed for bootstrap function setindex!(A::Array{Any}, @nospecialize(x), i::Int) @_noub_if_noinbounds_meta - @boundscheck _checkbounds_array(A, i) + @boundscheck checkbounds(A, i) memoryrefset!(memoryrefnew(getfield(A, :ref), i, false), x, :not_atomic, false) return A end diff --git a/base/genericmemory.jl b/base/genericmemory.jl index 1b45ba23bcded..d25d213a3546e 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -106,7 +106,7 @@ sizeof(a::GenericMemory) = Core.sizeof(a) # multi arg case will be overwritten later. This is needed for bootstrapping function isassigned(a::GenericMemory, i::Int) @inline - @boundscheck (i - 1)%UInt < length(a)%UInt || return false + @boundscheck checkbounds(Bool, a, i) || return false return @inbounds memoryref_isassigned(memoryref(a, i), default_access_order(a), false) end @@ -227,7 +227,7 @@ Memory{T}(x::AbstractArray{S,1}) where {T,S} = copyto_axcheck!(Memory{T}(undef, function _iterate_array(A::Union{Memory, Array}, i::Int) @inline - (i - 1)%UInt < length(A)%UInt ? (A[i], i + 1) : nothing + checkbounds(Bool, A, i) ? (A[i], i + 1) : nothing end iterate(A::Memory, i=1) = (@inline; _iterate_array(A, i)) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 314903898b92a..6619a2b25574e 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -797,7 +797,7 @@ size(s::CodeUnits) = (length(s),) elsize(s::Type{<:CodeUnits{T}}) where {T} = sizeof(T) @propagate_inbounds getindex(s::CodeUnits, i::Int) = codeunit(s.s, i) IndexStyle(::Type{<:CodeUnits}) = IndexLinear() -@inline iterate(s::CodeUnits, i=1) = (i % UInt) - 1 < length(s) ? (@inbounds s[i], i + 1) : nothing +@inline iterate(s::CodeUnits, i=1) = checkbounds(Bool, s, i) ? (@inbounds s[i], i + 1) : nothing write(io::IO, s::CodeUnits) = write(io, s.s) From 1998d5b530374142649365370f355bc2ff08d8de Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Tue, 24 Jun 2025 07:42:26 -0400 Subject: [PATCH 443/662] Chained hash pipelining in array hashing (#58252) the proposed switch in https://github.com/JuliaLang/julia/pull/57509 from `3h - hash_finalizer(x)` to `hash_finalizer(3h -x)` should increase the hash quality of chained hashes, as the expanded expression goes from something like `sum((-3)^k * hash(x) for k in ...)` to a non-simplifiable composition this does have the unfortunate impact of long chains of hashes getting a bit slower as there is more data dependency and the CPU cannot work on the next element's hash before combining the previous one (I think --- I'm not particularly an expert on this low level stuff). As far as I know this only really impacts `AbstractArray` so, I've implemented a proposal that does some unrolling / pipelining manually to recover `AbstractArray` hashing performance. in fact, it's quite a lot faster now for most lengths. I tuned the thresholds (8 accumulators, certain length breakpoints) by hand on my own machine. --- base/abstractarray.jl | 76 ----------------------------- base/hashing.jl | 2 +- base/multidimensional.jl | 102 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 77 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 8f55e6a56eba8..94bf3170feb38 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3567,81 +3567,6 @@ pushfirst!(A, a, b, c...) = pushfirst!(pushfirst!(A, c...), a, b) # sizehint! does not nothing by default sizehint!(a::AbstractVector, _) = a -## hashing AbstractArray ## - -const hash_abstractarray_seed = UInt === UInt64 ? 0x7e2d6fb6448beb77 : 0xd4514ce5 -function hash(A::AbstractArray, h::UInt) - h ⊻= hash_abstractarray_seed - # Axes are themselves AbstractArrays, so hashing them directly would stack overflow - # Instead hash the tuple of firsts and lasts along each dimension - h = hash(map(first, axes(A)), h) - h = hash(map(last, axes(A)), h) - - # For short arrays, it's not worth doing anything complicated - if length(A) < 8192 - for x in A - h = hash(x, h) - end - return h - end - - # Goal: Hash approximately log(N) entries with a higher density of hashed elements - # weighted towards the end and special consideration for repeated values. Colliding - # hashes will often subsequently be compared by equality -- and equality between arrays - # works elementwise forwards and is short-circuiting. This means that a collision - # between arrays that differ by elements at the beginning is cheaper than one where the - # difference is towards the end. Furthermore, choosing `log(N)` arbitrary entries from a - # sparse array will likely only choose the same element repeatedly (zero in this case). - - # To achieve this, we work backwards, starting by hashing the last element of the - # array. After hashing each element, we skip `fibskip` elements, where `fibskip` - # is pulled from the Fibonacci sequence -- Fibonacci was chosen as a simple - # ~O(log(N)) algorithm that ensures we don't hit a common divisor of a dimension - # and only end up hashing one slice of the array (as might happen with powers of - # two). Finally, we find the next distinct value from the one we just hashed. - - # This is a little tricky since skipping an integer number of values inherently works - # with linear indices, but `findprev` uses `keys`. Hoist out the conversion "maps": - ks = keys(A) - key_to_linear = LinearIndices(ks) # Index into this map to compute the linear index - linear_to_key = vec(ks) # And vice-versa - - # Start at the last index - keyidx = last(ks) - linidx = key_to_linear[keyidx] - fibskip = prevfibskip = oneunit(linidx) - first_linear = first(LinearIndices(linear_to_key)) - n = 0 - while true - n += 1 - # Hash the element - elt = A[keyidx] - h = hash(keyidx=>elt, h) - - # Skip backwards a Fibonacci number of indices -- this is a linear index operation - linidx = key_to_linear[keyidx] - linidx < fibskip + first_linear && break - linidx -= fibskip - keyidx = linear_to_key[linidx] - - # Only increase the Fibonacci skip once every N iterations. This was chosen - # to be big enough that all elements of small arrays get hashed while - # obscenely large arrays are still tractable. With a choice of N=4096, an - # entirely-distinct 8000-element array will have ~75% of its elements hashed, - # with every other element hashed in the first half of the array. At the same - # time, hashing a `typemax(Int64)`-length Float64 range takes about a second. - if rem(n, 4096) == 0 - fibskip, prevfibskip = fibskip + prevfibskip, fibskip - end - - # Find a key index with a value distinct from `elt` -- might be `keyidx` itself - keyidx = findprev(!isequal(elt), A, keyidx) - keyidx === nothing && break - end - - return h -end - # The semantics of `collect` are weird. Better to write our own function rest(a::AbstractArray{T}, state...) where {T} v = Vector{T}(undef, 0) @@ -3650,7 +3575,6 @@ function rest(a::AbstractArray{T}, state...) where {T} return foldl(push!, Iterators.rest(a, state...), init=v) end - ## keepat! ## # NOTE: since these use `@inbounds`, they are actually only intended for Vector and BitVector diff --git a/base/hashing.jl b/base/hashing.jl index 1b323f2e9097e..897a0d73cb874 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -45,7 +45,7 @@ end hash_mix(a::UInt64, b::UInt64) = ⊻(mul_parts(a, b)...) # faster-but-weaker than hash_mix intended for small keys -hash_mix_linear(x::UInt64, h::UInt) = 3h - x +hash_mix_linear(x::Union{UInt64, UInt32}, h::UInt) = 3h - x function hash_finalizer(x::UInt64) x ⊻= (x >> 32) x *= 0x63652a4cd374b267 diff --git a/base/multidimensional.jl b/base/multidimensional.jl index edf71927661ca..83a03fc1b45bf 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -2017,3 +2017,105 @@ end getindex(b::Ref, ::CartesianIndex{0}) = getindex(b) setindex!(b::Ref, x, ::CartesianIndex{0}) = setindex!(b, x) + +## hashing AbstractArray ## can't be put in abstractarray.jl due to bootstrapping problems with the use of @nexpr + +function _hash_fib(A, h::UInt) + # Goal: Hash approximately log(N) entries with a higher density of hashed elements + # weighted towards the end and special consideration for repeated values. Colliding + # hashes will often subsequently be compared by equality -- and equality between arrays + # works elementwise forwards and is short-circuiting. This means that a collision + # between arrays that differ by elements at the beginning is cheaper than one where the + # difference is towards the end. Furthermore, choosing `log(N)` arbitrary entries from a + # sparse array will likely only choose the same element repeatedly (zero in this case). + + # To achieve this, we work backwards, starting by hashing the last element of the + # array. After hashing each element, we skip `fibskip` elements, where `fibskip` + # is pulled from the Fibonacci sequence -- Fibonacci was chosen as a simple + # ~O(log(N)) algorithm that ensures we don't hit a common divisor of a dimension + # and only end up hashing one slice of the array (as might happen with powers of + # two). Finally, we find the next distinct value from the one we just hashed. + + # This is a little tricky since skipping an integer number of values inherently works + # with linear indices, but `findprev` uses `keys`. Hoist out the conversion "maps": + ks = keys(A) + key_to_linear = LinearIndices(ks) # Index into this map to compute the linear index + linear_to_key = vec(ks) # And vice-versa + + # Start at the last index + keyidx = last(ks) + linidx = key_to_linear[keyidx] + fibskip = prevfibskip = oneunit(linidx) + first_linear = first(LinearIndices(linear_to_key)) + @nexprs 4 i -> p_i = h + + n = 0 + while true + n += 1 + # Hash the element + elt = A[keyidx] + + stream_idx = mod1(n, 4) + @nexprs 4 i -> stream_idx == i && (p_i = hash_mix_linear(hash(keyidx, p_i), hash(elt, p_i))) + + # Skip backwards a Fibonacci number of indices -- this is a linear index operation + linidx = key_to_linear[keyidx] + linidx < fibskip + first_linear && break + linidx -= fibskip + keyidx = linear_to_key[linidx] + + # Only increase the Fibonacci skip once every N iterations. This was chosen + # to be big enough that all elements of small arrays get hashed while + # obscenely large arrays are still tractable. With a choice of N=4096, an + # entirely-distinct 8000-element array will have ~75% of its elements hashed, + # with every other element hashed in the first half of the array. At the same + # time, hashing a `typemax(Int64)`-length Float64 range takes about a second. + if rem(n, 4096) == 0 + fibskip, prevfibskip = fibskip + prevfibskip, fibskip + end + + # Find a key index with a value distinct from `elt` -- might be `keyidx` itself + keyidx = findprev(!isequal(elt), A, keyidx) + keyidx === nothing && break + end + + @nexprs 4 i -> h = hash_mix_linear(p_i, h) + return hash_uint(h) +end + +function hash_shaped(A, h::UInt) + # Axes are themselves AbstractArrays, so hashing them directly would stack overflow + # Instead hash the tuple of firsts and lasts along each dimension + h = hash(map(first, axes(A)), h) + h = hash(map(last, axes(A)), h) + len = length(A) + + if len < 8 + # for the shortest arrays we chain directly + for elt in A + h = hash(elt, h) + end + return h + elseif len < 32768 + # separate accumulator streams, unrolled + @nexprs 8 i -> p_i = h + n = 1 + limit = len - 7 + while n <= limit + @nexprs 8 i -> p_i = hash(A[n + i - 1], p_i) + n += 8 + end + while n <= len + p_1 = hash(A[n], p_1) + n += 1 + end + # fold all streams back together + @nexprs 8 i -> h = hash_mix_linear(p_i, h) + return hash_uint(h) + else + return _hash_fib(A, h) + end +end + +const hash_abstractarray_seed = UInt === UInt64 ? 0x7e2d6fb6448beb77 : 0xd4514ce5 +hash(A::AbstractArray, h::UInt) = hash_shaped(A, h ⊻ hash_abstractarray_seed) From a452acc1d43a238dce3d360b538526176f11aee8 Mon Sep 17 00:00:00 2001 From: Navdeep Rana Date: Wed, 25 Jun 2025 05:38:11 +0200 Subject: [PATCH 444/662] Require all tuples in eachindex to have the same length. (#48125) Potential fix for #47898 --------- Co-authored-by: navdeep rana Co-authored-by: Oscar Smith Co-authored-by: Jerry Ling Co-authored-by: Andy Dienes <51664769+adienes@users.noreply.github.com> --- base/tuple.jl | 9 +++------ test/tuple.jl | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/tuple.jl b/base/tuple.jl index 5255f8ba07539..937e130d3cd93 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -149,12 +149,9 @@ nextind(@nospecialize(t::Tuple), i::Integer) = Int(i)+1 function keys(t::Tuple, t2::Tuple...) @inline - OneTo(_maxlength(t, t2...)) -end -_maxlength(t::Tuple) = length(t) -function _maxlength(t::Tuple, t2::Tuple, t3::Tuple...) - @inline - max(length(t), _maxlength(t2, t3...)) + lent = length(t) + all(x->length(x) == lent, t2) || throw_eachindex_mismatch_indices(IndexLinear(), t, t2...) + Base.OneTo(lent) end # this allows partial evaluation of bounded sequences of next() calls on tuples, diff --git a/test/tuple.jl b/test/tuple.jl index 560a1425c6bb6..30782367803c5 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -208,7 +208,7 @@ end @test iterate(t, y3[2]) === nothing @test eachindex((2,5,"foo")) === Base.OneTo(3) - @test eachindex((2,5,"foo"), (1,2,5,7)) === Base.OneTo(4) + @test_throws DimensionMismatch eachindex((2,5,"foo"), (1,2,5,7)) @test Core.Compiler.is_nothrow(Base.infer_effects(iterate, (Tuple{Int,Int,Int}, Int))) end From a48dcad6af4ababbeeeffeb6f54927953db7df15 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Wed, 25 Jun 2025 00:17:57 -0400 Subject: [PATCH 445/662] trailing dimensions in eachslice (#58791) fixes https://github.com/JuliaLang/julia/issues/51692 --- base/slicearray.jl | 17 +++++++++++------ test/arrayops.jl | 28 +++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/base/slicearray.jl b/base/slicearray.jl index 65d3b6c4d75fd..1928020a1155a 100644 --- a/base/slicearray.jl +++ b/base/slicearray.jl @@ -25,7 +25,7 @@ struct Slices{P,SM,AX,S,N} <: AbstractSlices{S,N} """ parent::P """ - A tuple of length `ndims(parent)`, denoting how each dimension should be handled: + A tuple of length at least `ndims(parent)`, denoting how each dimension should be handled: - an integer `i`: this is the `i`th dimension of the outer `Slices` object. - `:`: an "inner" dimension """ @@ -39,34 +39,39 @@ end unitaxis(::AbstractArray) = Base.OneTo(1) function Slices(A::P, slicemap::SM, ax::AX) where {P,SM,AX} + length(slicemap) >= ndims(A) || + throw(ArgumentError("Slices cannot be constructed with a slicemap of fewer elements than the parent has dimensions")) N = length(ax) - argT = map((a,l) -> l === (:) ? Colon : eltype(a), axes(A), slicemap) + parent_axes = ntuple(d -> axes(A, d), length(slicemap)) + argT = map((a,l) -> l === (:) ? Colon : eltype(a), parent_axes, slicemap) S = Base.promote_op(view, P, argT...) Slices{P,SM,AX,S,N}(A, slicemap, ax) end _slice_check_dims(N) = nothing function _slice_check_dims(N, dim, dims...) - 1 <= dim <= N || throw(DimensionMismatch("Invalid dimension $dim")) + 1 <= dim || throw(DimensionMismatch("Invalid dimension $dim")) dim in dims && throw(DimensionMismatch("Dimensions $dims are not unique")) _slice_check_dims(N,dims...) end @constprop :aggressive function _eachslice(A::AbstractArray{T,N}, dims::NTuple{M,Integer}, drop::Bool) where {T,N,M} _slice_check_dims(N,dims...) + N_ = foldl(max, dims; init=N) + if drop # if N = 4, dims = (3,1) then # axes = (axes(A,3), axes(A,1)) # slicemap = (2, :, 1, :) ax = map(dim -> axes(A,dim), dims) - slicemap = ntuple(dim -> something(findfirst(isequal(dim), dims), (:)), N) + slicemap = ntuple(dim -> something(findfirst(isequal(dim), dims), (:)), N_) return Slices(A, slicemap, ax) else # if N = 4, dims = (3,1) then # axes = (axes(A,1), OneTo(1), axes(A,3), OneTo(1)) # slicemap = (1, :, 3, :) - ax = ntuple(dim -> dim in dims ? axes(A,dim) : unitaxis(A), N) - slicemap = ntuple(dim -> dim in dims ? dim : (:), N) + ax = ntuple(dim -> dim in dims ? axes(A,dim) : unitaxis(A), N_) + slicemap = ntuple(dim -> dim in dims ? dim : (:), N_) return Slices(A, slicemap, ax) end end diff --git a/test/arrayops.jl b/test/arrayops.jl index 5d68f70e9caf1..84b9c8e7b2f6a 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -2402,7 +2402,7 @@ end M = [1 2 3; 4 5 6; 7 8 9] @test eachrow(M) == eachslice(M, dims = 1) == [[1, 2, 3], [4, 5, 6], [7, 8, 9]] @test eachcol(M) == eachslice(M, dims = 2) == [[1, 4, 7], [2, 5, 8], [3, 6, 9]] - @test_throws DimensionMismatch eachslice(M, dims = 4) + @test eachslice(M, dims = 4) == [[1 2 3; 4 5 6; 7 8 9;;;]] SR = @inferred eachrow(M) @test SR[2] isa eltype(SR) @@ -2467,6 +2467,32 @@ end @test_throws BoundsError A[2,3] = [4,5] @test_throws BoundsError A[2,3] .= [4,5] end + + @testset "trailing dimensions" begin + v = collect(1:3) + + S2 = eachslice(v; dims = 2, drop=true) + @test S2 isa AbstractSlices{<:AbstractVector, 1} + @test size(S2) == (1,) + @test S2[1] == v + + S2K = eachslice(v; dims = 2, drop=false) + @test S2K isa AbstractSlices{<:AbstractVector, 2} + @test size(S2K) == (1,1) + @test S2K[1,1] == v + + M = reshape(1:6, 2, 3) + + S13 = eachslice(M; dims = (1,3)) + @test size(S13) == (2,1) + @test S13[2,1] == M[2,:,1] + + S13K = eachslice(M; dims = (1,3), drop=false) + @test size(S13K) == (2,1,1) + @test S13K[1,1,1] == M[1,:] + @test S13K[2,1,1] == M[2,:] + end + end ### From b06d26075bf7b3f4e7f1b64b120f5665d8ed76f9 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Wed, 25 Jun 2025 05:32:31 -0700 Subject: [PATCH 446/662] Allow underscore (unused) args in presence of kwargs (#58803) Admittedly fixed because I thought I introduced this bug recently, but actually, fix #32727. `f(_; kw) = 1` should now lower in a similar way to `f(::Any; kw) = 1`, where we use a gensym for the first argument. Not in this PR, but TODO: `nospecialize` underscore-only names --- src/julia-syntax.scm | 20 +++++++++++--------- test/syntax.jl | 3 +++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 808943c05af5b..ad78221c540c5 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3,11 +3,16 @@ ;; pass 1: syntax desugaring -;; allow (:: T) => (:: #gensym T) in formal argument lists +;; unnamed or all-underscore arguments may still be read from internally, so +;; convert (:: T) => (:: #gensym T) and _ => #gensym in formal argument lists (define (fill-missing-argname a unused) - (if (and (pair? a) (eq? (car a) '|::|) (null? (cddr a))) - `(|::| ,(if unused UNUSED (gensy)) ,(cadr a)) - a)) + (define (replace-if-underscore u) + (if (underscore-symbol? u) (if unused UNUSED (gensy)) u)) + (if (and (pair? a) (eq? (car a) '|::|)) + (cond ((null? (cddr a)) `(|::| ,(if unused UNUSED (gensy)) ,(cadr a))) + ((null? (cdddr a)) `(|::| ,(replace-if-underscore (cadr a)) ,(caddr a))) + (else a)) + (replace-if-underscore a))) (define (fix-arglist l (unused #t)) (if (any vararg? (butlast l)) (error "invalid \"...\" on non-final argument")) @@ -165,10 +170,7 @@ ;; GF method does not need to keep decl expressions on lambda args ;; except for rest arg (define (method-lambda-expr argl body rett) - (let ((argl (map (lambda (x) - (let ((n (arg-name x))) - (if (underscore-symbol? n) UNUSED n))) - argl)) + (let ((argl (map arg-name argl)) (body (blockify body))) `(lambda ,argl () (scope-block @@ -374,7 +376,7 @@ (append req opt vararg) rett))))) ;; no optional positional args (let* ((names (map car sparams)) - (anames (map (lambda (x) (if (underscore-symbol? x) UNUSED x)) (llist-vars argl))) + (anames (llist-vars argl)) (unused_anames (filter (lambda (x) (not (eq? x UNUSED))) anames)) (ename (if (nodot-sym-ref? name) name (if (overlay? name) (cadr name) `(null))))) diff --git a/test/syntax.jl b/test/syntax.jl index 92d8391345312..dcd921823d273 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2897,6 +2897,9 @@ end @test Meta.isexpr(Meta.lower(Main, :(for _ in 1:2; 1; end)), :thunk) @test (try; throw(1); catch _; 2; end) === 2 @test (let _ = 1; 2; end) === 2 + @test (function f(_, _); 2; end)(0,0) === 2 + @test (function f(_, _=1); 2; end)(0,0) === 2 + @test (function f(_, _; kw1=2); kw1; end)(0,0) === 2 # ERROR: syntax: all-underscore identifiers are write-only and their values cannot be used in expressions @test Meta.isexpr(Meta.lower(Main, :(_ = 1; a = _)), :error) @test Meta.isexpr(Meta.lower(Main, :(let; function f(); _; end; end)), :error) From 2ed85614708c3ca7788e17c73ca3ccc68040a06b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 25 Jun 2025 14:07:30 -0400 Subject: [PATCH 447/662] codegen: slightly optimize gc-frame allocation (#58794) Try to avoid allocating frames for some very simple function that only have the safepoint on entry and don't define any values themselves. --- src/cgutils.cpp | 4 +- src/codegen.cpp | 27 ++-- src/llvm-gc-interface-passes.h | 33 ++-- src/llvm-late-gc-lowering.cpp | 284 ++++++++++++++------------------- 4 files changed, 159 insertions(+), 189 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b720945647756..07b304e5256d1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -341,11 +341,11 @@ static void find_perm_offsets(jl_datatype_t *typ, SmallVectorImpl &res } // load a pointer to N inlined_roots into registers (as a SmallVector) -static llvm::SmallVector load_gc_roots(jl_codectx_t &ctx, Value *inline_roots_ptr, size_t npointers, bool isVolatile=false) +static llvm::SmallVector load_gc_roots(jl_codectx_t &ctx, Value *inline_roots_ptr, size_t npointers, MDNode *tbaa, bool isVolatile=false) { SmallVector gcroots(npointers); Type *T_prjlvalue = ctx.types().T_prjlvalue; - auto roots_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + auto roots_ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); for (size_t i = 0; i < npointers; i++) { auto *ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue, emit_ptrgep(ctx, inline_roots_ptr, i * sizeof(jl_value_t*)), Align(sizeof(void*)), isVolatile); roots_ai.decorateInst(ptr); diff --git a/src/codegen.cpp b/src/codegen.cpp index 3ea65d46dcfb3..bdbf459908b36 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5054,7 +5054,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos break; case jl_returninfo_t::SRet: assert(result); - retval = mark_julia_slot(result, jlretty, NULL, ctx.tbaa().tbaa_gcframe, load_gc_roots(ctx, return_roots, returninfo.return_roots)); + retval = mark_julia_slot(result, jlretty, NULL, ctx.tbaa().tbaa_gcframe, load_gc_roots(ctx, return_roots, returninfo.return_roots, ctx.tbaa().tbaa_gcframe)); break; case jl_returninfo_t::Union: { Value *box = ctx.builder.CreateExtractValue(call, 0); @@ -5603,7 +5603,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va T_prjlvalue = AT->getElementType(); } assert(T_prjlvalue == ctx.types().T_prjlvalue); - v.inline_roots = load_gc_roots(ctx, varslot, nroots, vi.isVolatile); + v.inline_roots = load_gc_roots(ctx, varslot, nroots, ctx.tbaa().tbaa_gcframe, vi.isVolatile); } if (vi.usedUndef) { assert(vi.defFlag); @@ -6927,7 +6927,7 @@ static void emit_specsig_to_specsig( auto tracked = CountTrackedPointers(et); SmallVector roots; if (tracked.count && !tracked.all) { - roots = load_gc_roots(ctx, &*AI, tracked.count); + roots = load_gc_roots(ctx, &*AI, tracked.count, ctx.tbaa().tbaa_const); ++AI; } myargs[i] = mark_julia_slot(arg_v, jt, NULL, ctx.tbaa().tbaa_const, roots); @@ -8511,7 +8511,7 @@ static jl_llvm_functions_t ctx.spvals_ptr = &*AI++; } } - // step 6. set up GC frame and special arguments + // step 6a. set up special arguments and attributes Function::arg_iterator AI = f->arg_begin(); SmallVector attrs(f->arg_size()); // function declaration attributes @@ -8558,7 +8558,11 @@ static jl_llvm_functions_t attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); } + // step 6b. Setup the GC frame and entry safepoint before any loads allocate_gc_frame(ctx, b0); + if (params.safepoint_on_entry && JL_FEAT_TEST(ctx, safepoint_on_entry)) + emit_gc_safepoint(ctx.builder, ctx.types().T_size, get_current_ptls(ctx), ctx.tbaa().tbaa_const); + Value *last_age = NULL; Value *world_age_field = NULL; if (ctx.is_opaque_closure) { @@ -8716,7 +8720,14 @@ static jl_llvm_functions_t SmallVector roots; auto tracked = CountTrackedPointers(llvmArgType); if (tracked.count && !tracked.all) { - roots = load_gc_roots(ctx, &*AI, tracked.count); + Argument *RootArg = &*AI; + roots = load_gc_roots(ctx, RootArg, tracked.count, ctx.tbaa().tbaa_const); + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + param.addAttribute(Attribute::NonNull); + param.addAttribute(Attribute::NoUndef); + param.addDereferenceableAttr(tracked.count * sizeof(void*)); + param.addAlignmentAttr(alignof(void*)); + attrs[RootArg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); ++AI; } theArg = mark_julia_slot(Arg, argType, NULL, ctx.tbaa().tbaa_const, roots); // this argument is by-pointer @@ -9026,11 +9037,7 @@ static jl_llvm_functions_t Instruction &prologue_end = ctx.builder.GetInsertBlock()->back(); - // step 11a. Emit the entry safepoint - if (params.safepoint_on_entry && JL_FEAT_TEST(ctx, safepoint_on_entry)) - emit_gc_safepoint(ctx.builder, ctx.types().T_size, get_current_ptls(ctx), ctx.tbaa().tbaa_const); - - // step 11b. Do codegen in control flow order + // step 11. Do codegen in control flow order SmallVector workstack; DenseMap BB; DenseMap come_from_bb; diff --git a/src/llvm-gc-interface-passes.h b/src/llvm-gc-interface-passes.h index d1bb1fae01446..0d21ea0a66cd8 100644 --- a/src/llvm-gc-interface-passes.h +++ b/src/llvm-gc-interface-passes.h @@ -251,11 +251,13 @@ struct BBState { // These get updated during dataflow LargeSparseBitVector LiveIn; LargeSparseBitVector LiveOut; - SmallVector Safepoints; - int TopmostSafepoint = -1; + // auto Safepoints = std::range(LastSafepoint, FirstSafepoint); bool HasSafepoint = false; - // Have we gone through this basic block in our local scan yet? - bool Done = false; + // This lets us refine alloca tracking to avoid creating GC frames in + // some simple functions that only have the initial safepoint. + int FirstSafepoint = -1; + int LastSafepoint = -1; + int FirstSafepointAfterFirstDef = -1; }; struct State { @@ -292,21 +294,18 @@ struct State { // of its uses need to preserve the values listed in the map value. std::map> GCPreserves; - // The assignment of numbers to safepoints. The indices in the map - // are indices into the next three maps which store safepoint properties - std::map SafepointNumbering; + // The assignment of numbers to safepoints. These have the same ordering as + // LiveSets, LiveIfLiveOut, and CalleeRoots. + SmallVector SafepointNumbering; - // Reverse mapping index -> safepoint - SmallVector ReverseSafepointNumbering; - - // Instructions that can return twice. For now, all values live at these - // instructions will get their own, dedicated GC frame slots, because they - // have unobservable control flow, so we can't be sure where they're - // actually live. All of these are also considered safepoints. - SmallVector ReturnsTwice; + // Safepoint number of instructions that can return twice. For now, all + // values live at these instructions will get their own, dedicated GC frame + // slots, because they have unobservable control flow, so we can't be sure + // where they're actually live. + SmallVector ReturnsTwice; // The set of values live at a particular safepoint - SmallVector< LargeSparseBitVector , 0> LiveSets; + SmallVector LiveSets; // Those values that - if live out from our parent basic block - are live // at this safepoint. SmallVector> LiveIfLiveOut; @@ -332,7 +331,7 @@ struct LateLowerGCFrame: private JuliaPassContext { Value *pgcstack; Function *smallAllocFunc; - void MaybeNoteDef(State &S, BBState &BBS, Value *Def, const ArrayRef &SafepointsSoFar, + bool MaybeNoteDef(State &S, BBState &BBS, Value *Def, SmallVector &&RefinedPtr = SmallVector()); void NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitVector &Uses, Function &F); void NoteUse(State &S, BBState &BBS, Value *V, Function &F) { diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 22d730621b80c..d4e11afcfe2e1 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -676,16 +676,6 @@ SmallVector LateLowerGCFrame::NumberAll(State &S, Value *V) { } -static void MaybeResize(BBState &BBS, unsigned Idx) { - /* - if (BBS.Defs.size() <= Idx) { - BBS.Defs.resize(Idx + 1); - BBS.UpExposedUses.resize(Idx + 1); - BBS.PhiOuts.resize(Idx + 1); - } - */ -} - static bool HasBitSet(const LargeSparseBitVector &BV, unsigned Bit) { return BV.test(Bit); } @@ -694,47 +684,47 @@ static bool HasBitSet(const BitVector &BV, unsigned Bit) { return Bit < BV.size() && BV[Bit]; } -static void NoteDef(State &S, BBState &BBS, int Num, const ArrayRef &SafepointsSoFar) { +static void NoteDef(State &S, BBState &BBS, int Num) { assert(Num >= 0); - MaybeResize(BBS, Num); assert(!BBS.Defs.test(Num) && "SSA Violation or misnumbering?"); BBS.Defs.set(Num); BBS.UpExposedUses.reset(Num); // This value could potentially be live at any following safe point // if it ends up live out, so add it to the LiveIfLiveOut lists for all // following safepoints. - for (int Safepoint : SafepointsSoFar) { - S.LiveIfLiveOut[Safepoint].push_back(Num); - } + if (BBS.HasSafepoint) + for (int Safepoint = BBS.FirstSafepoint; Safepoint >= BBS.LastSafepoint; --Safepoint) + S.LiveIfLiveOut[Safepoint].push_back(Num); } -void LateLowerGCFrame::MaybeNoteDef(State &S, BBState &BBS, Value *Def, - const ArrayRef &SafepointsSoFar, +bool LateLowerGCFrame::MaybeNoteDef(State &S, BBState &BBS, Value *Def, SmallVector &&RefinedPtr) { Type *RT = Def->getType(); if (isa(RT)) { if (!isSpecialPtr(RT)) - return; + return false; assert(isTrackedValue(Def) && "Returned value of GC interest, but not tracked?"); int Num = Number(S, Def); - NoteDef(S, BBS, Num, SafepointsSoFar); + NoteDef(S, BBS, Num); if (!RefinedPtr.empty()) S.Refinements[Num] = std::move(RefinedPtr); + return true; } else { SmallVector Nums = NumberAll(S, Def); for (int Num : Nums) { - NoteDef(S, BBS, Num, SafepointsSoFar); + NoteDef(S, BBS, Num); if (!RefinedPtr.empty()) S.Refinements[Num] = RefinedPtr; } + return !Nums.empty(); } } static int NoteSafepoint(State &S, BBState &BBS, CallInst *CI, SmallVectorImpl &CalleeRoots) { + assert(BBS.FirstSafepoint == -1 || BBS.FirstSafepoint == S.MaxSafepointNumber); int Number = ++S.MaxSafepointNumber; - S.SafepointNumbering[CI] = Number; - S.ReverseSafepointNumbering.push_back(CI); + S.SafepointNumbering.push_back(CI); // Note which pointers are upward exposed live here. They need to be // considered live at this safepoint even when they have a def earlier // in this BB (i.e. even when they don't participate in the dataflow @@ -742,6 +732,10 @@ static int NoteSafepoint(State &S, BBState &BBS, CallInst *CI, SmallVectorImpl{}); S.CalleeRoots.push_back(std::move(CalleeRoots)); + BBS.HasSafepoint = true; + if (BBS.LastSafepoint == -1) + BBS.LastSafepoint = Number; + BBS.FirstSafepoint = Number; return Number; } @@ -761,7 +755,6 @@ void LateLowerGCFrame::NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitV int Num = Number(S, V); if (Num < 0) return; - MaybeResize(BBS, Num); Uses.set(Num); } } else { @@ -769,7 +762,6 @@ void LateLowerGCFrame::NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitV for (int Num : Nums) { if (Num < 0) continue; - MaybeResize(BBS, Num); Uses.set(Num); } } @@ -970,15 +962,6 @@ static uint64_t getLoadValueAlign(LoadInst *LI) return mdconst::extract(md->getOperand(0))->getLimitedValue(); } -static bool LooksLikeFrameRef(Value *V) { - if (isSpecialPtr(V->getType())) - return false; - V = V->stripInBoundsOffsets(); - if (isSpecialPtr(V->getType())) - return false; - return isa(V); -} - SmallVector LateLowerGCFrame::GetPHIRefinements(PHINode *Phi, State &S) { // The returned vector can violate the domination property of the Refinements map. @@ -1213,6 +1196,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { SmallVector PHINumbers; for (BasicBlock &BB : F) { BBState &BBS = S.BBStates[&BB]; + // Avoid tracking safepoints until we reach the first instruction the defines a value. for (auto it = BB.rbegin(); it != BB.rend(); ++it) { Instruction &I = *it; if (CallInst *CI = dyn_cast(&I)) { @@ -1251,18 +1235,21 @@ State LateLowerGCFrame::LocalScan(Function &F) { } auto callee = CI->getCalledFunction(); if (callee && callee == typeof_func) { - MaybeNoteDef(S, BBS, CI, BBS.Safepoints, SmallVector{-2}); + MaybeNoteDef(S, BBS, CI, SmallVector{-2}); } else if (callee && callee->getName() == "julia.gc_loaded") { continue; } else { - MaybeNoteDef(S, BBS, CI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, CI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } + bool HasDefBefore = false; if (CI->hasStructRetAttr()) { Type *ElT = getAttributeAtIndex(CI->getAttributes(), 1, Attribute::StructRet).getValueAsType(); auto tracked = CountTrackedPointers(ElT, true); if (tracked.count) { + HasDefBefore = true; auto allocas_opt = FindSretAllocas((CI->arg_begin()[0])->stripInBoundsOffsets()); // We know that with the right optimizations we can forward a sret directly from an argument // This hasn't been seen without adding IPO effects to julia functions but it's possible we need to handle that too @@ -1305,57 +1292,54 @@ State LateLowerGCFrame::LocalScan(Function &F) { } } NoteOperandUses(S, BBS, I); - if (CI->canReturnTwice()) { - S.ReturnsTwice.push_back(CI); - } - if (callee) { - if (callee == gc_preserve_begin_func) { - SmallVector args; - for (Use &U : CI->args()) { - Value *V = U; - if (isa(V)) - continue; - if (isa(V->getType())) { - if (isSpecialPtr(V->getType())) { - int Num = Number(S, V); - if (Num >= 0) + if (!CI->canReturnTwice()) { + if (callee) { + if (callee == gc_preserve_begin_func) { + SmallVector args; + for (Use &U : CI->args()) { + Value *V = U; + if (isa(V)) + continue; + if (isa(V->getType())) { + if (isSpecialPtr(V->getType())) { + int Num = Number(S, V); + if (Num >= 0) + args.push_back(Num); + } + } else { + SmallVector Nums = NumberAll(S, V); + for (int Num : Nums) { + if (Num < 0) + continue; args.push_back(Num); - } - } else { - SmallVector Nums = NumberAll(S, V); - for (int Num : Nums) { - if (Num < 0) - continue; - args.push_back(Num); + } } } + S.GCPreserves[CI] = args; + continue; + } + // Known functions emitted in codegen that are not safepoints + if (callee == pointer_from_objref_func || callee == gc_preserve_begin_func || + callee == gc_preserve_end_func || callee == typeof_func || + callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || + callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || + callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || + callee == write_barrier_func || callee == gc_loaded_func || callee == pop_handler_noexcept_func || + callee->getName() == "memcmp") { + continue; + } + if (callee->getMemoryEffects().onlyReadsMemory() || + callee->getMemoryEffects().onlyAccessesArgPointees()) { + continue; } - S.GCPreserves[CI] = args; - continue; } - // Known functions emitted in codegen that are not safepoints - if (callee == pointer_from_objref_func || callee == gc_preserve_begin_func || - callee == gc_preserve_end_func || callee == typeof_func || - callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || - callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || - callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || - callee == write_barrier_func || callee == gc_loaded_func || callee == pop_handler_noexcept_func || - callee->getName() == "memcmp") { + if (isa(CI)) + // Intrinsics are never safepoints. continue; - } - if (callee->getMemoryEffects().onlyReadsMemory() || - callee->getMemoryEffects().onlyAccessesArgPointees()) { + auto effects = CI->getMemoryEffects(); + if (effects.onlyAccessesArgPointees() || effects.onlyReadsMemory()) + // Readonly functions and functions that cannot change GC state (which is inaccessiblemem) are not safepoints continue; - } - if (MemTransferInst *MI = dyn_cast(CI)) { - MaybeTrackDst(S, MI); - } - } - if (isa(CI) || - CI->getMemoryEffects().onlyAccessesArgPointees() || - CI->getMemoryEffects().onlyReadsMemory()) { - // Intrinsics are never safepoints. - continue; } SmallVector CalleeRoots; for (Use &U : CI->args()) { @@ -1376,10 +1360,15 @@ State LateLowerGCFrame::LocalScan(Function &F) { CalleeRoots.push_back(Num); } int SafepointNumber = NoteSafepoint(S, BBS, CI, CalleeRoots); - BBS.HasSafepoint = true; - BBS.TopmostSafepoint = SafepointNumber; - BBS.Safepoints.push_back(SafepointNumber); - } else if (LoadInst *LI = dyn_cast(&I)) { + if (CI->canReturnTwice()) { + S.ReturnsTwice.push_back(SafepointNumber); + HasDefBefore = true; + } + if (HasDefBefore) // With sret, the Def happens before the instruction instead of after + BBS.FirstSafepointAfterFirstDef = SafepointNumber; + continue; + } + if (LoadInst *LI = dyn_cast(&I)) { // If this is a load from an immutable, we know that // this object will always be rooted as long as the // object we're loading from is, so we can refine uses @@ -1387,14 +1376,10 @@ State LateLowerGCFrame::LocalScan(Function &F) { // from. SmallVector RefinedPtr{}; Type *Ty = LI->getType()->getScalarType(); + bool refined_globally = false; bool task_local = false; if (isLoadFromImmut(LI) && isSpecialPtr(LI->getPointerOperand()->getType())) { RefinedPtr.push_back(Number(S, LI->getPointerOperand())); - } else if (LI->getType()->isPointerTy() && - isSpecialPtr(Ty) && - LooksLikeFrameRef(LI->getPointerOperand())) { - // Loads from a jlcall argument array - RefinedPtr.push_back(-1); } else if (isLoadFromConstGV(LI, task_local)) { // If this is a const load from a global, @@ -1402,21 +1387,26 @@ State LateLowerGCFrame::LocalScan(Function &F) { // If this is a task local constant, we don't need to root it within the // task but we do need to issue write barriers for when the current task dies. RefinedPtr.push_back(task_local ? -1 : -2); + refined_globally = true; } if (!hasLoadedTy(Ty)) - MaybeNoteDef(S, BBS, LI, BBS.Safepoints, std::move(RefinedPtr)); + if (MaybeNoteDef(S, BBS, LI, std::move(RefinedPtr))) + if (!refined_globally) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; NoteOperandUses(S, BBS, I); } else if (auto *LI = dyn_cast(&I)) { Type *Ty = LI->getNewValOperand()->getType()->getScalarType(); if (!Ty->isPointerTy() || Ty->getPointerAddressSpace() != AddressSpace::Loaded) { - MaybeNoteDef(S, BBS, LI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, LI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } NoteOperandUses(S, BBS, I); // TODO: do we need MaybeTrackStore(S, LI); } else if (auto *LI = dyn_cast(&I)) { Type *Ty = LI->getType()->getScalarType(); if (!Ty->isPointerTy() || Ty->getPointerAddressSpace() != AddressSpace::Loaded) { - MaybeNoteDef(S, BBS, LI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, LI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } NoteOperandUses(S, BBS, I); // TODO: do we need MaybeTrackStore(S, LI); @@ -1432,7 +1422,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { Number(S, SI->getFalseValue()) }; } - MaybeNoteDef(S, BBS, SI, BBS.Safepoints, std::move(RefinedPtr)); + MaybeNoteDef(S, BBS, SI, std::move(RefinedPtr)); NoteOperandUses(S, BBS, I); } else if (tracked.count) { // We need to insert extra selects for the GC roots @@ -1446,7 +1436,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { if (isa(Phi->getType())) // TODO: Vector refinements PHIRefinements = GetPHIRefinements(Phi, S); - MaybeNoteDef(S, BBS, Phi, BBS.Safepoints, std::move(PHIRefinements)); + MaybeNoteDef(S, BBS, Phi, std::move(PHIRefinements)); if (isa(Phi->getType())) { PHINumbers.push_back(Number(S, Phi)); } else { @@ -1478,7 +1468,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { RefinedPtr.push_back(task_local ? -1 : -2); } } - MaybeNoteDef(S, BBS, ASCI, BBS.Safepoints, std::move(RefinedPtr)); + MaybeNoteDef(S, BBS, ASCI, std::move(RefinedPtr)); } } else if (auto *AI = dyn_cast(&I)) { Type *ElT = AI->getAllocatedType(); @@ -1489,7 +1479,6 @@ State LateLowerGCFrame::LocalScan(Function &F) { } // Pre-seed the dataflow variables; BBS.LiveIn = BBS.UpExposedUses; - BBS.Done = true; } FixUpRefinements(PHINumbers, S); return S; @@ -1588,35 +1577,6 @@ SmallVector ExtractTrackedValues(Value *Src, Type *STy, bool isptr, I // return Ptrs.size(); //} -// turn a memcpy into a set of loads -void LateLowerGCFrame::MaybeTrackDst(State &S, MemTransferInst *MI) { - //Value *Dst = MI->getRawDest()->stripInBoundsOffsets(); - //if (AllocaInst *AI = dyn_cast(Dst)) { - // Type *STy = AI->getAllocatedType(); - // if (!AI->isStaticAlloca() || (isa(STy) && STy->getPointerAddressSpace() == AddressSpace::Tracked) || S.ArrayAllocas.count(AI)) - // return; // already numbered this - // auto tracked = CountTrackedPointers(STy); - // unsigned nroots = tracked.count * cast(AI->getArraySize())->getZExtValue(); - // if (nroots) { - // assert(!tracked.derived); - // if (!tracked.all) { - // // materialize shadow LoadInst and StoreInst ops to make a copy of just the tracked values inside - // //assert(MI->getLength() == DL.getTypeAllocSize(AI->getAllocatedType()) && !AI->isArrayAllocation()); // XXX: handle partial copy - // Value *Src = MI->getSource(); - // Src = new BitCastInst(Src, STy->getPointerTo(MI->getSourceAddressSpace()), "", MI); - // auto &Shadow = S.ShadowAllocas[AI]; - // if (!Shadow) - // Shadow = new AllocaInst(ArrayType::get(T_prjlvalue, nroots), 0, "", MI); - // AI = Shadow; - // unsigned count = TrackWithShadow(Src, STy, true, AI, IRBuilder<>(MI)); - // assert(count == tracked.count); (void)count; - // } - // S.ArrayAllocas[AI] = nroots; - // } - //} - //// TODO: else??? -} - void LateLowerGCFrame::MaybeTrackStore(State &S, StoreInst *I) { Value *PtrBase = I->getPointerOperand()->stripInBoundsOffsets(); auto tracked = CountTrackedPointers(I->getValueOperand()->getType()); @@ -1690,10 +1650,11 @@ void LateLowerGCFrame::ComputeLiveness(State &S) { // For debugging JL_USED_FUNC static void dumpSafepointsForBBName(Function &F, State &S, const char *BBName) { - for (auto it : S.SafepointNumbering) { - if (it.first->getParent()->getName() == BBName) { - dbgs() << "Live at " << *it.first << "\n"; - LargeSparseBitVector &LS = S.LiveSets[it.second]; + for (Instruction *&it : S.SafepointNumbering) { + if (it->getParent()->getName() == BBName) { + int idx = &it - S.SafepointNumbering.begin(); + dbgs() << "Live at " << idx << "\n"; + LargeSparseBitVector &LS = S.LiveSets[idx]; for (auto Idx : LS) { dbgs() << "\t"; S.ReversePtrNumbering[Idx]->printAsOperand(dbgs()); @@ -1772,9 +1733,8 @@ void LateLowerGCFrame::RefineLiveSet(LargeSparseBitVector &LS, State &S, ArrayRe void LateLowerGCFrame::ComputeLiveSets(State &S) { // Iterate over all safe points. Add to live sets all those variables that // are now live across their parent block. - for (auto it : S.SafepointNumbering) { - int idx = it.second; - Instruction *Safepoint = it.first; + for (Instruction *&Safepoint : S.SafepointNumbering) { + int idx = &Safepoint - S.SafepointNumbering.begin(); BasicBlock *BB = Safepoint->getParent(); BBState &BBS = S.BBStates[BB]; LargeSparseBitVector LiveAcross = BBS.LiveIn; @@ -1814,8 +1774,9 @@ void LateLowerGCFrame::ComputeLiveSets(State &S) { } // Compute the interference graph S.Neighbors.resize(S.MaxPtrNumber+1); - for (auto it : S.SafepointNumbering) { - const LargeSparseBitVector &LS = S.LiveSets[it.second]; + for (Instruction *&Safepoint : S.SafepointNumbering) { + int idx = &Safepoint - S.SafepointNumbering.begin(); + const LargeSparseBitVector &LS = S.LiveSets[idx]; for (int idx : LS) { S.Neighbors[idx] |= LS; } @@ -1902,8 +1863,7 @@ std::pair, int> LateLowerGCFrame::ColorRoots(const State &S) int PreAssignedColors = 0; /* First assign permanent slots to things that need them due to returns_twice */ - for (auto it : S.ReturnsTwice) { - int Num = S.SafepointNumbering.at(it); + for (int Num : S.ReturnsTwice) { const LargeSparseBitVector &LS = S.LiveSets[Num]; for (int Idx : LS) { if (Colors[Idx] == -1) @@ -1964,11 +1924,6 @@ Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Type *T_size, Value * return load; } -// Enable this optimization only on LLVM 4.0+ since this cause LLVM to optimize -// constant store loop to produce a `memset_pattern16` with a global variable -// that's initialized by `addrspacecast`. Such a global variable is not supported by the backend. -// This is not a problem on 4.0+ since that transformation (in loop-idiom) is disabled -// for NI pointers. static SmallVector *FindRefinements(Value *V, State *S) { if (!S) @@ -2361,7 +2316,9 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { return ChangesMade; } -static void AddInPredLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, State &S) +// Compute the set of all objects that are live in from all predecessors +// TODO: reset any slots that contain values which are only live from some predecessors +static void AddInPredecessorLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, State &S) { bool First = true; std::set Visited; @@ -2382,7 +2339,7 @@ static void AddInPredLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, Stat WorkList.push_back(Pred); continue; } else { - int LastSP = S.BBStates[Pred].Safepoints.front(); + int LastSP = S.BBStates[Pred].LastSafepoint; if (First) { LiveIn |= S.LiveSets[LastSP]; First = false; @@ -2447,20 +2404,18 @@ void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, { for (auto &BB : *S.F) { const BBState &BBS = S.BBStates[&BB]; - if (!BBS.HasSafepoint) { + if (!BBS.HasSafepoint) continue; - } LargeSparseBitVector LiveIn; - AddInPredLiveOuts(&BB, LiveIn, S); + AddInPredecessorLiveOuts(&BB, LiveIn, S); const LargeSparseBitVector *LastLive = &LiveIn; - for(auto rit = BBS.Safepoints.rbegin(); - rit != BBS.Safepoints.rend(); ++rit ) { - const LargeSparseBitVector &NowLive = S.LiveSets[*rit]; + for (int Safepoint = BBS.FirstSafepoint; Safepoint >= BBS.LastSafepoint; --Safepoint) { + const LargeSparseBitVector &NowLive = S.LiveSets[Safepoint]; // reset slots which are no longer alive for (int Idx : *LastLive) { if (Colors[Idx] >= PreAssignedColors && !HasBitSet(NowLive, Idx)) { PlaceGCFrameReset(S, Idx, MinColorRoot, Colors, GCFrame, - S.ReverseSafepointNumbering[*rit]); + S.SafepointNumbering[Safepoint]); } } // store values which are alive in this safepoint but @@ -2468,7 +2423,7 @@ void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, for (int Idx : NowLive) { if (!HasBitSet(*LastLive, Idx)) { PlaceGCFrameStore(S, Idx, MinColorRoot, Colors, GCFrame, - S.ReverseSafepointNumbering[*rit]); + S.SafepointNumbering[Safepoint]); } } LastLive = &NowLive; @@ -2628,16 +2583,25 @@ bool LateLowerGCFrame::runOnFunction(Function &F, bool *CFGModified) { LLVM_DEBUG(dbgs() << "GC ROOT PLACEMENT: Processing function " << F.getName() << "\n"); pgcstack = getPGCstack(F); - if (!pgcstack) - return CleanupIR(F, nullptr, CFGModified); - - State S = LocalScan(F); - ComputeLiveness(S); - auto Colors = ColorRoots(S); - std::map> CallFrames; // = OptimizeCallFrames(S, Ordering); - PlaceRootsAndUpdateCalls(Colors.first, Colors.second, S, CallFrames); - CleanupIR(F, &S, CFGModified); - + if (pgcstack) { + State S = LocalScan(F); + // If there is no safepoint after the first reachable def, then we don't need any roots (even those for allocas) + if (std::any_of(S.BBStates.begin(), S.BBStates.end(), + [&F](auto BBS) { + if (BBS.first == &F.getEntryBlock()) + return BBS.second.FirstSafepointAfterFirstDef != -1; + return BBS.second.HasSafepoint; + })) { + ComputeLiveness(S); + auto Colors = ColorRoots(S); + std::map> CallFrames; // = OptimizeCallFrames(S, Ordering); + PlaceRootsAndUpdateCalls(Colors.first, Colors.second, S, CallFrames); + } + CleanupIR(F, &S, CFGModified); + } + else { + CleanupIR(F, nullptr, CFGModified); + } // We lower the julia.gc_alloc_bytes intrinsic in this pass to insert slowpath/fastpath blocks for MMTk // For now, we do nothing for the Stock GC From c7a092a9c44956554d1609c0771481f85e2b5bb4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 25 Jun 2025 14:08:41 -0400 Subject: [PATCH 448/662] codegen: ensure safepoint functions can read the pgcstack (#58804) This needs to be readOnly over all memory, since GC could read anything (especially pgcstack), and which is not just argmem:read, but also the pointer accessed from argmem that is read from. Fix #58801 Note that this is thought not to be a problem for CleanupWriteBarriers, since while that does read the previously-inaccessibleMemOnly state, these functions are not marked nosync, so as long as the global state can be read, it also must be assumed that it might observe another thread has written to any global state. --- src/codegen.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index bdbf459908b36..e960a7ca3e5b8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -641,7 +641,7 @@ static AttributeList get_attrs_box_float(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addDereferenceableAttr(nbytes); @@ -657,7 +657,7 @@ static AttributeList get_attrs_box_sext(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addAttribute(Attribute::getWithDereferenceableBytes(C, nbytes)); @@ -674,7 +674,7 @@ static AttributeList get_attrs_box_zext(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addDereferenceableAttr(nbytes); @@ -1125,7 +1125,7 @@ static const auto jl_alloc_obj_func = new JuliaFunction{ auto FnAttrs = AttrBuilder(C); FnAttrs.addAllocSizeAttr(1, None); // returns %1 bytes FnAttrs.addAllocKindAttr(AllocFnKind::Alloc); - FnAttrs.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Ref) | MemoryEffects::inaccessibleMemOnly(ModRefInfo::ModRef)); + FnAttrs.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Ref) | MemoryEffects::inaccessibleMemOnly()); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); auto RetAttrs = AttrBuilder(C); @@ -1149,7 +1149,7 @@ static const auto jl_alloc_genericmemory_unchecked_func = new JuliaFunction{ }, [](LLVMContext &C) { AttrBuilder FnAttrs(C); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); return AttributeList::get(C, From 3ae69f688db8e5dc30bcc1256731c4732dfcd24e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 25 Jun 2025 14:14:38 -0400 Subject: [PATCH 449/662] Revert code changes from "strengthen assume_effects doc" PR (#58289) Reverts only the functional changes from JuliaLang/julia#58254, not the docs. Accessing this field here assumes that the counter valid is numeric and relevant to the current inference frame, neither of which is intended to be true, as we continue to add interfaces to execute methods outside of their current specific implementation with a monotonic world counter (e.g. with invoke on a Method, with precompile files, with external MethodTables, or with static compilation). --- Compiler/src/abstractinterpretation.jl | 14 ------------- Compiler/src/typeinfer.jl | 29 +++++++------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 8d64575331e4a..924b83c4d2142 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3527,20 +3527,6 @@ function merge_override_effects!(interp::AbstractInterpreter, effects::Effects, # It is possible for arguments (GlobalRef/:static_parameter) to throw, # but these will be recomputed during SSA construction later. override = decode_statement_effects_override(sv) - if override.consistent - m = sv.linfo.def - if isa(m, Method) - # N.B.: We'd like deleted_world here, but we can't add an appropriate edge at this point. - # However, in order to reach here in the first place, ordinary method lookup would have - # had to add an edge and appropriate invalidation trigger. - valid_worlds = WorldRange(m.primary_world, typemax(UInt)) - if sv.world.this in valid_worlds - update_valid_age!(sv, valid_worlds) - else - override = EffectsOverride(override, consistent=false) - end - end - end effects = override_effects(effects, override) set_curr_ssaflag!(sv, flags_for_effects(effects), IR_FLAGS_EFFECTS) merge_effects!(interp, sv, effects) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index dacde593fe1f2..0af8b61e74970 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -469,17 +469,11 @@ function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) return typ end -function adjust_effects(ipo_effects::Effects, def::Method, world::UInt) +function adjust_effects(ipo_effects::Effects, def::Method) # override the analyzed effects using manually annotated effect settings override = decode_effects_override(def.purity) - valid_worlds = WorldRange(0, typemax(UInt)) if is_effect_overridden(override, :consistent) - # See note on `typemax(Int)` instead of `deleted_world` in adjust_effects! - override_valid_worlds = WorldRange(def.primary_world, typemax(UInt)) - if world in override_valid_worlds - ipo_effects = Effects(ipo_effects; consistent=ALWAYS_TRUE) - valid_worlds = override_valid_worlds - end + ipo_effects = Effects(ipo_effects; consistent=ALWAYS_TRUE) end if is_effect_overridden(override, :effect_free) ipo_effects = Effects(ipo_effects; effect_free=ALWAYS_TRUE) @@ -507,7 +501,7 @@ function adjust_effects(ipo_effects::Effects, def::Method, world::UInt) if is_effect_overridden(override, :nortcall) ipo_effects = Effects(ipo_effects; nortcall=true) end - return (ipo_effects, valid_worlds) + return ipo_effects end function adjust_effects(sv::InferenceState) @@ -561,8 +555,7 @@ function adjust_effects(sv::InferenceState) # override the analyzed effects using manually annotated effect settings def = sv.linfo.def if isa(def, Method) - (ipo_effects, valid_worlds) = adjust_effects(ipo_effects, def, sv.world.this) - update_valid_age!(sv, valid_worlds) + ipo_effects = adjust_effects(ipo_effects, def) end return ipo_effects @@ -601,9 +594,9 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: end end result = me.result + result.valid_worlds = me.world.valid_worlds result.result = bestguess ipo_effects = result.ipo_effects = me.ipo_effects = adjust_effects(me) - result.valid_worlds = me.world.valid_worlds result.exc_result = me.exc_bestguess = refine_exception_type(me.exc_bestguess, ipo_effects) me.src.rettype = widenconst(ignorelimited(bestguess)) me.src.ssaflags = me.ssaflags @@ -1108,13 +1101,8 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize update_valid_age!(caller, frame.world.valid_worlds) local isinferred = is_inferred(frame) local edge = isinferred ? edge_ci : nothing - local effects, valid_worlds - if isinferred - effects = frame.result.ipo_effects # effects are adjusted already within `finish` for ipo_effects - else - (effects, valid_worlds) = adjust_effects(effects_for_cycle(frame.ipo_effects), method, frame.world.this) - update_valid_age!(caller, valid_worlds) - end + local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects + adjust_effects(effects_for_cycle(frame.ipo_effects), method) local bestguess = frame.bestguess local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: @@ -1137,8 +1125,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize # return the current knowledge about this cycle frame = frame::InferenceState update_valid_age!(caller, frame.world.valid_worlds) - (effects, valid_worlds) = adjust_effects(effects_for_cycle(frame.ipo_effects), method, frame.world.this) - update_valid_age!(caller, valid_worlds) + effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) bestguess = frame.bestguess exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, nothing, edgecycle, edgelimited)) From 9e69f44802972355441972c327044b39dfee3195 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 25 Jun 2025 22:45:59 -0400 Subject: [PATCH 450/662] build: Error when attempting to set USECLANG/USEGCC (#58795) Way back in the good old days, these used to switch between GCC and Clang. I guess these days we always auto-switch based on the CC value. If you try to directly set USECLANG, things get into a bad state. Give a better error message for that case. --- Make.inc | 8 ++++++-- contrib/asan/Make.user.asan | 1 - contrib/pgo-lto/Makefile | 2 +- contrib/tsan/Make.user.tsan | 1 - doc/src/devdocs/sanitizers.md | 7 +++---- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Make.inc b/Make.inc index f14714044b6fb..4d5d17817fa2e 100644 --- a/Make.inc +++ b/Make.inc @@ -521,6 +521,10 @@ endif # Compiler specific stuff +ifneq ($(USECLANG)$(USEGCC),) +$(error "These internal variables are not overridable. Set CC instead.") +endif + ifeq (default,$(origin CC)) CC := $(CROSS_COMPILE)$(CC) # attempt to add cross-compiler prefix, if the user # is not overriding the default, to form target-triple-cc (which @@ -1661,8 +1665,8 @@ endif # Note: we're passing *FLAGS here computed based on your system compiler to # clang. If that causes you problems, you might want to build and/or run # specific clang-sa-* files with clang explicitly selected: -# make CC=~+/../usr/tools/clang CXX=~+/../usr/tools/clang USECLANG=1 analyzegc -# make USECLANG=1 clang-sa-* +# make CC=~+/../usr/tools/clang CXX=~+/../usr/tools/clang analyzegc +# make clang-sa-* CLANGSA_FLAGS := CLANGSA_CXXFLAGS := ifeq ($(OS), Darwin) # on new XCode, the files are hidden diff --git a/contrib/asan/Make.user.asan b/contrib/asan/Make.user.asan index 025cfad82214b..1ad8d3c8fb1f7 100644 --- a/contrib/asan/Make.user.asan +++ b/contrib/asan/Make.user.asan @@ -3,7 +3,6 @@ BINDIR=$(TOOLCHAIN)/usr/bin TOOLDIR=$(TOOLCHAIN)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLDIR)/clang override CXX=$(TOOLDIR)/clang++ override PATCHELF=$(TOOLDIR)/patchelf diff --git a/contrib/pgo-lto/Makefile b/contrib/pgo-lto/Makefile index ddd86f5d5b39a..5902d4ad08151 100644 --- a/contrib/pgo-lto/Makefile +++ b/contrib/pgo-lto/Makefile @@ -32,7 +32,7 @@ STAGE2_FLAGS:=LDFLAGS="-fuse-ld=lld -flto=thin -Wl,--undefined-version -fprofile CFLAGS="-fprofile-use=$(PROFILE_FILE)" $\ CXXFLAGS="-fprofile-use=$(PROFILE_FILE)" -COMMON_FLAGS:=USECLANG=1 USE_BINARYBUILDER_LLVM=0 +COMMON_FLAGS:=USE_BINARYBUILDER_LLVM=0 all: stage2 # Default target as first in file diff --git a/contrib/tsan/Make.user.tsan b/contrib/tsan/Make.user.tsan index b192c36e4cfee..e4107222aba29 100644 --- a/contrib/tsan/Make.user.tsan +++ b/contrib/tsan/Make.user.tsan @@ -3,7 +3,6 @@ BINDIR=$(TOOLCHAIN)/usr/bin TOOLDIR=$(TOOLCHAIN)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLDIR)/clang override CXX=$(TOOLDIR)/clang++ diff --git a/doc/src/devdocs/sanitizers.md b/doc/src/devdocs/sanitizers.md index 5eaf4b45d9f57..dd5b518f54840 100644 --- a/doc/src/devdocs/sanitizers.md +++ b/doc/src/devdocs/sanitizers.md @@ -21,14 +21,14 @@ If you require customization or further detail, see the documentation below. ## General considerations -Using Clang's sanitizers obviously requires you to use Clang (`USECLANG=1`), but there's another +Using Clang's sanitizers obviously requires you to use Clang, but there's another catch: most sanitizers require a run-time library, provided by the host compiler, while the instrumented code generated by Julia's JIT relies on functionality from that library. This implies that the LLVM version of your host compiler must match that of the LLVM library used within Julia. An easy solution is to have a dedicated build folder for providing a matching toolchain, by building with `BUILD_LLVM_CLANG=1`. You can then refer to this toolchain from another build -folder by specifying `USECLANG=1` while overriding the `CC` and `CXX` variables. +folder by overriding the `CC` and `CXX` variables. The sanitizers error out when they detect a shared library being opened using `RTLD_DEEPBIND` (ref: [google/sanitizers#611](https://github.com/google/sanitizers/issues/611)). @@ -44,7 +44,7 @@ look like this, plus one or more of the `SANITIZE_*` flags listed below: make -C deps USE_BINARYBUILDER_LLVM=0 LLVM_VER=svn stage-llvm - make -C src SANITIZE=1 USECLANG=1 \ + make -C src SANITIZE=1 \ CC=~+/deps/scratch/llvm-svn/build_Release/bin/clang \ CXX=~+/deps/scratch/llvm-svn/build_Release/bin/clang++ \ CPPFLAGS="-isysroot $(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" \ @@ -99,7 +99,6 @@ Checkout a Git worktree (or create out-of-tree build directory) at TOOLCHAIN=$(TOOLCHAIN_WORKTREE)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLCHAIN)/clang override CXX=$(TOOLCHAIN)/clang++ export ASAN_SYMBOLIZER_PATH=$(TOOLCHAIN)/llvm-symbolizer From b4a6288a38a39296feb712abc83ae7beefcdff37 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 25 Jun 2025 22:46:26 -0400 Subject: [PATCH 451/662] build: Add --no-same-owner to TAR (#58796) tar changes behavior when the current uid is 0 to try to also restore owner uids/gids (if recorded). It is possible for the uid to be 0 in single-uid environments like user namespace sandboxes, in which case the attempt to change the uid/gid fails. Of course ideally, the tars would have been created non-archival (so that the uid/gid wasn't recorded in the first place), but we get source tars from various places, so we can't guarantee this. To make sure we don't run into trouble, manually add the --no-same-owner flag to disable this behavior. --- Make.inc | 3 +++ Makefile | 2 +- contrib/mac/app/Makefile | 2 +- deps/curl.mk | 2 +- deps/patchelf.mk | 2 +- deps/pcre.mk | 2 +- deps/unwind.mk | 4 ++-- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Make.inc b/Make.inc index 4d5d17817fa2e..10a3ccaf5e229 100644 --- a/Make.inc +++ b/Make.inc @@ -831,6 +831,9 @@ $(error "please install either GNU tar or bsdtar") endif endif +# Do not try to extract owner uids, even if we're root (e.g. in sandboxes) +TAR += --no-same-owner + ifeq ($(WITH_GC_VERIFY), 1) JCXXFLAGS += -DGC_VERIFY JCFLAGS += -DGC_VERIFY diff --git a/Makefile b/Makefile index b3aeb4fb56286..c5367a5a429a4 100644 --- a/Makefile +++ b/Makefile @@ -590,7 +590,7 @@ endif ifeq ($(OS), WINNT) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe endif - cd $(BUILDROOT) && $(TAR) zcvf $(JULIA_BINARYDIST_FILENAME).tar.gz julia-$(JULIA_COMMIT) + cd $(BUILDROOT) && $(TAR) -zcvf $(JULIA_BINARYDIST_FILENAME).tar.gz julia-$(JULIA_COMMIT) exe: diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index 81b7e47cdf2cf..a8fd6e16b3f44 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -48,7 +48,7 @@ dmg/$(APP_NAME): startup.applescript julia.icns plutil -insert NSHumanReadableCopyright -string "$(APP_COPYRIGHT)" $@/Contents/Info.plist -mkdir -p $@/Contents/Resources/julia make -C $(JULIAHOME) binary-dist - tar zxf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 + $(TAR) -xzf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 find $@/Contents/Resources/julia -type f -exec chmod -w {} \; # Even though the tarball may already be signed, we re-sign here to make it easier to add # unsigned executables (like the app launcher) and whatnot, without needing to maintain lists diff --git a/deps/curl.mk b/deps/curl.mk index 6232d56e5e333..fa106ac2259fc 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -31,7 +31,7 @@ $(SRCCACHE)/curl-$(CURL_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/curl-$(CURL_VER)/source-extracted: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) jxf $(notdir $<) + cd $(dir $<) && $(TAR) -jxf $(notdir $<) echo 1 > $@ checksum-curl: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 diff --git a/deps/patchelf.mk b/deps/patchelf.mk index c019892058d0e..aaf0ecb313b80 100644 --- a/deps/patchelf.mk +++ b/deps/patchelf.mk @@ -7,7 +7,7 @@ $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted: $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2 $(JLCHECKSUM) $< mkdir $(dir $@) - cd $(dir $@) && $(TAR) jxf $< --strip-components=1 + cd $(dir $@) && $(TAR) -jxf $< --strip-components=1 touch -c $(SRCCACHE)/patchelf-$(PATCHELF_VER)/configure # old target echo 1 > $@ diff --git a/deps/pcre.mk b/deps/pcre.mk index 3ff85d5569ad9..8fbdb1ba79a35 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -21,7 +21,7 @@ $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) jxf $(notdir $<) + cd $(dir $<) && $(TAR) -jxf $(notdir $<) echo 1 > $@ checksum-pcre: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 diff --git a/deps/unwind.mk b/deps/unwind.mk index 0808ae6ca4b1c..c11338d0162ed 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -20,7 +20,7 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz: | $(SRCCACHE) $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted: $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) xfz $< + cd $(dir $<) && $(TAR) -xfz $< touch -c $(SRCCACHE)/libunwind-$(UNWIND_VER)/configure # old target echo 1 > $@ @@ -102,7 +102,7 @@ $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).tar.xz: | $(SRCCACHE) $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER)/source-extracted: $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).tar.xz $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) xf $< + cd $(dir $<) && $(TAR) -xf $< mv $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).src $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER) echo 1 > $@ From 3076a9683609b5c587ae3cbc70b25b7332018c7f Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Thu, 26 Jun 2025 07:35:02 -0400 Subject: [PATCH 452/662] Add `cfunction` support for `--trim` (#58812) --- Compiler/src/typeinfer.jl | 26 ++++++++++++++++++++------ Compiler/src/verifytrim.jl | 24 +++++++++++++++++++++--- Compiler/test/verifytrim.jl | 27 +++++++++++++++------------ src/codegen.cpp | 20 ++++++++++---------- test/trimming/basic_jll.jl | 15 +++++++++++++-- 5 files changed, 79 insertions(+), 33 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 0af8b61e74970..e5291bbaebd3d 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1480,16 +1480,30 @@ function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vec # No dynamic dispatch to resolve / enqueue continue end + elseif isexpr(stmt, :cfunction) && length(stmt.args) == 5 + (pointer_type, f, rt, at, call_type) = stmt.args + linfo = ci.parent - let workqueue = invokelatest_queue - # make a best-effort attempt to enqueue the relevant code for the finalizer - mi = compileable_specialization_for_call(workqueue.interp, atype) - mi === nothing && continue + linfo isa MethodInstance || continue + at isa SimpleVector || continue - push!(workqueue, mi) + ft = argextype(f, ci, sptypes) + argtypes = Any[ft] + for i = 1:length(at) + push!(argtypes, sp_type_rewrap(at[i], linfo, #= isreturn =# false)) end + atype = argtypes_to_type(argtypes) + else + # TODO: handle other StmtInfo like OpaqueClosure? + continue + end + let workqueue = invokelatest_queue + # make a best-effort attempt to enqueue the relevant code for the dynamic invokelatest call + mi = compileable_specialization_for_call(workqueue.interp, atype) + mi === nothing && continue + + push!(workqueue, mi) end - # TODO: handle other StmtInfo like @cfunction and OpaqueClosure? end end diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 09a189b2ff223..eb775bfa290ce 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -14,7 +14,7 @@ using ..Compiler: argextype, empty!, error, get, get_ci_mi, get_world_counter, getindex, getproperty, hasintersect, haskey, in, isdispatchelem, isempty, isexpr, iterate, length, map!, max, pop!, popfirst!, push!, pushfirst!, reinterpret, reverse!, reverse, setindex!, - setproperty!, similar, singleton_type, sptypes_from_meth_instance, + setproperty!, similar, singleton_type, sptypes_from_meth_instance, sp_type_rewrap, unsafe_pointer_to_objref, widenconst, isconcretetype, # misc @nospecialize, @assert, C_NULL @@ -256,9 +256,27 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, warn = true # downgrade must-throw calls to be only a warning end elseif isexpr(stmt, :cfunction) + length(stmt.args) != 5 && continue # required by IR legality + (pointer_type, f, rt, at, call_type) = stmt.args + + at isa SimpleVector || continue # required by IR legality + ft = argextype(f, codeinfo, sptypes) + argtypes = Any[ft] + for i = 1:length(at) + push!(argtypes, sp_type_rewrap(at[i], get_ci_mi(codeinst), #= isreturn =# false)) + end + atype = argtypes_to_type(argtypes) + + mi = compileable_specialization_for_call(interp, atype) + if mi !== nothing + # n.b.: Codegen may choose unpredictably to emit this `@cfunction` as a dynamic invoke or a full + # dynamic call, but in either case it guarantees that the required adapter(s) are emitted. All + # that we are required to verify here is that the callee CodeInstance is covered. + ci = get(caches, mi, nothing) + ci isa CodeInstance && continue + end + error = "unresolved cfunction" - #TODO: parse the cfunction expression to check the target is defined - warn = true elseif isexpr(stmt, :foreigncall) foreigncall = stmt.args[1] if foreigncall isa QuoteNode diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index e7d57571b1db0..a84afd6933266 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -36,24 +36,27 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), \[1\] finalizer\(f::Any, o::Any\)""", repr) end +# test that basic `cfunction` generation is allowed, when the dispatch target can be resolved make_cfunction() = @cfunction(+, Float64, (Int64,Int64)) +let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) +end # use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel -let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE) +make_cfunction_bad(@nospecialize(f::Any)) = @cfunction($f, Float64, (Int64,Int64))::Base.CFunction +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.CFunction, Tuple{typeof(make_cfunction_bad), Any})], [Base.get_world_counter()], TRIM_UNSAFE) errors, parents = get_verify_typeinf_trim(infos) - @test_broken isempty(errors) # missing cfunction + @test !isempty(errors) # missing cfunction - desc = only(errors) - @test desc.first - desc = desc.second + (is_warning, desc) = only(errors) + @test !is_warning @test desc isa CallMissing @test occursin("cfunction", desc.desc) repr = sprint(verify_print_error, desc, parents) - @test occursin( - r"""^unresolved cfunction from statement \$\(Expr\(:cfunction, Ptr{Nothing}, :\(\$\(QuoteNode\(\+\)\)\), Float64, :\(svec\(Int64, Int64\)::Core.SimpleVector\), :\(:ccall\)\)\)::Ptr{Nothing} + @test occursin(r"""^unresolved cfunction from statement \$\(Expr\(:cfunction, Base.CFunction, :\(f::Any\), Float64, :\(svec\(Int64, Int64\)::Core.SimpleVector\), :\(:ccall\)\)\)::Base.CFunction Stacktrace: - \[1\] make_cfunction\(\)""", repr) - + \[1\] make_cfunction_bad\(f::Any\)""", repr) resize!(infos, 1) @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type errors, parents = get_verify_typeinf_trim(infos) @@ -61,11 +64,11 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfu @test !desc.first desc = desc.second @test desc isa CCallableMissing - @test desc.rt == Ptr{Cvoid} - @test desc.sig == Tuple{typeof(make_cfunction)} + @test desc.rt == Base.CFunction + @test desc.sig == Tuple{typeof(make_cfunction_bad), Any} @test occursin("unresolved ccallable", desc.desc) repr = sprint(verify_print_error, desc, parents) - @test repr == "unresolved ccallable for Tuple{$(typeof(make_cfunction))} => Ptr{Nothing}\n\n" + @test repr == "unresolved ccallable for Tuple{$(typeof(make_cfunction_bad)), Any} => Base.CFunction\n\n" end let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) diff --git a/src/codegen.cpp b/src/codegen.cpp index e960a7ca3e5b8..88b6e9f1f4513 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6356,8 +6356,8 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } else if (head == jl_cfunction_sym) { assert(nargs == 5); - jl_cgval_t fexpr_rt = emit_expr(ctx, args[1]); - return emit_cfunction(ctx, args[0], fexpr_rt, args[2], (jl_svec_t*)args[3]); + jl_cgval_t fexpr_val = emit_expr(ctx, args[1]); + return emit_cfunction(ctx, args[0], fexpr_val, args[2], (jl_svec_t*)args[3]); } else if (head == jl_assign_sym) { assert(nargs == 2); @@ -7576,7 +7576,7 @@ static const char *derive_sigt_name(jl_value_t *jargty) // Get the LLVM Function* for the C-callable entry point for a certain function // and argument types. // here argt does not include the leading function type argument -static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, const jl_cgval_t &fexpr_rt, jl_value_t *declrt, jl_svec_t *argt) +static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, const jl_cgval_t &fexpr_val, jl_value_t *declrt, jl_svec_t *argt) { jl_unionall_t *unionall_env = (jl_is_method(ctx.linfo->def.method) && jl_is_unionall(ctx.linfo->def.method->sig)) ? (jl_unionall_t*)ctx.linfo->def.method->sig @@ -7634,8 +7634,8 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // compute+verify the dispatch signature, and see if it depends on the environment sparams bool approx = false; sigt = (jl_value_t*)jl_alloc_svec(nargt + 1); - jl_svecset(sigt, 0, fexpr_rt.typ); - if (!fexpr_rt.constant && (!jl_is_concrete_type(fexpr_rt.typ) || jl_is_kind(fexpr_rt.typ))) + jl_svecset(sigt, 0, fexpr_val.typ); + if (!fexpr_val.constant && (!jl_is_concrete_type(fexpr_val.typ) || jl_is_kind(fexpr_val.typ))) approx = true; for (size_t i = 0; i < nargt; i++) { jl_value_t *jargty = jl_svecref(argt, i); @@ -7664,7 +7664,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con unionall_env = NULL; } - bool nest = (!fexpr_rt.constant || unionall_env); + bool nest = (!fexpr_val.constant || unionall_env); if (ctx.emission_context.TargetTriple.isAArch64() || ctx.emission_context.TargetTriple.isARM() || ctx.emission_context.TargetTriple.isPPC64()) { if (nest) { emit_error(ctx, "cfunction: closures are not supported on this platform"); @@ -7672,17 +7672,17 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } } - const char *name = derive_sigt_name(fexpr_rt.typ); + const char *name = derive_sigt_name(fexpr_val.typ); Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, - sig, fexpr_rt.constant, name, + sig, fexpr_val.constant, name, declrt, sigt, unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { // F is actually an init_trampoline function that returns the real address // Now fill in the nest parameters - Value *fobj = boxed(ctx, fexpr_rt); + Value *fobj = boxed(ctx, fexpr_val); jl_svec_t *fill = jl_emptysvec; if (closure_types) { assert(ctx.spvals_ptr); @@ -7722,7 +7722,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.decorateInst(ctx.builder.CreateStore(F, derived_strct)); ai.decorateInst(ctx.builder.CreateStore( - ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, fexpr_rt.constant), ctx.types().T_size), + ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, fexpr_val.constant), ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 1))); ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 2))); diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index 659495e3340a6..fc34b34096c26 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -1,10 +1,21 @@ using Libdl using Zstd_jll # Note this uses the vendored older non-LazyLibrary version of Zstd_jll +function print_string(fptr::Ptr{Cvoid}) + println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) +end + function @main(args::Vector{String})::Cint + # Test the basic "Hello, world!" println(Core.stdout, "Julia! Hello, world!") - fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) - println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) + + # Make sure that JLL's are working as expected println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ()))) + + # Add an indirection via `@cfunction` + cfunc = @cfunction(print_string, Cvoid, (Ptr{Cvoid},)) + fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) + ccall(cfunc, Cvoid, (Ptr{Cvoid},), fptr) + return 0 end From 5416edb80767aa0b5d00b18baf26bf9245ad3d9d Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:56:22 +0200 Subject: [PATCH 453/662] fix error message for `eachindex(::Vararg{Tuple})` (#58811) Make the error message in case of mismatch less confusing and consistent with the error message for arrays. While at it, also made other changes of the same line of source code: * use function composition instead of an anonymous closure * expand the one-liner into a multiline `if` --------- Co-authored-by: Andy Dienes <51664769+adienes@users.noreply.github.com> --- base/tuple.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/tuple.jl b/base/tuple.jl index 937e130d3cd93..ac42c667269e6 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -150,7 +150,11 @@ nextind(@nospecialize(t::Tuple), i::Integer) = Int(i)+1 function keys(t::Tuple, t2::Tuple...) @inline lent = length(t) - all(x->length(x) == lent, t2) || throw_eachindex_mismatch_indices(IndexLinear(), t, t2...) + if !all(==(lent) ∘ length, t2) + let inds = map(only ∘ axes, (t, t2...)) + throw_eachindex_mismatch_indices("indices", inds...) + end + end Base.OneTo(lent) end From 4d40c6480bff6cefcb2112653b4e64d63e2547ea Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:37:11 +0900 Subject: [PATCH 454/662] use more canonical way to check binding existence (#58809) --- Compiler/src/ssair/verify.jl | 2 +- base/docs/bindings.jl | 4 ++-- base/errorshow.jl | 4 ++-- base/show.jl | 8 ++++---- src/gen_sysimg_symtab.jl | 4 ++-- stdlib/InteractiveUtils/src/InteractiveUtils.jl | 10 +++++----- test/core.jl | 4 ++-- test/docs.jl | 2 +- test/error.jl | 4 ++-- test/numbers.jl | 4 ++-- test/precompile.jl | 10 +++++----- test/reflection.jl | 4 ++-- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index f9a9bb674a536..3aa1e00e3f2d3 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -irshow_was_loaded() = invokelatest(isdefined, Compiler.IRShow, :debuginfo_firstline) +irshow_was_loaded() = invokelatest(isdefinedglobal, Compiler.IRShow, :debuginfo_firstline) function maybe_show_ir(ir::IRCode) if irshow_was_loaded() # ensure we use I/O that does not yield, as this gets called during compilation diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index 5a0e8e01762e2..34aa87bd13076 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -16,8 +16,8 @@ end bindingexpr(x) = Expr(:call, Binding, splitexpr(x)...) -defined(b::Binding) = invokelatest(isdefined, b.mod, b.var) -resolve(b::Binding) = invokelatest(getfield, b.mod, b.var) +defined(b::Binding) = invokelatest(isdefinedglobal, b.mod, b.var) +resolve(b::Binding) = invokelatest(getglobal, b.mod, b.var) function splitexpr(x::Expr) isexpr(x, :macrocall) ? splitexpr(x.args[1]) : diff --git a/base/errorshow.jl b/base/errorshow.jl index 624461a31c641..d876f71520df6 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -339,8 +339,8 @@ function showerror(io::IO, ex::MethodError) # Check all modules (sorted for consistency) sorted_modules = sort!(collect(modules_to_check), by=nameof) for mod in sorted_modules - if isdefined(mod, name) - candidate = getfield(mod, name) + if isdefinedglobal(mod, name) + candidate = getglobal(mod, name) if candidate !== f && hasmethod(candidate, arg_types; world=ex.world) if mod === Base print(io, "\nYou may have intended to import ") diff --git a/base/show.jl b/base/show.jl index 73a3a0fcad66f..f89e72ff3ab53 100644 --- a/base/show.jl +++ b/base/show.jl @@ -632,8 +632,8 @@ function make_typealias(@nospecialize(x::Type)) x isa UnionAll && push!(xenv, x) for mod in mods for name in unsorted_names(mod) - if isdefined(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) - alias = getfield(mod, name) + if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) + alias = getglobal(mod, name) if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && x <: alias if alias isa UnionAll (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector @@ -836,8 +836,8 @@ function make_typealiases(@nospecialize(x::Type)) x isa UnionAll && push!(xenv, x) for mod in mods for name in unsorted_names(mod) - if isdefined(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) - alias = getfield(mod, name) + if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) + alias = getglobal(mod, name) if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && !(alias <: Tuple) (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector ti === Union{} && continue diff --git a/src/gen_sysimg_symtab.jl b/src/gen_sysimg_symtab.jl index a91f2f994194c..110d83ba9083d 100644 --- a/src/gen_sysimg_symtab.jl +++ b/src/gen_sysimg_symtab.jl @@ -11,8 +11,8 @@ import Base.Iterators: take, drop function _eachmethod(f, m::Module, visited, vmt) push!(visited, m) for nm in names(m, all=true) - if isdefined(m, nm) - x = getfield(m, nm) + if isdefinedglobal(m, nm) + x = getglobal(m, nm) if isa(x, Module) && !in(x, visited) _eachmethod(f, x, visited, vmt) elseif isa(x, Type) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 0f69860f39d78..8499027fae6cc 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -53,7 +53,7 @@ function varinfo(m::Module=Base.active_module(), pattern::Regex=r""; all::Bool = if !isdefined(m2, v) || !occursin(pattern, string(v)) continue end - value = getfield(m2, v) + value = getglobal(m2, v) isbuiltin = value === Base || value === Base.active_module() || value === Core if recursive && !isbuiltin && isa(value, Module) && value !== m2 && nameof(value) === v && parentmodule(value) === m2 push!(workqueue, (value, "$prep$v.")) @@ -232,8 +232,8 @@ end function _methodswith(@nospecialize(t::Type), m::Module, supertypes::Bool) meths = Method[] for nm in names(m) - if isdefined(m, nm) - f = getfield(m, nm) + if isdefinedglobal(m, nm) + f = getglobal(m, nm) if isa(f, Base.Callable) methodswith(t, f, meths; supertypes = supertypes) end @@ -264,8 +264,8 @@ function _subtypes_in!(mods::Array, x::Type) m = pop!(mods) xt = xt::DataType for s in names(m, all = true) - if !isdeprecated(m, s) && isdefined(m, s) - t = getfield(m, s) + if !isdeprecated(m, s) && isdefinedglobal(m, s) + t = getglobal(m, s) dt = isa(t, UnionAll) ? unwrap_unionall(t) : t if isa(dt, DataType) if dt.name.name === s && dt.name.module == m && supertype(dt).name == xt.name diff --git a/test/core.jl b/test/core.jl index 4abe4f95a2e0b..997677cd69e39 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1206,8 +1206,8 @@ end # Module() constructor @test names(Module(:anonymous), all = true, imported = true) == [:anonymous] @test names(Module(:anonymous, false), all = true, imported = true) == [:anonymous] -@test invokelatest(getfield, Module(:anonymous, false, true), :Core) == Core -@test_throws UndefVarError invokelatest(getfield, Module(:anonymous, false, false), :Core) +@test invokelatest(getglobal, Module(:anonymous, false, true), :Core) == Core +@test_throws UndefVarError invokelatest(getglobal, Module(:anonymous, false, false), :Core) # exception from __init__() let didthrow = diff --git a/test/docs.jl b/test/docs.jl index 568cab48bd1e5..6c625735ecc7e 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -119,7 +119,7 @@ end # issue #38819 module NoDocStrings end -@test meta(NoDocStrings) === invokelatest(getfield, NoDocStrings, Base.Docs.META) +@test meta(NoDocStrings) === invokelatest(getglobal, NoDocStrings, Base.Docs.META) # General tests for docstrings. diff --git a/test/error.jl b/test/error.jl index f76a7809b08a9..0d8047aa92a44 100644 --- a/test/error.jl +++ b/test/error.jl @@ -109,8 +109,8 @@ end if mod ∉ visited push!(visited, mod) for name in names(mod, all=true) - isdefined(mod, name) || continue - value = getfield(mod, name) + isdefinedglobal(mod, name) || continue + value = getglobal(mod, name) if value isa Module value === Main && continue test_exceptions(value, visited) diff --git a/test/numbers.jl b/test/numbers.jl index 42eeeec4257c7..0553ab342f7a2 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2420,8 +2420,8 @@ end function allsubtypes!(m::Module, x::DataType, sts::Set) for s in names(m, all = true) - if isdefined(m, s) && !Base.isdeprecated(m, s) - t = getfield(m, s) + if isdefinedglobal(m, s) && !Base.isdeprecated(m, s) + t = getglobal(m, s) if isa(t, Type) && t <: x && t != Union{} push!(sts, t) elseif isa(t, Module) && t !== m && nameof(t) === s && parentmodule(t) === m diff --git a/test/precompile.jl b/test/precompile.jl index e95df0d50ae60..7cf9bdffadfc1 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -778,12 +778,12 @@ precompile_test_harness("code caching") do dir Base.compilecache(pkgid) @test Base.isprecompiled(pkgid) @eval using $Cache_module - M = invokelatest(getfield, @__MODULE__, Cache_module) + M = invokelatest(getglobal, @__MODULE__, Cache_module) Mid = rootid(M) invokelatest() do # Test that this cache file "owns" all the roots for name in (:f, :fpush, :callboth) - func = getfield(M, name) + func = getglobal(M, name) m = only(collect(methods(func))) @test all(i -> root_provenance(m, i) == Mid, 1:length(m.roots)) end @@ -1033,7 +1033,7 @@ precompile_test_harness("code caching") do dir Base.compilecache(Base.PkgId(string(pkg))) end @eval using $StaleA - MA = invokelatest(getfield, @__MODULE__, StaleA) + MA = invokelatest(getglobal, @__MODULE__, StaleA) Base.eval(MA, :(nbits(::UInt8) = 8)) Base.eval(MA, quote struct InvalidatedBinding @@ -1154,7 +1154,7 @@ precompile_test_harness("precompiletools") do dir Base.compilecache(pkgid) @test Base.isprecompiled(pkgid) @eval using $PrecompileToolsModule - M = invokelatest(getfield, @__MODULE__, PrecompileToolsModule) + M = invokelatest(getglobal, @__MODULE__, PrecompileToolsModule) invokelatest() do m = which(Tuple{typeof(findfirst), Base.Fix2{typeof(==), T}, Vector{T}} where T) success = 0 @@ -1281,7 +1281,7 @@ precompile_test_harness("invoke") do dir """) Base.compilecache(Base.PkgId(string(CallerModule))) @eval using $InvokeModule: $InvokeModule - MI = invokelatest(getfield, @__MODULE__, InvokeModule) + MI = invokelatest(getglobal, @__MODULE__, InvokeModule) @eval $MI.getlast(a::UnitRange) = a.stop @eval using $CallerModule invokelatest() do diff --git a/test/reflection.jl b/test/reflection.jl index 8cf2385ab8b18..29774c2bfa069 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -617,9 +617,9 @@ function module_depth(from::Module, to::Module) end function has_backslashes(mod::Module) for n in names(mod, all = true, imported = true) - isdefined(mod, n) || continue + isdefinedglobal(mod, n) || continue Base.isdeprecated(mod, n) && continue - f = getfield(mod, n) + f = getglobal(mod, n) if isa(f, Module) && module_depth(Main, f) <= module_depth(Main, mod) continue end From 4ae3f5ebed5378909f6355fb03b00057052c6766 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 27 Jun 2025 10:32:19 -0400 Subject: [PATCH 455/662] Add `trim_mode` parameter to JIT type-inference entrypoint (#58817) Resolves https://github.com/JuliaLang/julia/issues/58786. I think this is only a partial fix, since we can still end up loading code from pkgimages that has been poorly inferred due to running without these `InferenceParams`. However, many of the common scenarios (such as JLL's depending on each other) seem to be OK since we have a targeted heuristic that adds `__init__()` to a pkgimage only if the module has inference enabled. --- Compiler/src/bootstrap.jl | 4 ++-- Compiler/src/typeinfer.jl | 15 ++++++++------- src/aotcompile.cpp | 4 ++-- src/gf.c | 15 ++++++++------- src/julia_internal.h | 2 +- src/runtime_ccall.cpp | 2 +- src/toplevel.c | 2 +- test/trimming/basic_jll.jl | 12 +++++++++--- 8 files changed, 32 insertions(+), 24 deletions(-) diff --git a/Compiler/src/bootstrap.jl b/Compiler/src/bootstrap.jl index a847d1fb835c7..74943fc765f17 100644 --- a/Compiler/src/bootstrap.jl +++ b/Compiler/src/bootstrap.jl @@ -10,7 +10,7 @@ function activate_codegen!() Core.eval(Compiler, quote let typeinf_world_age = Base.tls_world_age() @eval Core.OptimizedGenerics.CompilerPlugins.typeinf(::Nothing, mi::MethodInstance, source_mode::UInt8) = - Base.invoke_in_world($(Expr(:$, :typeinf_world_age)), typeinf_ext_toplevel, mi, Base.tls_world_age(), source_mode) + Base.invoke_in_world($(Expr(:$, :typeinf_world_age)), typeinf_ext_toplevel, mi, Base.tls_world_age(), source_mode, Compiler.TRIM_NO) end end) end @@ -67,7 +67,7 @@ function bootstrap!() end mi = specialize_method(m.method, Tuple{params...}, m.sparams) #isa_compileable_sig(mi) || println(stderr, "WARNING: inferring `", mi, "` which isn't expected to be called.") - typeinf_ext_toplevel(mi, world, isa_compileable_sig(mi) ? SOURCE_MODE_ABI : SOURCE_MODE_NOT_REQUIRED) + typeinf_ext_toplevel(mi, world, isa_compileable_sig(mi) ? SOURCE_MODE_ABI : SOURCE_MODE_NOT_REQUIRED, TRIM_NO) end end end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index e5291bbaebd3d..9849d22a9ce67 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1564,8 +1564,9 @@ function typeinf_ext_toplevel(interp::AbstractInterpreter, mi::MethodInstance, s end # This is a bridge for the C code calling `jl_typeinf_func()` on a single Method match -function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt8) - interp = NativeInterpreter(world) +function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt8, trim_mode::UInt8) + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + interp = NativeInterpreter(world; inf_params) return typeinf_ext_toplevel(interp, mi, source_mode) end @@ -1648,11 +1649,11 @@ end # This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches # The trim_mode can be any of: -const TRIM_NO = 0 -const TRIM_SAFE = 1 -const TRIM_UNSAFE = 2 -const TRIM_UNSAFE_WARN = 3 -function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) +const TRIM_NO = 0x0 +const TRIM_SAFE = 0x1 +const TRIM_UNSAFE = 0x2 +const TRIM_UNSAFE_WARN = 0x3 +function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::UInt8) inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) # Create an "invokelatest" queue to enable eager compilation of speculative diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 72b68c14636be..9bbc70ce0b001 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -712,7 +712,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[2] = (jl_value_t*)worlds; jl_array_data(worlds, size_t)[0] = jl_typeinf_world; jl_array_data(worlds, size_t)[compiler_world] = world; // might overwrite previous - fargs[3] = jl_box_long(trim); + fargs[3] = jl_box_uint8(trim); size_t last_age = ct->world_age; ct->world_age = jl_typeinf_world; codeinfos = (jl_array_t*)jl_apply(fargs, 4); @@ -2488,7 +2488,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); if (mi == nullptr) continue; - jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED); + jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); if (codeinst == nullptr || compiled_functions.count(codeinst)) continue; orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx, DL, TT); diff --git a/src/gf.c b/src/gf.c index afd0199a61518..75398cbb39b37 100644 --- a/src/gf.c +++ b/src/gf.c @@ -403,7 +403,7 @@ static jl_code_instance_t *jl_method_inferred_with_abi(jl_method_instance_t *mi // returns the inferred source, and may cache the result in mi // if successful, also updates the mi argument to describe the validity of this src // if inference doesn't occur (or can't finish), returns NULL instead -jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_t source_mode) +jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_t source_mode, uint8_t trim_mode) { if (jl_typeinf_func == NULL) { if (source_mode == SOURCE_MODE_ABI) @@ -427,11 +427,12 @@ jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_ return NULL; JL_TIMING(INFERENCE, INFERENCE); jl_value_t **fargs; - JL_GC_PUSHARGS(fargs, 4); + JL_GC_PUSHARGS(fargs, 5); fargs[0] = (jl_value_t*)jl_typeinf_func; fargs[1] = (jl_value_t*)mi; fargs[2] = jl_box_ulong(world); fargs[3] = jl_box_uint8(source_mode); + fargs[4] = jl_box_uint8(trim_mode); int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -458,7 +459,7 @@ jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_ // allocate another bit for the counter. ct->reentrant_timing += 0b10; JL_TRY { - ci = (jl_code_instance_t*)jl_apply(fargs, 4); + ci = (jl_code_instance_t*)jl_apply(fargs, 5); } JL_CATCH { jl_value_t *e = jl_current_exception(ct); @@ -3196,7 +3197,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t int should_skip_inference = !jl_is_method(mi->def.method) || jl_symbol_name(mi->def.method->name)[0] == '@'; if (!should_skip_inference) { - codeinst = jl_type_infer(mi, world, SOURCE_MODE_ABI); + codeinst = jl_type_infer(mi, world, SOURCE_MODE_ABI, jl_options.trim); } } @@ -3516,7 +3517,7 @@ static void _generate_from_hint(jl_method_instance_t *mi, size_t world) { jl_value_t *codeinst = jl_rettype_inferred_native(mi, world, world); if (codeinst == jl_nothing) { - (void)jl_type_infer(mi, world, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi, world, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); codeinst = jl_rettype_inferred_native(mi, world, world); } if (codeinst != jl_nothing) { @@ -3559,10 +3560,10 @@ JL_DLLEXPORT void jl_compile_method_instance(jl_method_instance_t *mi, jl_tuplet miflags = jl_atomic_load_relaxed(&mi2->flags) | JL_MI_FLAGS_MASK_PRECOMPILED; jl_atomic_store_relaxed(&mi2->flags, miflags); if (jl_rettype_inferred_native(mi2, world, world) == jl_nothing) - (void)jl_type_infer(mi2, world, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi2, world, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); if (jl_typeinf_func && jl_atomic_load_relaxed(&mi->def.method->primary_world) <= tworld) { if (jl_rettype_inferred_native(mi2, tworld, tworld) == jl_nothing) - (void)jl_type_infer(mi2, tworld, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi2, tworld, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); } } } diff --git a/src/julia_internal.h b/src/julia_internal.h index 60a3b99c18afc..3a85b523bd3e3 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -693,7 +693,7 @@ JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) void jl_engine_sweep(jl_ptls_t *gc_all_tls_states) JL_NOTSAFEPOINT; int jl_engine_hasreserved(jl_method_instance_t *m, jl_value_t *owner) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t world, uint8_t source_mode); +JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t world, uint8_t source_mode, uint8_t trim_mode); JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 73f4a4b5d4e3d..a653b027a861a 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -403,7 +403,7 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) } JL_UNLOCK(&cfun_lock); // next, try to figure out what the target should look like (outside of the lock since this is very slow) - codeinst = mi ? jl_type_infer(mi, world, SOURCE_MODE_ABI) : nullptr; + codeinst = mi ? jl_type_infer(mi, world, SOURCE_MODE_ABI, jl_options.trim) : nullptr; // relock for the remainder of the function JL_LOCK(&cfun_lock); } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart entirely, since jl_world_counter changed thus jl_get_specialization1 might have changed diff --git a/src/toplevel.c b/src/toplevel.c index 41ce5a1901dbf..4628842619d05 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -757,7 +757,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val size_t world = jl_atomic_load_acquire(&jl_world_counter); ct->world_age = world; if (!has_defs && jl_get_module_infer(m) != 0) { - (void)jl_type_infer(mfunc, world, SOURCE_MODE_ABI); + (void)jl_type_infer(mfunc, world, SOURCE_MODE_ABI, jl_options.trim); } result = jl_invoke(/*func*/NULL, /*args*/NULL, /*nargs*/0, mfunc); ct->world_age = last_age; diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index fc34b34096c26..2a63797b4d13a 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -1,6 +1,10 @@ using Libdl using Zstd_jll # Note this uses the vendored older non-LazyLibrary version of Zstd_jll +# JLL usage at build-time should function as expected +Zstd_jll.__init__() +const build_ver = unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ())) + function print_string(fptr::Ptr{Cvoid}) println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) end @@ -9,10 +13,12 @@ function @main(args::Vector{String})::Cint # Test the basic "Hello, world!" println(Core.stdout, "Julia! Hello, world!") - # Make sure that JLL's are working as expected - println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ()))) + # JLL usage at run-time should function as expected + ver = unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ())) + println(Core.stdout, ver) + @assert ver == build_ver - # Add an indirection via `@cfunction` + # Add an indirection via `@cfunction` / 1-arg ccall cfunc = @cfunction(print_string, Cvoid, (Ptr{Cvoid},)) fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) ccall(cfunc, Cvoid, (Ptr{Cvoid},), fptr) From 309b1b158f59485772d5f5fe0a762f20185cf799 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 27 Jun 2025 11:58:03 -0400 Subject: [PATCH 456/662] codegen: gc wb for atomic FCA stores (#58792) Need to re-load the correct `r` since issetfield skips the intcast, resulting in no gc wb for the FCA. Fix #58760 --- Compiler/test/codegen.jl | 6 ++++++ src/cgutils.cpp | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index ea83f43382582..379d06ede9fef 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1069,3 +1069,9 @@ let io = IOBuffer() str = String(take!(io)) @test !occursin("jtbaa_unionselbyte", str) end + +let io = IOBuffer() + code_llvm(io, (x, y) -> (@atomic x[1] = y; nothing), (AtomicMemory{Pair{Any,Any}}, Pair{Any,Any},), raw=true, optimize=false) + str = String(take!(io)) + @test occursin("julia.write_barrier", str) +end diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 07b304e5256d1..6cdfbc7add6c1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2683,6 +2683,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.CreateStore(r, intcast); r = ctx.builder.CreateLoad(intcast_eltyp, intcast); } + else if (!isboxed && intcast_eltyp) { + assert(issetfield); + // issetfield doesn't use intcast, so need to reload rhs with the correct type + r = emit_unbox(ctx, intcast_eltyp, rhs, jltype); + } if (!isboxed) emit_write_multibarrier(ctx, parent, r, rhs.typ); else From d48a675b8fce71760c69bed61898556424c6be39 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 27 Jun 2025 13:38:13 -0700 Subject: [PATCH 457/662] codegen: relaxed jl_tls_states_t.safepoint load (#58828) Every function with a safepoint causes spurious thread sanitizer warnings without this change. Codegen is unaffected, except when we build with `ThreadSanitizerPass`. --- src/llvm-codegen-shared.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index 1dece818fa998..d5d7ae3d50113 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -197,6 +197,7 @@ static inline llvm::Value *get_current_signal_page_from_ptls(llvm::IRBuilder<> & llvm::Value *psafepoint = builder.CreateConstInBoundsGEP1_32(i8, ptls, nthfield); LoadInst *ptls_load = builder.CreateAlignedLoad( T_ptr, psafepoint, Align(sizeof(void *)), "safepoint"); + ptls_load->setOrdering(AtomicOrdering::Monotonic); tbaa_decorate(tbaa, ptls_load); return ptls_load; } From 77b90b9e621d025a11780ae94c8e0afcaa7c2aee Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 28 Jun 2025 18:46:31 -0400 Subject: [PATCH 458/662] bpart: Properly track methods with invalidated source after require_world (#58830) There are three categories of methods we need to worry about during staticdata validation: 1. New methods added to existing generic functions 2. New methods added to new generic functions 3. Existing methods that now have new CodeInstances In each of these cases, we need to check whether any of the implicit binding edges from the method's source was invalidated. Currently, we handle this for 1 and 2 by explicitly scanning the method on load. However, we were not tracking it for case 3. Fix that by using an extra bit in did_scan_method that gets set when we see an existing method getting invalidated, so we know that we need to drop the corresponding CodeInstances during load. Fixes #58346 --- base/invalidation.jl | 16 +++++++++------- base/runtime_internals.jl | 2 ++ base/staticdata.jl | 32 +++++++++++++++++--------------- src/jltypes.c | 2 +- src/julia.h | 1 + test/core.jl | 2 +- test/precompile.jl | 5 ++--- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 8057c16bd9fa9..0d814b3f31f67 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -69,6 +69,9 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid src = _uncompressed_ir(method) invalidate_all = should_invalidate_code_for_globalref(gr, src) end + if invalidate_all && !Base.generating_output() + @atomic method.did_scan_source |= 0x4 + end invalidated_any = false for mi in specializations(method) isdefined(mi, :cache) || continue @@ -182,7 +185,7 @@ function binding_was_invalidated(b::Core.Binding) b.partitions.min_world > unsafe_load(cglobal(:jl_require_world, UInt)) end -function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method::Method, image_backedges_only::Bool) +function scan_new_method!(method::Method, image_backedges_only::Bool) isdefined(method, :source) || return if image_backedges_only && !has_image_globalref(method) return @@ -195,21 +198,20 @@ function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method # TODO: We could turn this into an addition if condition. For now, use it as a reasonably cheap # additional consistency check @assert !image_backedges_only - push!(methods_with_invalidated_source, method) + @atomic method.did_scan_source |= 0x4 end maybe_add_binding_backedge!(b, method) end + @atomic method.did_scan_source |= 0x1 end -function scan_new_methods(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) - methods_with_invalidated_source = IdSet{Method}() +function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) for method in internal_methods if isa(method, Method) - scan_new_method!(methods_with_invalidated_source, method, image_backedges_only) + scan_new_method!(method, image_backedges_only) end end for tme::Core.TypeMapEntry in extext_methods - scan_new_method!(methods_with_invalidated_source, tme.func::Method, image_backedges_only) + scan_new_method!(tme.func::Method, image_backedges_only) end - return methods_with_invalidated_source end diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 4347c67438577..98dd111ccbf68 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1414,6 +1414,8 @@ Returns the world the [current_task()](@ref) is executing within. """ tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) +get_require_world() = unsafe_load(cglobal(:jl_require_world, UInt)) + """ propertynames(x, private=false) diff --git a/base/staticdata.jl b/base/staticdata.jl index 04fb6f0cfa263..f24dfd361a1e0 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -17,20 +17,20 @@ function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{ # determine which CodeInstance objects are still valid in our image # to enable any applicable new codes backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) - methods_with_invalidated_source = Base.scan_new_methods(extext_methods, internal_methods, backedges_only) + Base.scan_new_methods!(extext_methods, internal_methods, backedges_only) stack = CodeInstance[] visiting = IdDict{CodeInstance,Int}() - _insert_backedges(edges, stack, visiting, methods_with_invalidated_source) + _insert_backedges(edges, stack, visiting) if ext_ci_list !== nothing - _insert_backedges(ext_ci_list, stack, visiting, methods_with_invalidated_source, #=external=#true) + _insert_backedges(ext_ci_list, stack, visiting, #=external=#true) end end -function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, external::Bool=false) +function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, external::Bool=false) for i = 1:length(edges) codeinst = edges[i]::CodeInstance validation_world = get_world_counter() - verify_method_graph(codeinst, stack, visiting, mwis, validation_world) + verify_method_graph(codeinst, stack, visiting, validation_world) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @@ -54,16 +54,14 @@ function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visi end end -function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) +function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, validation_world::UInt) @assert isempty(stack); @assert isempty(visiting); - child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, mwis, validation_world) + child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, validation_world) @assert child_cycle == 0 @assert isempty(stack); @assert isempty(visiting); nothing end -get_require_world() = unsafe_load(cglobal(:jl_require_world, UInt)) - function gen_staged_sig(def::Method, mi::MethodInstance) isdefined(def, :generator) || return nothing isdispatchtuple(mi.specTypes) || return nothing @@ -113,7 +111,7 @@ end # - Visit the entire call graph, starting from edges[idx] to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable # and slightly modified with an early termination option once the computation reaches its minimum -function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) +function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, validation_world::UInt) world = codeinst.min_world let max_valid2 = codeinst.max_world if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL @@ -127,13 +125,13 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end # Implicitly referenced bindings in the current module do not get explicit edges. - # If they were invalidated, they'll be in `mwis`. If they weren't, they imply a minworld + # If they were invalidated, they'll have the flag set in did_scan_source. If they weren't, they imply a minworld # of `get_require_world`. In principle, this is only required for methods that do reference # an implicit globalref. However, we already don't perform this validation for methods that # don't have any (implicit or explicit) edges at all. The remaining corner case (some explicit, # but no implicit edges) is rare and there would be little benefit to lower the minworld for it # in any case, so we just always use `get_require_world` here. - local minworld::UInt, maxworld::UInt = get_require_world(), validation_world + local minworld::UInt, maxworld::UInt = Base.get_require_world(), validation_world if haskey(visiting, codeinst) return visiting[codeinst], minworld, maxworld end @@ -143,7 +141,11 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi # TODO JL_TIMING(VERIFY_IMAGE, VERIFY_Methods) callees = codeinst.edges # Check for invalidation of the implicit edges from GlobalRef in the Method source - if def in mwis + if (def.did_scan_source & 0x1) == 0x0 + backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) + Base.scan_new_method!(def, backedges_only) + end + if (def.did_scan_source & 0x4) != 0x0 maxworld = 0 invalidations = _jl_debug_method_invalidation[] if invalidations !== nothing @@ -153,7 +155,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi # verify current edges if isempty(callees) # quick return: no edges to verify (though we probably shouldn't have gotten here from WORLD_AGE_REVALIDATION_SENTINEL) - elseif maxworld == get_require_world() + elseif maxworld == Base.get_require_world() # if no new worlds were allocated since serializing the base module, then no new validation is worth doing right now either else j = 1 @@ -231,7 +233,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end callee = edge local min_valid2::UInt, max_valid2::UInt - child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, mwis, validation_world) + child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, validation_world) if minworld < min_valid2 minworld = min_valid2 end diff --git a/src/jltypes.c b/src/jltypes.c index 9b664ce25861c..ec7aad023164e 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3607,7 +3607,7 @@ void jl_init_types(void) JL_GC_DISABLED 0, 1, 10); //const static uint32_t method_constfields[1] = { 0b0 }; // (1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<6)|(1<<9)|(1<<10)|(1<<17)|(1<<21)|(1<<22)|(1<<23)|(1<<24)|(1<<25)|(1<<26)|(1<<27)|(1<<28)|(1<<29)|(1<<30); //jl_method_type->name->constfields = method_constfields; - const static uint32_t method_atomicfields[1] = { 0x00000030 }; // (1<<4)|(1<<5) + const static uint32_t method_atomicfields[1] = { 0x10000030 }; // (1<<4)|(1<<5)||(1<<28) jl_method_type->name->atomicfields = method_atomicfields; jl_method_instance_type = diff --git a/src/julia.h b/src/julia.h index 8018928b8a154..980489c0b53fb 100644 --- a/src/julia.h +++ b/src/julia.h @@ -368,6 +368,7 @@ typedef struct _jl_method_t { uint8_t nospecializeinfer; // bit flags, 0x01 = scanned // 0x02 = added to module scanned list (either from scanning or inference edge) + // 0x04 = Source was invalidated since jl_require_world _Atomic(uint8_t) did_scan_source; // uint8 settings diff --git a/test/core.jl b/test/core.jl index 997677cd69e39..46503ae678e98 100644 --- a/test/core.jl +++ b/test/core.jl @@ -36,7 +36,7 @@ end for (T, c) in ( (Core.CodeInfo, []), (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), - (Core.Method, [:primary_world, :dispatch_status]), + (Core.Method, [:primary_world, :did_scan_source, :dispatch_status]), (Core.MethodInstance, [:cache, :flags]), (Core.MethodTable, [:defs]), (Core.MethodCache, [:leafcache, :cache, :var""]), diff --git a/test/precompile.jl b/test/precompile.jl index 7cf9bdffadfc1..afb3dcc154853 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1083,9 +1083,8 @@ precompile_test_harness("code caching") do dir @test hasvalid(mi, world) # was compiled with the new method m = only(methods(MA.fib)) mi = m.specializations::Core.MethodInstance - @test isdefined(mi, :cache) # it was precompiled by StaleB - @test_broken !hasvalid(mi, world) # invalidated by redefining `gib` before loading StaleB - @test_broken MA.fib() === 2.0 + @test !hasvalid(mi, world) # invalidated by redefining `gib` before loading StaleB + @test MA.fib() === 2.0 # Reporting test (ensure SnoopCompile works) @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) From c985dba510d4c602866542f535ab500ad81c2305 Mon Sep 17 00:00:00 2001 From: gitboy16 <82724369+gitboy16@users.noreply.github.com> Date: Sun, 29 Jun 2025 19:09:47 +0100 Subject: [PATCH 459/662] Limit --help and --help-hidden to 100 character line length (#58835) Just fixing the command line description to make sure it is not more than 100 characters wide as discussed with @oscardssmith in PR #54066 and PR #53759. I also added a test to make sure that nothing more than 100 characters is inserted. Thank you. --- src/jloptions.c | 43 +++++++++++++++++++++++-------------------- test/cmdlineargs.jl | 4 ++++ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/jloptions.c b/src/jloptions.c index 0f0c6060bfc79..7c12fd07bf5ac 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -179,8 +179,9 @@ static const char opts[] = " Or, create a temporary environment with `@temp`\n" " The default @. option will search through parent\n" " directories until a Project.toml or JuliaProject.toml\n" - " file is found. @script is similar, but searches up from\n" - " the programfile or a path relative to programfile.\n" + " file is found. @script is similar, but searches up\n" + " from the programfile or a path relative to\n" + " programfile.\n" " -J, --sysimage Start up with the given system image file\n" " -H, --home Set location of `julia` executable\n" " --startup-file={yes*|no} Load `JULIA_DEPOT_PATH/config/startup.jl`; \n" @@ -291,9 +292,9 @@ static const char opts[] = " information, see --bug-report=help.\n\n" " --heap-size-hint=[] Forces garbage collection if memory usage is higher\n" " than the given value. The value may be specified as a\n" - " number of bytes, optionally in units of: B, K (kibibytes),\n" - " M (mebibytes), G (gibibytes), T (tebibytes), or % (percentage\n" - " of physical memory).\n\n" + " number of bytes, optionally in units of: B,\n" + " K (kibibytes), M (mebibytes), G (gibibytes),\n" + " T (tebibytes), or % (percentage of physical memory).\n\n" ; static const char opts_hidden[] = @@ -319,24 +320,26 @@ static const char opts_hidden[] = " --output-asm Generate an assembly file (.s)\n" " --output-incremental={yes|no*} Generate an incremental output file (rather than\n" " complete)\n" - " --timeout-for-safepoint-straggler If this value is set, then we will dump the backtrace for a thread\n" - " that fails to reach a safepoint within the specified time\n" + " --timeout-for-safepoint-straggler If this value is set, then we will dump the backtrace\n" + " for a thread that fails to reach a safepoint within\n" + " the specified time\n" " --trace-compile={stderr|name} Print precompile statements for methods compiled\n" - " during execution or save to stderr or a path. Methods that\n" - " were recompiled are printed in yellow or with a trailing\n" - " comment if color is not supported\n" - " --trace-compile-timing If --trace-compile is enabled show how long each took to\n" - " compile in ms\n" + " during execution or save to stderr or a path. Methods\n" + " that were recompiled are printed in yellow or with\n" + " a trailing comment if color is not supported\n" + " --trace-compile-timing If --trace-compile is enabled show how long each took\n" + " to compile in ms\n" " --task-metrics={yes|no*} Enable collection of per-task timing data.\n" " --image-codegen Force generate code in imaging mode\n" - " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n" - " --trim={no*|safe|unsafe|unsafe-warn}\n" - " Build a sysimage including only code provably reachable\n" - " from methods marked by calling `entrypoint`. In unsafe\n" - " mode, the resulting binary might be missing needed code\n" - " and can throw errors. With unsafe-warn warnings will be\n" - " printed for dynamic call sites that might lead to such\n" - " errors. In safe mode compile-time errors are given instead.\n" + " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n\n" + + " --trim={no*|safe|unsafe|unsafe-warn} Build a sysimage including only code provably\n" + " reachable from methods marked by calling\n" + " `entrypoint`. In unsafe mode, the resulting binary\n" + " might be missing needed code and can throw errors.\n" + " With unsafe-warn warnings will be printed for\n" + " dynamic call sites that might lead to such errors.\n" + " In safe mode compile-time errors are given instead.\n" " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever\n" " go above this limit, we will abort. The value\n" " may be specified as a number of bytes,\n" diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 5a131eca3001f..4b0c8b6b59a8f 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -240,6 +240,10 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test startswith(read(`$exename --help`, String), header) end + # Test to make sure that command line --help and --help-hidden do not return a description which is more than 100 characters wide + @test isempty(filter(x->length(x) > 100, readlines(`$exename -h`))) + @test isempty(filter(x->length(x) > 100, readlines(`$exename --help-hidden`))) + # ~ expansion in --project and JULIA_PROJECT if !Sys.iswindows() let expanded = abspath(expanduser("~/foo/Project.toml")) From 8caacdb69e3a9d649016d95bd032a8bd17ad4992 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 30 Jun 2025 08:09:32 -0400 Subject: [PATCH 460/662] libuv: Mark `(un)preserve_handle` as `@nospecialize` (#58844) These functions only worry about object identity, so there's no need for them to specialize them on their type. --- base/libuv.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/libuv.jl b/base/libuv.jl index 35b1a9097293e..8f066971a639d 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -39,7 +39,7 @@ macro handle_as(hand, typ) end end -associate_julia_struct(handle::Ptr{Cvoid}, @nospecialize(jlobj)) = +@nospecializeinfer associate_julia_struct(handle::Ptr{Cvoid}, @nospecialize(jlobj)) = ccall(:jl_uv_associate_julia_struct, Cvoid, (Ptr{Cvoid}, Any), handle, jlobj) disassociate_julia_struct(uv) = disassociate_julia_struct(uv.handle) disassociate_julia_struct(handle::Ptr{Cvoid}) = @@ -52,14 +52,14 @@ iolock_end() = ccall(:jl_iolock_end, Cvoid, ()) # and should thus not be garbage collected const uvhandles = IdDict() const preserve_handle_lock = Threads.SpinLock() -function preserve_handle(x) +@nospecializeinfer function preserve_handle(@nospecialize(x)) lock(preserve_handle_lock) v = get(uvhandles, x, 0)::Int uvhandles[x] = v + 1 unlock(preserve_handle_lock) nothing end -function unpreserve_handle(x) +@nospecializeinfer function unpreserve_handle(@nospecialize(x)) lock(preserve_handle_lock) v = get(uvhandles, x, 0)::Int if v == 0 From b459d8885cabeb4c16089933b76c0d4c1605236f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 30 Jun 2025 15:00:38 -0400 Subject: [PATCH 461/662] add METHOD_SIG_LATEST_ONLY optimization to MethodInstance too (#58825) Add the same optimization from Method to MethodInstance, although the performance gain seems to be negligible in my specific testing, there doesn't seem any likely downside to adding one caching bit to avoid some recomputations. --- base/essentials.jl | 2 +- base/staticdata.jl | 19 ++++- doc/src/devdocs/locks.md | 4 + src/gf.c | 163 +++++++++++++++++++++++++++++++-------- src/jltypes.c | 14 ++-- src/julia.h | 3 +- src/method.c | 1 + src/staticdata.c | 46 +---------- test/core.jl | 2 +- 9 files changed, 165 insertions(+), 89 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index e1bc648e7dbd0..dd5e398c7e93e 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -731,7 +731,7 @@ Neither `convert` nor `cconvert` should take a Julia object and turn it into a ` """ function cconvert end -cconvert(T::Type, x) = x isa T ? x : convert(T, x) # do the conversion eagerly in most cases +cconvert(::Type{T}, x) where {T} = x isa T ? x : convert(T, x) # do the conversion eagerly in most cases cconvert(::Type{Union{}}, x...) = convert(Union{}, x...) cconvert(::Type{<:Ptr}, x) = x # but defer the conversion to Ptr to unsafe_convert unsafe_convert(::Type{T}, x::T) where {T} = x # unsafe_convert (like convert) defaults to assuming the convert occurred diff --git a/base/staticdata.jl b/base/staticdata.jl index f24dfd361a1e0..b4ff41133d893 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -281,6 +281,7 @@ end function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt) # verify that these edges intersect with the same methods as before + mi = nothing if n == 1 # first, fast-path a check if the expected method simply dominates its sig anyways # so the result of ml_matches is already simply known @@ -289,11 +290,18 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n meth = t else if t isa CodeInstance - t = get_ci_mi(t) + mi = get_ci_mi(t)::MethodInstance else - t = t::MethodInstance + mi = t::MethodInstance + end + meth = mi.def::Method + if !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + result = Any[] # result is unused + return minworld, maxworld, result end - meth = t.def::Method end if !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) minworld = meth.primary_world @@ -327,7 +335,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n meth = t else if t isa CodeInstance - t = get_ci_mi(t) + t = get_ci_mi(t)::MethodInstance else t = t::MethodInstance end @@ -354,6 +362,9 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n resize!(result, ins) end end + if maxworld[] == typemax(UInt) && mi isa MethodInstance + ccall(:jl_promote_mi_to_current, Cvoid, (Any, UInt, UInt), mi, minworld[], world) + end return minworld[], maxworld[], result end diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index 8d6672842c3c8..c89499a4162a9 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -85,6 +85,10 @@ may result in pernicious and hard-to-find deadlocks. BE VERY CAREFUL! > > * Libdl.LazyLibrary lock +The following is a level 7 lock, which can only be acquired when not holding any other locks: + +> * world_counter_lock + The following is the root lock, meaning no other lock shall be held when trying to acquire it: diff --git a/src/gf.c b/src/gf.c index 75398cbb39b37..50a2e32718163 100644 --- a/src/gf.c +++ b/src/gf.c @@ -308,7 +308,7 @@ jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args m->isva = 1; m->nargs = 2; jl_atomic_store_relaxed(&m->primary_world, 1); - jl_atomic_store_relaxed(&m->dispatch_status, METHOD_SIG_LATEST_ONLY | METHOD_SIG_LATEST_ONLY); + jl_atomic_store_relaxed(&m->dispatch_status, METHOD_SIG_LATEST_ONLY | METHOD_SIG_LATEST_WHICH); m->sig = (jl_value_t*)jl_anytuple_type; m->slot_syms = jl_an_empty_string; m->nospecialize = 0; @@ -1506,22 +1506,26 @@ jl_method_instance_t *cache_method( size_t world, size_t min_valid, size_t max_valid, jl_svec_t *sparams) { - // caller must hold the parent->writelock + // caller must hold the parent->writelock, which this releases // short-circuit (now that we hold the lock) if this entry is already present int8_t offs = mc ? jl_cachearg_offset() : 1; { // scope block if (mc) { jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); - if (entry) + if (entry) { + if (mc) JL_UNLOCK(&mc->writelock); return entry->func.linfo; + } } struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; jl_typemap_t *cacheentry = jl_atomic_load_relaxed(cache); assert(cacheentry != NULL); jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(cacheentry, &search, offs, /*subtype*/1); - if (entry && entry->func.value) + if (entry && entry->func.value) { + if (mc) JL_UNLOCK(&mc->writelock); return entry->func.linfo; + } } jl_method_instance_t *newmeth = NULL; @@ -1534,6 +1538,7 @@ jl_method_instance_t *cache_method( JL_GC_PUSH1(&newentry); jl_typemap_insert(cache, parent, newentry, offs); JL_GC_POP(); + if (mc) JL_UNLOCK(&mc->writelock); return newmeth; } } @@ -1578,8 +1583,11 @@ jl_method_instance_t *cache_method( if (newmeth->cache_with_orig) cache_with_orig = 1; + // Capture world counter at start to detect races + size_t current_world = mc ? jl_atomic_load_acquire(&jl_world_counter) : ~(size_t)0; + jl_tupletype_t *cachett = tt; - jl_svec_t* guardsigs = jl_emptysvec; + jl_svec_t *guardsigs = jl_emptysvec; if (!cache_with_orig && mt) { // now examine what will happen if we chose to use this sig in the cache size_t min_valid2 = 1; @@ -1649,6 +1657,10 @@ jl_method_instance_t *cache_method( } } + int unconstrained_max = max_valid == ~(size_t)0; + if (max_valid > current_world) + max_valid = current_world; + // now scan `cachett` and ensure that `Type{T}` in the cache will be matched exactly by `typeof(T)` // and also reduce the complexity of rejecting this entry in the cache // by replacing non-simple types with jl_any_type to build a new `type` @@ -1727,12 +1739,91 @@ jl_method_instance_t *cache_method( } } } + if (mc) { + JL_UNLOCK(&mc->writelock); + + // Only set METHOD_SIG_LATEST_ONLY on method instance if method does NOT have the bit, no guards required, and min_valid == primary_world + int should_set_dispatch_status = !(jl_atomic_load_relaxed(&definition->dispatch_status) & METHOD_SIG_LATEST_ONLY) && + (!cache_with_orig && jl_svec_len(guardsigs) == 0) && + min_valid == jl_atomic_load_relaxed(&definition->primary_world) && + !(jl_atomic_load_relaxed(&newmeth->dispatch_status) & METHOD_SIG_LATEST_ONLY); + + // Combined trylock for both dispatch_status setting and max_world restoration + if ((should_set_dispatch_status || unconstrained_max) && + jl_atomic_load_relaxed(&jl_world_counter) == current_world) { + JL_LOCK(&world_counter_lock); + if (jl_atomic_load_relaxed(&jl_world_counter) == current_world) { + if (should_set_dispatch_status) { + jl_atomic_store_relaxed(&newmeth->dispatch_status, METHOD_SIG_LATEST_ONLY); + } + if (unconstrained_max) { + jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); + } + } + JL_UNLOCK(&world_counter_lock); + } + } JL_GC_POP(); return newmeth; } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, size_t *min_valid, size_t *max_valid); +static void _jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) JL_NOTSAFEPOINT +{ + if (jl_atomic_load_relaxed(&ci->max_world) != validated_world) + return; + jl_atomic_store_relaxed(&ci->max_world, ~(size_t)0); + jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); + for (size_t i = 0; i < jl_svec_len(edges); i++) { + jl_value_t *edge = jl_svecref(edges, i); + if (!jl_is_code_instance(edge)) + continue; + _jl_promote_ci_to_current((jl_code_instance_t *)edge, validated_world); + } +} + +JL_DLLEXPORT void jl_promote_cis_to_current(jl_code_instance_t **cis, size_t n, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + for (size_t i = 0; i < n; i++) { + _jl_promote_ci_to_current(cis[i], validated_world); + } + } + JL_UNLOCK(&world_counter_lock); +} + +JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) +{ + jl_promote_cis_to_current(&ci, 1, validated_world); +} + +JL_DLLEXPORT void jl_promote_mi_to_current(jl_method_instance_t *mi, size_t min_world, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + // Only set METHOD_SIG_LATEST_ONLY on method instance if method does NOT have the bit and min_valid == primary_world + jl_method_t *definition = mi->def.method; + if ((jl_atomic_load_relaxed(&definition->dispatch_status) & METHOD_SIG_LATEST_ONLY) || + min_world != jl_atomic_load_relaxed(&definition->primary_world) || + (jl_atomic_load_relaxed(&mi->dispatch_status) & METHOD_SIG_LATEST_ONLY)) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + jl_atomic_store_relaxed(&mi->dispatch_status, METHOD_SIG_LATEST_ONLY); + } + JL_UNLOCK(&world_counter_lock); +} + +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, int cache, size_t *min_valid, size_t *max_valid); JL_DLLEXPORT jl_typemap_entry_t *jl_mt_find_cache_entry(jl_methcache_t *mc JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED JL_ROOTS_TEMPORARILY, size_t world) { // exported only for debugging purposes, not for casual use @@ -1766,11 +1857,13 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methcache_t *mc JL_PROPAGATE if (!mi) { size_t min_valid = 0; size_t max_valid = ~(size_t)0; - matc = _gf_invoke_lookup((jl_value_t*)tt, jl_method_table, world, &min_valid, &max_valid); + matc = _gf_invoke_lookup((jl_value_t*)tt, jl_method_table, world, 0, &min_valid, &max_valid); if (matc) { jl_method_t *m = matc->method; jl_svec_t *env = matc->sparams; mi = cache_method(jl_method_table, mc, &mc->cache, (jl_value_t*)mc, tt, m, world, min_valid, max_valid, env); + JL_GC_POP(); + return mi; } } JL_UNLOCK(&mc->writelock); @@ -2127,6 +2220,7 @@ static int _invalidate_dispatch_backedges(jl_method_instance_t *mi, jl_value_t * // invalidate cached methods that overlap this definition static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, const char *why) { + // Reset dispatch_status when method instance is replaced JL_LOCK(&replaced_mi->def.method->writelock); _invalidate_backedges(replaced_mi, NULL, max_world, 1); JL_UNLOCK(&replaced_mi->def.method->writelock); @@ -2137,6 +2231,7 @@ static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_w jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); JL_GC_POP(); } + jl_atomic_store_relaxed(&replaced_mi->dispatch_status, 0); } // add a backedge from callee to caller @@ -2634,6 +2729,8 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); // but ignore invoke-type edges int invalidatedmi = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); + if (replaced_dispatch) + jl_atomic_store_relaxed(&mi->dispatch_status, 0); jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); if (_jl_debug_method_invalidation && invalidatedmi) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); @@ -3405,7 +3502,6 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc assert(mc); JL_LOCK(&mc->writelock); mi = cache_method(jl_method_get_table(m), mc, &mc->cache, (jl_value_t*)mc, ti, m, world, min_valid, max_valid, env); - JL_UNLOCK(&mc->writelock); } else { jl_value_t *tt = jl_normalize_to_compilable_sig(ti, env, m, 1); @@ -3893,7 +3989,7 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t *F, jl_value_t **args, uint return _jl_invoke(F, args, nargs, mfunc, world); } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, size_t *min_valid, size_t *max_valid) +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, int cache_result, size_t *min_valid, size_t *max_valid) { jl_value_t *unw = jl_unwrap_unionall((jl_value_t*)types); if (!jl_is_tuple_type(unw)) @@ -3901,7 +3997,7 @@ static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT if (jl_tparam0(unw) == jl_bottom_type) return NULL; jl_methcache_t *mc = ((jl_methtable_t*)mt)->cache; - jl_value_t *matches = ml_matches((jl_methtable_t*)mt, mc, (jl_tupletype_t*)types, 1, 0, 0, world, 1, min_valid, max_valid, NULL); + jl_value_t *matches = ml_matches((jl_methtable_t*)mt, mc, (jl_tupletype_t*)types, 1, 0, 0, world, cache_result, min_valid, max_valid, NULL); if (matches == jl_nothing || jl_array_nrows(matches) != 1) return NULL; jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); @@ -3915,7 +4011,7 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types, jl_value_t *mt, size_t max_valid = ~(size_t)0; if (mt == jl_nothing) mt = (jl_value_t*)jl_method_table; - jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, &min_valid, &max_valid); + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, 1, &min_valid, &max_valid); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc->method; @@ -3926,7 +4022,7 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup_worlds(jl_value_t *types, jl_value_ { if (mt == jl_nothing) mt = (jl_value_t*)jl_method_table; - jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, min_world, max_world); + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, 1, min_world, max_world); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc; @@ -3988,7 +4084,6 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value int sub = jl_subtype_matching((jl_value_t*)tt, (jl_value_t*)method->sig, &tpenv); assert(sub); (void)sub; } - mfunc = cache_method(NULL, NULL, &method->invokes, (jl_value_t*)method, tt, method, 1, 1, ~(size_t)0, tpenv); } JL_UNLOCK(&method->writelock); @@ -4487,26 +4582,29 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, if (entry && (((jl_datatype_t*)unw)->isdispatchtuple || entry->guardsigs == jl_emptysvec)) { jl_method_instance_t *mi = entry->func.linfo; jl_method_t *meth = mi->def.method; - if (!jl_is_unionall(meth->sig) && ((jl_datatype_t*)unw)->isdispatchtuple) { - env.match.env = jl_emptysvec; - env.match.ti = unw; - } - else { - // this just calls jl_subtype_env (since we know that `type <: meth->sig` by transitivity) - env.match.ti = jl_type_intersection_env((jl_value_t*)type, (jl_value_t*)meth->sig, &env.match.env); - } - env.matc = make_method_match((jl_tupletype_t*)env.match.ti, - env.match.env, meth, FULLY_COVERS); - env.t = (jl_value_t*)jl_alloc_vec_any(1); - jl_array_ptr_set(env.t, 0, env.matc); size_t min_world = jl_atomic_load_relaxed(&entry->min_world); - size_t max_world = jl_atomic_load_relaxed(&entry->max_world); - if (*min_valid < min_world) - *min_valid = min_world; - if (*max_valid > max_world) - *max_valid = max_world; - JL_GC_POP(); - return env.t; + // only return this if it appears min_would is fully computed, otherwise do the full lookup to compute min_world exactly + if (min_world == jl_atomic_load_relaxed(&meth->primary_world)) { + size_t max_world = jl_atomic_load_relaxed(&entry->max_world); + if (!jl_is_unionall(meth->sig) && ((jl_datatype_t*)unw)->isdispatchtuple) { + env.match.env = jl_emptysvec; + env.match.ti = unw; + } + else { + // this just calls jl_subtype_env (since we know that `type <: meth->sig` by transitivity) + env.match.ti = jl_type_intersection_env((jl_value_t*)type, (jl_value_t*)meth->sig, &env.match.env); + } + env.matc = make_method_match((jl_tupletype_t*)env.match.ti, + env.match.env, meth, FULLY_COVERS); + env.t = (jl_value_t*)jl_alloc_vec_any(1); + jl_array_ptr_set(env.t, 0, env.matc); + if (*min_valid < min_world) + *min_valid = min_world; + if (*max_valid > max_world) + *max_valid = max_world; + JL_GC_POP(); + return env.t; + } } } } @@ -4771,7 +4869,6 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, jl_svec_t *tpenv = env.matc->sparams; JL_LOCK(&mc->writelock); cache_method(mt, mc, &mc->cache, (jl_value_t*)mc, (jl_tupletype_t*)unw, meth, world, env.match.min_valid, env.match.max_valid, tpenv); - JL_UNLOCK(&mc->writelock); } } *min_valid = env.match.min_valid; diff --git a/src/jltypes.c b/src/jltypes.c index ec7aad023164e..dac68506cb0d4 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3575,7 +3575,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_module_type, jl_symbol_type, jl_int32_type, - jl_int32_type, + jl_uint8_type, jl_ulong_type, jl_type_type, jl_any_type, // union(jl_simplevector_type, jl_method_instance_type), @@ -3613,27 +3613,29 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_instance_type = jl_new_datatype(jl_symbol("MethodInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(7, + jl_perm_symsvec(8, "def", "specTypes", "sparam_vals", "backedges", "cache", "cache_with_orig", - "flags"), - jl_svec(7, + "flags", + "dispatch_status"), + jl_svec(8, jl_new_struct(jl_uniontype_type, jl_method_type, jl_module_type), jl_any_type, jl_simplevector_type, jl_array_any_type, jl_any_type/*jl_code_instance_type*/, jl_bool_type, - jl_bool_type), + jl_bool_type, + jl_uint8_type), jl_emptysvec, 0, 1, 3); // These fields should be constant, but Serialization wants to mutate them in initialization //const static uint32_t method_instance_constfields[1] = { 0b0000111 }; // fields 1, 2, 3 - const static uint32_t method_instance_atomicfields[1] = { 0b1010000 }; // fields 5, 7 + const static uint32_t method_instance_atomicfields[1] = { 0b11010000 }; // fields 5, 7, 8 //Fields 4 and 5 must be protected by method->write_lock, and thus all operations on jl_method_instance_t are threadsafe. //jl_method_instance_type->name->constfields = method_instance_constfields; jl_method_instance_type->name->atomicfields = method_instance_atomicfields; diff --git a/src/julia.h b/src/julia.h index 980489c0b53fb..19c8408f22b4a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -323,7 +323,7 @@ typedef struct _jl_method_t { struct _jl_module_t *module; jl_sym_t *file; int32_t line; - _Atomic(int32_t) dispatch_status; // bits defined in staticdata.jl + _Atomic(uint8_t) dispatch_status; // bits defined in staticdata.jl _Atomic(size_t) primary_world; // method's type signature. redundant with TypeMapEntry->specTypes @@ -408,6 +408,7 @@ struct _jl_method_instance_t { // bit 2: The ->backedges field is currently being walked higher up the stack - entries may be deleted, but not moved // bit 3: The ->backedges field was modified and should be compacted when clearing bit 2 _Atomic(uint8_t) flags; + _Atomic(uint8_t) dispatch_status; // bits defined in staticdata.jl }; #define JL_MI_FLAGS_MASK_PRECOMPILED 0x01 #define JL_MI_FLAGS_MASK_DISPATCHED 0x02 diff --git a/src/method.c b/src/method.c index eba485ff9eb69..a04035086df90 100644 --- a/src/method.c +++ b/src/method.c @@ -609,6 +609,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void) jl_atomic_store_relaxed(&mi->cache, NULL); mi->cache_with_orig = 0; jl_atomic_store_relaxed(&mi->flags, 0); + jl_atomic_store_relaxed(&mi->dispatch_status, 0); return mi; } diff --git a/src/staticdata.c b/src/staticdata.c index 5c4bf99883eb3..5d61b35c90122 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1853,6 +1853,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(f == s->s); jl_method_instance_t *newmi = (jl_method_instance_t*)&f->buf[reloc_offset]; jl_atomic_store_relaxed(&newmi->flags, 0); + if (s->incremental) { + jl_atomic_store_relaxed(&newmi->dispatch_status, 0); + } } else if (jl_is_code_instance(v)) { assert(f == s->s); @@ -4547,49 +4550,6 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j return mod; } -JL_DLLEXPORT void _jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) JL_NOTSAFEPOINT -{ - if (jl_atomic_load_relaxed(&ci->max_world) != validated_world) - return; - jl_atomic_store_relaxed(&ci->max_world, ~(size_t)0); - jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); - for (size_t i = 0; i < jl_svec_len(edges); i++) { - jl_value_t *edge = jl_svecref(edges, i); - if (!jl_is_code_instance(edge)) - continue; - _jl_promote_ci_to_current((jl_code_instance_t *)edge, validated_world); - } -} - -JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) -{ - size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); - // No need to acquire the lock if we've been invalidated anyway - if (current_world > validated_world) - return; - JL_LOCK(&world_counter_lock); - current_world = jl_atomic_load_relaxed(&jl_world_counter); - if (current_world == validated_world) { - _jl_promote_ci_to_current(ci, validated_world); - } - JL_UNLOCK(&world_counter_lock); -} - -JL_DLLEXPORT void jl_promote_cis_to_current(jl_code_instance_t **cis, size_t n, size_t validated_world) -{ - size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); - // No need to acquire the lock if we've been invalidated anyway - if (current_world > validated_world) - return; - JL_LOCK(&world_counter_lock); - current_world = jl_atomic_load_relaxed(&jl_world_counter); - if (current_world == validated_world) { - for (size_t i = 0; i < n; i++) { - _jl_promote_ci_to_current(cis[i], validated_world); - } - } - JL_UNLOCK(&world_counter_lock); -} #ifdef __cplusplus } diff --git a/test/core.jl b/test/core.jl index 46503ae678e98..a824c0abb674a 100644 --- a/test/core.jl +++ b/test/core.jl @@ -37,7 +37,7 @@ for (T, c) in ( (Core.CodeInfo, []), (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), (Core.Method, [:primary_world, :did_scan_source, :dispatch_status]), - (Core.MethodInstance, [:cache, :flags]), + (Core.MethodInstance, [:cache, :flags, :dispatch_status]), (Core.MethodTable, [:defs]), (Core.MethodCache, [:leafcache, :cache, :var""]), (Core.TypeMapEntry, [:next, :min_world, :max_world]), From 46513e6b6840430de7971faf1011495c78a0fa62 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 30 Jun 2025 11:28:03 -0400 Subject: [PATCH 462/662] Encode fully_covers=false edges using negative of method count This change allows edges that don't fully cover their method matches to be properly tracked through serialization. When fully_covers is false (indicating incomplete method coverage), we encode the method count as negative in the edges array to signal that compactly. --- Compiler/src/stmtinfo.jl | 12 +++++++----- base/staticdata.jl | 13 ++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 8f08748e1bc57..d6a63d8f71abf 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -60,7 +60,7 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo end end nmatches = length(info.results) - if nmatches == length(info.edges) == 1 + if nmatches == length(info.edges) == 1 && fully_covering(info) # try the optimized format for the representation, if possible and applicable # if this doesn't succeed, the backedge will be less precise, # but the forward edge will maintain the precision @@ -78,13 +78,15 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo end end # add check for whether this lookup already existed in the edges list + # encode nmatches as negative if fully_covers is false + encoded_nmatches = fully_covering(info) ? nmatches : -nmatches for i in 1:length(edges) - if edges[i] === nmatches && edges[i+1] == info.atype + if edges[i] === encoded_nmatches && edges[i+1] == info.atype # TODO: must also verify the CodeInstance match too return nothing end end - push!(edges, nmatches, info.atype) + push!(edges, encoded_nmatches, info.atype) for i = 1:nmatches edge = info.edges[i] m = info.results[i] @@ -101,7 +103,7 @@ function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) i = 1 while i <= length(edges) edgeᵢ = edges[i] - edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) + edgeᵢ isa Int && (i += 2 + abs(edgeᵢ); continue) edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge && !(i > 1 && edges[i-1] isa Type) @@ -116,7 +118,7 @@ function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) i = 1 while i <= length(edges) edgeᵢ_orig = edgeᵢ = edges[i] - edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) + edgeᵢ isa Int && (i += 2 + abs(edgeᵢ); continue) edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge.def && !(i > 1 && edges[i-1] isa Type) diff --git a/base/staticdata.jl b/base/staticdata.jl index b4ff41133d893..e874d27e24c2b 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -168,12 +168,15 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end if edge isa MethodInstance sig = edge.specTypes - min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world) + min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world, true) j += 1 elseif edge isa Int sig = callees[j+1] - min_valid2, max_valid2, matches = verify_call(sig, callees, j+2, edge, world) - j += 2 + edge + # Handle negative counts (fully_covers=false) + nmatches = abs(edge) + fully_covers = edge > 0 + min_valid2, max_valid2, matches = verify_call(sig, callees, j+2, nmatches, world, fully_covers) + j += 2 + nmatches edge = sig elseif edge isa Core.Binding j += 1 @@ -279,10 +282,10 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi return 0, minworld, maxworld end -function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt) +function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool) # verify that these edges intersect with the same methods as before mi = nothing - if n == 1 + if n == 1 && fully_covers # first, fast-path a check if the expected method simply dominates its sig anyways # so the result of ml_matches is already simply known let t = expecteds[i], meth, minworld, maxworld, result From d7c70bcbab2cba3f7decb3ffa21cc4ef73ed87cb Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 30 Jun 2025 17:33:10 -0400 Subject: [PATCH 463/662] move trim patches to separate files, only load if trimming (#58826) fixes part of #58458 --- Makefile | 2 +- contrib/juliac-buildscript.jl | 243 +--------------------------------- contrib/juliac-trim-base.jl | 161 ++++++++++++++++++++++ contrib/juliac-trim-stdlib.jl | 83 ++++++++++++ contrib/juliac.jl | 2 + 5 files changed, 253 insertions(+), 238 deletions(-) create mode 100644 contrib/juliac-trim-base.jl create mode 100644 contrib/juliac-trim-stdlib.jl diff --git a/Makefile b/Makefile index c5367a5a429a4..fc73b897c3aa3 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac.jl $(build_datarootdir)/julia/juliac-buildscript.jl +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac.jl $(build_datarootdir)/julia/juliac-buildscript.jl $(build_datarootdir)/julia/juliac-trim-base.jl $(build_datarootdir)/julia/juliac-trim-stdlib.jl @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 1697ed3fd1f03..db1cb75339e4d 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + # Script to run in the process that generates juliac's object file output # Run the verifier in the current world (before modifications), so that error @@ -21,164 +23,8 @@ if Base.get_bool_env("JULIA_USE_FLISP_PARSER", false) === false Base.JuliaSyntax.enable_in_core!() end -# Patch methods in Core and Base - -@eval Core begin - DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) -end - -(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) - -@eval Base begin - depwarn(msg, funcsym; force::Bool=false) = nothing - _assert_tostring(msg) = "" - reinit_stdio() = nothing - JuliaSyntax.enable_in_core!() = nothing - init_active_project() = ACTIVE_PROJECT[] = nothing - set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile - disable_library_threading() = nothing - start_profile_listener() = nothing - invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) - const invokelatest = invokelatest_trimmed - function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} - s = IOBuffer(sizehint=sizehint) - if context isa Tuple - f(IOContext(s, context...), args...) - elseif context !== nothing - f(IOContext(s, context), args...) - else - f(s, args...) - end - String(_unsafe_take!(s)) - end - function show_typeish(io::IO, @nospecialize(T)) - if T isa Type - show(io, T) - elseif T isa TypeVar - print(io, (T::TypeVar).name) - else - print(io, "?") - end - end - function show(io::IO, T::Type) - if T isa DataType - print(io, T.name.name) - if T !== T.name.wrapper && length(T.parameters) > 0 - print(io, "{") - first = true - for p in T.parameters - if !first - print(io, ", ") - end - first = false - if p isa Int - show(io, p) - elseif p isa Type - show(io, p) - elseif p isa Symbol - print(io, ":") - print(io, p) - elseif p isa TypeVar - print(io, p.name) - else - print(io, "?") - end - end - print(io, "}") - end - elseif T isa Union - print(io, "Union{") - show_typeish(io, T.a) - print(io, ", ") - show_typeish(io, T.b) - print(io, "}") - elseif T isa UnionAll - print(io, T.body::Type) - print(io, " where ") - print(io, T.var.name) - end - end - show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) - - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) where {F, F2} = - _mapreduce_dim(f, op, init, A, dims) - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted...; kw...) where {F, F2} = - reduce(op, map(f, A...); kw...) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - mapfoldl_impl(f, op, nt, A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - _mapreduce(f, op, IndexStyle(A), A) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, dims) where {F, F2} = - mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) where {F,F2} = - mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) - - mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F, F2} = - reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) - mapreduce_first(f::F, op::F2, x) where {F,F2} = reduce_first(op, f(x)) - - _mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted) where {F,F2} = _mapreduce(f, op, IndexStyle(A), A) - mapreduce_empty(::typeof(identity), op::F, T) where {F} = reduce_empty(op, T) - mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) - mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) -end -@eval Base.Sys begin - __init_build() = nothing -end -@eval Base.GMP begin - function __init__() - try - ccall((:__gmp_set_memory_functions, libgmp), Cvoid, - (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), - cglobal(:jl_gc_counted_malloc), - cglobal(:jl_gc_counted_realloc_with_old_size), - cglobal(:jl_gc_counted_free_with_size)) - ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL - ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) - catch ex - Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") - end - # This only works with a patched version of GMP, ignore otherwise - try - ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, - (Ptr{Cvoid},), - cglobal(:jl_throw_out_of_memory_error)) - ALLOC_OVERFLOW_FUNCTION[] = true - catch ex - # ErrorException("ccall: could not find function...") - if typeof(ex) != ErrorException - rethrow() - end - end - end -end -@eval Base.Sort begin - issorted(itr; - lt::T=isless, by::F=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) where {T,F} = - issorted(itr, ord(lt,by,rev,order)) -end -@eval Base.TOML begin - function try_return_datetime(p, year, month, day, h, m, s, ms) - return DateTime(year, month, day, h, m, s, ms) - end - function try_return_date(p, year, month, day) - return Date(year, month, day) - end - function parse_local_time(l::Parser) - h = @try parse_int(l, false) - h in 0:23 || return ParserError(ErrParsingDateTime) - _, m, s, ms = @try _parse_local_time(l, true) - # TODO: Could potentially parse greater accuracy for the - # fractional seconds here. - return try_return_time(l, h, m, s, ms) - end - function try_return_time(p, h, m, s, ms) - return Time(h, m, s, ms) - end +if Base.JLOptions().trim != 0 + include(joinpath(@__DIR__, "juliac-trim-base.jl")) end # Load user code @@ -229,85 +75,8 @@ let mod = Base.include(Main, ARGS[1]) end end -# Additional method patches depending on whether user code loads certain stdlibs -let - find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) - - SparseArrays = find_loaded_root_module(Base.PkgId( - Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) - if SparseArrays !== nothing - @eval SparseArrays.CHOLMOD begin - function __init__() - ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) - end - end - end - - Artifacts = find_loaded_root_module(Base.PkgId( - Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) - if Artifacts !== nothing - @eval Artifacts begin - function _artifact_str( - __module__, - artifacts_toml, - name, - path_tail, - artifact_dict, - hash, - platform, - _::Val{LazyArtifacts} - ) where LazyArtifacts - # If the artifact exists, we're in the happy path and we can immediately - # return the path to the artifact: - dirs = artifacts_dirs(bytes2hex(hash.bytes)) - for dir in dirs - if isdir(dir) - return jointail(dir, path_tail) - end - end - error("Artifact not found") - end - end - end - - Pkg = find_loaded_root_module(Base.PkgId( - Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) - if Pkg !== nothing - @eval Pkg begin - __init__() = rand() #TODO, methods that do nothing don't get codegened - end - end - - StyledStrings = find_loaded_root_module(Base.PkgId( - Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) - if StyledStrings !== nothing - @eval StyledStrings begin - __init__() = rand() - end - end - - Markdown = find_loaded_root_module(Base.PkgId( - Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) - if Markdown !== nothing - @eval Markdown begin - __init__() = rand() - end - end - - JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( - Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) - if JuliaSyntaxHighlighting !== nothing - @eval JuliaSyntaxHighlighting begin - __init__() = rand() - end - end +if Base.JLOptions().trim != 0 + include(joinpath(@__DIR__, "juliac-trim-stdlib.jl")) end empty!(Core.ARGS) diff --git a/contrib/juliac-trim-base.jl b/contrib/juliac-trim-base.jl new file mode 100644 index 0000000000000..96fed77969c97 --- /dev/null +++ b/contrib/juliac-trim-base.jl @@ -0,0 +1,161 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Patches to Base needed for trimming + +@eval Core begin + DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) +end + +(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) + +@eval Base begin + depwarn(msg, funcsym; force::Bool=false) = nothing + _assert_tostring(msg) = "" + reinit_stdio() = nothing + JuliaSyntax.enable_in_core!() = nothing + init_active_project() = ACTIVE_PROJECT[] = nothing + set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile + disable_library_threading() = nothing + start_profile_listener() = nothing + invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) + const invokelatest = invokelatest_trimmed + function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} + s = IOBuffer(sizehint=sizehint) + if context isa Tuple + f(IOContext(s, context...), args...) + elseif context !== nothing + f(IOContext(s, context), args...) + else + f(s, args...) + end + String(_unsafe_take!(s)) + end + function show_typeish(io::IO, @nospecialize(T)) + if T isa Type + show(io, T) + elseif T isa TypeVar + print(io, (T::TypeVar).name) + else + print(io, "?") + end + end + function show(io::IO, T::Type) + if T isa DataType + print(io, T.name.name) + if T !== T.name.wrapper && length(T.parameters) > 0 + print(io, "{") + first = true + for p in T.parameters + if !first + print(io, ", ") + end + first = false + if p isa Int + show(io, p) + elseif p isa Type + show(io, p) + elseif p isa Symbol + print(io, ":") + print(io, p) + elseif p isa TypeVar + print(io, p.name) + else + print(io, "?") + end + end + print(io, "}") + end + elseif T isa Union + print(io, "Union{") + show_typeish(io, T.a) + print(io, ", ") + show_typeish(io, T.b) + print(io, "}") + elseif T isa UnionAll + print(io, T.body::Type) + print(io, " where ") + print(io, T.var.name) + end + end + show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) + + mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) where {F, F2} = + _mapreduce_dim(f, op, init, A, dims) + mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted...; kw...) where {F, F2} = + reduce(op, map(f, A...); kw...) + + _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = + mapfoldl_impl(f, op, nt, A) + + _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = + _mapreduce(f, op, IndexStyle(A), A) + + _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, dims) where {F, F2} = + mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) + + _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) where {F,F2} = + mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) + + mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F, F2} = + reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) + mapreduce_first(f::F, op::F2, x) where {F,F2} = reduce_first(op, f(x)) + + _mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted) where {F,F2} = _mapreduce(f, op, IndexStyle(A), A) + mapreduce_empty(::typeof(identity), op::F, T) where {F} = reduce_empty(op, T) + mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) + mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) +end +@eval Base.Sys begin + __init_build() = nothing +end +@eval Base.GMP begin + function __init__() + try + ccall((:__gmp_set_memory_functions, libgmp), Cvoid, + (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), + cglobal(:jl_gc_counted_malloc), + cglobal(:jl_gc_counted_realloc_with_old_size), + cglobal(:jl_gc_counted_free_with_size)) + ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL + ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) + catch ex + Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") + end + # This only works with a patched version of GMP, ignore otherwise + try + ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, + (Ptr{Cvoid},), + cglobal(:jl_throw_out_of_memory_error)) + ALLOC_OVERFLOW_FUNCTION[] = true + catch ex + # ErrorException("ccall: could not find function...") + if typeof(ex) != ErrorException + rethrow() + end + end + end +end +@eval Base.Sort begin + issorted(itr; + lt::T=isless, by::F=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) where {T,F} = + issorted(itr, ord(lt,by,rev,order)) +end +@eval Base.TOML begin + function try_return_datetime(p, year, month, day, h, m, s, ms) + return DateTime(year, month, day, h, m, s, ms) + end + function try_return_date(p, year, month, day) + return Date(year, month, day) + end + function parse_local_time(l::Parser) + h = @try parse_int(l, false) + h in 0:23 || return ParserError(ErrParsingDateTime) + _, m, s, ms = @try _parse_local_time(l, true) + # TODO: Could potentially parse greater accuracy for the + # fractional seconds here. + return try_return_time(l, h, m, s, ms) + end + function try_return_time(p, h, m, s, ms) + return Time(h, m, s, ms) + end +end diff --git a/contrib/juliac-trim-stdlib.jl b/contrib/juliac-trim-stdlib.jl new file mode 100644 index 0000000000000..3e2501c2c3e50 --- /dev/null +++ b/contrib/juliac-trim-stdlib.jl @@ -0,0 +1,83 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Patches to stdlib needed for trimming + +let + find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) + + SparseArrays = find_loaded_root_module(Base.PkgId( + Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) + if SparseArrays !== nothing + @eval SparseArrays.CHOLMOD begin + function __init__() + ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) + end + end + end + + Artifacts = find_loaded_root_module(Base.PkgId( + Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) + if Artifacts !== nothing + @eval Artifacts begin + function _artifact_str( + __module__, + artifacts_toml, + name, + path_tail, + artifact_dict, + hash, + platform, + _::Val{LazyArtifacts} + ) where LazyArtifacts + # If the artifact exists, we're in the happy path and we can immediately + # return the path to the artifact: + dirs = artifacts_dirs(bytes2hex(hash.bytes)) + for dir in dirs + if isdir(dir) + return jointail(dir, path_tail) + end + end + error("Artifact not found") + end + end + end + + Pkg = find_loaded_root_module(Base.PkgId( + Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) + if Pkg !== nothing + @eval Pkg begin + __init__() = rand() #TODO, methods that do nothing don't get codegened + end + end + + StyledStrings = find_loaded_root_module(Base.PkgId( + Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) + if StyledStrings !== nothing + @eval StyledStrings begin + __init__() = rand() + end + end + + Markdown = find_loaded_root_module(Base.PkgId( + Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) + if Markdown !== nothing + @eval Markdown begin + __init__() = rand() + end + end + + JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( + Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) + if JuliaSyntaxHighlighting !== nothing + @eval JuliaSyntaxHighlighting begin + __init__() = rand() + end + end +end diff --git a/contrib/juliac.jl b/contrib/juliac.jl index b110f1d233690..d637f1f1bf3a2 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + # Julia compiler wrapper script # NOTE: The interface and location of this script are considered unstable/experimental From 1230853d0c68933e22d01993c0263174ad4f0406 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 30 Jun 2025 12:38:14 -0400 Subject: [PATCH 464/662] gf: Add METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC dispatch status bit This commit introduces a new dispatch status bit to track when a method has other methods that are not more specific than it, enabling better optimization decisions during method dispatch. Key changes: 1. Add METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC bit to track methods with non-morespecific intersections 2. Add corresponding METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC bit for precompiled methods 3. Refactor method insertion logic: - Remove morespec_unknown enum state, compute all morespec values upfront - Convert enum morespec_options to simple boolean logic (1/0) - Change 'only' from boolean to 'dispatch_bits' bitmask - Move dispatch status updates before early continues in the loop --- base/staticdata.jl | 2 ++ src/gf.c | 81 +++++++++++++++++++++----------------------- src/julia_internal.h | 2 ++ src/staticdata.c | 7 +++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index e874d27e24c2b..96fbe91100b09 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -376,6 +376,8 @@ end const METHOD_SIG_LATEST_WHICH = 0x1 # true indicates this method would be returned as the only result from `methods` when calling `method.sig` in the current latest world const METHOD_SIG_LATEST_ONLY = 0x2 +# true indicates there exists some other method that is not more specific than this one in the current latest world (which might be more fully covering) +const METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC = 0x8 function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt) @assert invokesig isa Type diff --git a/src/gf.c b/src/gf.c index 50a2e32718163..abb497235c442 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2140,12 +2140,6 @@ static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **is } -enum morespec_options { - morespec_unknown, - morespec_isnot, - morespec_is -}; - // check if `type` is replacing `m` with an ambiguity here, given other methods in `d` that already match it static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_t *const *d, size_t n, jl_value_t *isect, jl_value_t *isect2, char *morespec) { @@ -2155,9 +2149,7 @@ static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_ // see if m2 also fully covered this intersection if (m == m2 || !(jl_subtype(isect, m2->sig) || (isect2 && jl_subtype(isect2, m2->sig)))) continue; - if (morespec[k] == (char)morespec_unknown) - morespec[k] = (char)(jl_type_morespecific(m2->sig, type) ? morespec_is : morespec_isnot); - if (morespec[k] == (char)morespec_is) + if (morespec[k]) // not actually shadowing this--m2 will still be better return 0; // if type is not more specific than m (thus now dominating it) @@ -2165,7 +2157,7 @@ static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_ // since m2 was also a previous match over isect, // see if m was previously dominant over all m2 // or if this was already ambiguous before - if (ambig == morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { + if (ambig && !jl_type_morespecific(m->sig, m2->sig)) { // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type return 0; } @@ -2659,17 +2651,27 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, max_world); int invalidated = 0; - int only = !(jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_PRECOMPILE_MANY); // will compute if this will be currently the only result that would returned from `ml_matches` given `sig` + int dispatch_bits = METHOD_SIG_LATEST_WHICH; // Always set LATEST_WHICH + // Check precompiled dispatch status bits + int precompiled_status = jl_atomic_load_relaxed(&method->dispatch_status); + if (!(precompiled_status & METHOD_SIG_PRECOMPILE_MANY)) + dispatch_bits |= METHOD_SIG_LATEST_ONLY; // Tentatively set, will be cleared if not applicable + if (precompiled_status & METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC) + dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; if (replaced) { oldvalue = (jl_value_t*)replaced; jl_method_t *m = replaced->func.method; invalidated = 1; method_overwrite(newentry, m); - // this is an optimized version of below, given we know the type-intersection is exact + // This is an optimized version of below, given we know the type-intersection is exact jl_method_table_invalidate(m, max_world); int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); - jl_atomic_store_relaxed(&m->dispatch_status, 0); - only = m_dispatch & METHOD_SIG_LATEST_ONLY; + // Clear METHOD_SIG_LATEST_ONLY and METHOD_SIG_LATEST_WHICH bits, only keeping NOTMORESPECIFIC + jl_atomic_store_relaxed(&m->dispatch_status, m_dispatch & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC); + // Edge case: don't set dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC unconditionally since `m` is not an visible method for invalidations + dispatch_bits |= (m_dispatch & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC); + if (!(m_dispatch & METHOD_SIG_LATEST_ONLY)) + dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; } else { jl_method_t *const *d; @@ -2685,13 +2687,28 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) oldmi = jl_alloc_vec_any(0); char *morespec = (char*)alloca(n); - memset(morespec, morespec_unknown, n); + // Compute all morespec values upfront + for (j = 0; j < n; j++) + morespec[j] = (char)jl_type_morespecific(d[j]->sig, type); for (j = 0; j < n; j++) { jl_method_t *m = d[j]; - if (morespec[j] == (char)morespec_is) { - only = 0; - continue; + // Compute ambig state: is there an ambiguity between new method and old m? + char ambig = !morespec[j] && !jl_type_morespecific(type, m->sig); + // Compute updates to the dispatch state bits + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + if (morespec[j] || ambig) { + // !morespecific(new, old) + dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; + m_dispatch |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; + } + if (!morespec[j]) { + // !morespecific(old, new) + dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; + m_dispatch &= ~METHOD_SIG_LATEST_ONLY; } + jl_atomic_store_relaxed(&m->dispatch_status, m_dispatch); + if (morespec[j]) + continue; loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot _Atomic(jl_method_instance_t*) *data; size_t l; @@ -2703,27 +2720,19 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) data = (_Atomic(jl_method_instance_t*)*) &loctag; l = 1; } - enum morespec_options ambig = morespec_unknown; for (size_t i = 0; i < l; i++) { jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); if ((jl_value_t*)mi == jl_nothing) continue; isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes); if (jl_type_intersection2(type, isect3, &isect, &isect2)) { + // Replacing a method--see if this really was the selected method previously + // over the intersection (not ambiguous) and the new method will be selected now (morespec_is). // TODO: this only checks pair-wise for ambiguities, but the ambiguities could arise from the interaction of multiple methods // and thus might miss a case where we introduce an ambiguity between two existing methods // We could instead work to sort this into 3 groups `morespecific .. ambiguous .. lesspecific`, with `type` in ambiguous, // such that everything in `morespecific` dominates everything in `ambiguous`, and everything in `ambiguous` dominates everything in `lessspecific` // And then compute where each isect falls, and whether it changed group--necessitating invalidation--or not. - if (morespec[j] == (char)morespec_unknown) - morespec[j] = (char)(jl_type_morespecific(m->sig, type) ? morespec_is : morespec_isnot); - if (morespec[j] == (char)morespec_is) - // not actually shadowing--the existing method is still better - break; - if (ambig == morespec_unknown) - ambig = jl_type_morespecific(type, m->sig) ? morespec_isnot : morespec_is; - // replacing a method--see if this really was the selected method previously - // over the intersection (not ambiguous) and the new method will be selected now (morespec_is) int replaced_dispatch = is_replacing(ambig, type, m, d, n, isect, isect2, morespec); // found that this specialization dispatch got replaced by m // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); @@ -2740,20 +2749,6 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) invalidated |= invalidatedmi; } } - // now compute and store updates to METHOD_SIG_LATEST_ONLY - int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); - if (m_dispatch & METHOD_SIG_LATEST_ONLY) { - if (morespec[j] == (char)morespec_unknown) - morespec[j] = (char)(jl_type_morespecific(m->sig, type) ? morespec_is : morespec_isnot); - if (morespec[j] == (char)morespec_isnot) - jl_atomic_store_relaxed(&m->dispatch_status, ~METHOD_SIG_LATEST_ONLY & m_dispatch); - } - if (only) { - if (morespec[j] == (char)morespec_is || ambig == morespec_is || - (ambig == morespec_unknown && !jl_type_morespecific(type, m->sig))) { - only = 0; - } - } } } @@ -2802,7 +2797,7 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); - jl_atomic_store_relaxed(&method->dispatch_status, METHOD_SIG_LATEST_WHICH | (only ? METHOD_SIG_LATEST_ONLY : 0)); // TODO: this should be sequenced fully after the world counter store + jl_atomic_store_relaxed(&method->dispatch_status, dispatch_bits); // TODO: this should be sequenced fully after the world counter store JL_GC_POP(); } diff --git a/src/julia_internal.h b/src/julia_internal.h index 3a85b523bd3e3..c87f2cd648098 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -687,6 +687,8 @@ typedef union { #define METHOD_SIG_LATEST_WHICH 0b0001 #define METHOD_SIG_LATEST_ONLY 0b0010 #define METHOD_SIG_PRECOMPILE_MANY 0b0100 +#define METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC 0b1000 // indicates there exists some other method that is not more specific than this one +#define METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC 0b10000 // precompiled version of METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC JL_DLLEXPORT jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner); JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src); diff --git a/src/staticdata.c b/src/staticdata.c index 5d61b35c90122..c5e7139250bff 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1841,7 +1841,12 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (jl_atomic_load_relaxed(&newm->primary_world) > 1) { jl_atomic_store_relaxed(&newm->primary_world, ~(size_t)0); // min-world int dispatch_status = jl_atomic_load_relaxed(&newm->dispatch_status); - jl_atomic_store_relaxed(&newm->dispatch_status, dispatch_status & METHOD_SIG_LATEST_ONLY ? 0 : METHOD_SIG_PRECOMPILE_MANY); + int new_dispatch_status = 0; + if (!(dispatch_status & METHOD_SIG_LATEST_ONLY)) + new_dispatch_status |= METHOD_SIG_PRECOMPILE_MANY; + if (dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC) + new_dispatch_status |= METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC; + jl_atomic_store_relaxed(&newm->dispatch_status, new_dispatch_status); arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } From 908ef727da2ca5e9a148500da8af4d2ef595204d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 1 Jul 2025 09:15:25 -0400 Subject: [PATCH 465/662] optimize verify_call again --- base/staticdata.jl | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index 96fbe91100b09..e1180b69e8f5c 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -285,7 +285,7 @@ end function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool) # verify that these edges intersect with the same methods as before mi = nothing - if n == 1 && fully_covers + if n == 1 # first, fast-path a check if the expected method simply dominates its sig anyways # so the result of ml_matches is already simply known let t = expecteds[i], meth, minworld, maxworld, result @@ -298,7 +298,9 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n mi = t::MethodInstance end meth = mi.def::Method - if !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + # Fast path is legal when fully_covers=true OR when METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC is unset + if (fully_covers || iszero(meth.dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC)) && + !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) minworld = meth.primary_world @assert minworld ≤ world maxworld = typemax(UInt) @@ -306,12 +308,15 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n return minworld, maxworld, result end end - if !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) - minworld = meth.primary_world - @assert minworld ≤ world - maxworld = typemax(UInt) - result = Any[] # result is unused - return minworld, maxworld, result + # Fast path is legal when fully_covers=true OR when METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC is unset + if fully_covers || iszero(meth.dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC) + if !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + result = Any[] # result is unused + return minworld, maxworld, result + end end end end From 34bb3e794a3a4e7638e3f88408da7e6507daad23 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 1 Jul 2025 10:24:38 -0400 Subject: [PATCH 466/662] juliac: Add rudimentary Windows support (#57481) This was essentially working as-is, except for our reliance on a C compiler. Not sure how we feel about having an `Artifacts.toml` floating around our `contrib` folder, but I'm not aware of an alternative other than moving `juliac.jl` to a subdirectory. --- Makefile | 3 +- contrib/juliac/Artifacts.toml | 19 +++++++ contrib/{ => juliac}/juliac-buildscript.jl | 0 contrib/{ => juliac}/juliac-trim-base.jl | 0 contrib/{ => juliac}/juliac-trim-stdlib.jl | 0 contrib/{ => juliac}/juliac.jl | 64 ++++++++++++++++++++-- doc/src/devdocs/sysimg.md | 2 +- test/trimming/Makefile | 2 +- 8 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 contrib/juliac/Artifacts.toml rename contrib/{ => juliac}/juliac-buildscript.jl (100%) rename contrib/{ => juliac}/juliac-trim-base.jl (100%) rename contrib/{ => juliac}/juliac-trim-stdlib.jl (100%) rename contrib/{ => juliac}/juliac.jl (66%) diff --git a/Makefile b/Makefile index fc73b897c3aa3..39f34b5372262 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac.jl $(build_datarootdir)/julia/juliac-buildscript.jl $(build_datarootdir)/julia/juliac-trim-base.jl $(build_datarootdir)/julia/juliac-trim-stdlib.jl +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac/juliac.jl $(build_datarootdir)/julia/juliac/juliac-buildscript.jl $(build_datarootdir)/julia/juliac/juliac-trim-base.jl $(build_datarootdir)/julia/juliac/juliac-trim-stdlib.jl @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps @@ -184,6 +184,7 @@ $(build_sysconfdir)/julia/startup.jl: $(JULIAHOME)/etc/startup.jl | $(build_sysc @cp $< $@ $(build_datarootdir)/julia/%: $(JULIAHOME)/contrib/% | $(build_datarootdir)/julia + mkdir -p $(dir $@) $(INSTALL_M) $< $(dir $@) $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir) diff --git a/contrib/juliac/Artifacts.toml b/contrib/juliac/Artifacts.toml new file mode 100644 index 0000000000000..54771b41b21f7 --- /dev/null +++ b/contrib/juliac/Artifacts.toml @@ -0,0 +1,19 @@ +[[mingw-w64]] +arch = "x86_64" +git-tree-sha1 = "b17bda08a19173572926f43a48aad5ef3d845e7c" +os = "windows" +lazy = true + + [[mingw-w64.download]] + sha256 = "53645e06775a55733580426341395c67dda20a664af83bcda76a1d052b618b59" + url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/x86_64-14.2.0-release-posix-seh-msvcrt-rt_v12-rev0.tar.gz" + +[[mingw-w64]] +arch = "i686" +git-tree-sha1 = "76b9f278e7de1d7dfdfe3a786afbe9c1e29003ea" +os = "windows" +lazy = true + + [[mingw-w64.download]] + sha256 = "d049bd771e01b02f2ca9274435f0e6f9f4f295bf2af72a8059dd851c52144910" + url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/i686-14.2.0-release-posix-dwarf-msvcrt-rt_v12-rev0.tar.gz" diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac/juliac-buildscript.jl similarity index 100% rename from contrib/juliac-buildscript.jl rename to contrib/juliac/juliac-buildscript.jl diff --git a/contrib/juliac-trim-base.jl b/contrib/juliac/juliac-trim-base.jl similarity index 100% rename from contrib/juliac-trim-base.jl rename to contrib/juliac/juliac-trim-base.jl diff --git a/contrib/juliac-trim-stdlib.jl b/contrib/juliac/juliac-trim-stdlib.jl similarity index 100% rename from contrib/juliac-trim-stdlib.jl rename to contrib/juliac/juliac-trim-stdlib.jl diff --git a/contrib/juliac.jl b/contrib/juliac/juliac.jl similarity index 66% rename from contrib/juliac.jl rename to contrib/juliac/juliac.jl index d637f1f1bf3a2..ed80d88444639 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac/juliac.jl @@ -3,8 +3,10 @@ # Julia compiler wrapper script # NOTE: The interface and location of this script are considered unstable/experimental +using LazyArtifacts + module JuliaConfig - include(joinpath(@__DIR__, "julia-config.jl")) + include(joinpath(@__DIR__, "..", "julia-config.jl")) end julia_cmd = `$(Base.julia_cmd()) --startup-file=no --history-file=no` @@ -30,6 +32,57 @@ if help !== nothing exit(0) end +# Copied from PackageCompiler +# https://github.com/JuliaLang/PackageCompiler.jl/blob/1c35331d8ef81494f054bbc71214811253101993/src/PackageCompiler.jl#L147-L190 +function get_compiler_cmd(; cplusplus::Bool=false) + cc = get(ENV, "JULIA_CC", nothing) + path = nothing + @static if Sys.iswindows() + path = joinpath(LazyArtifacts.artifact"mingw-w64", + "extracted_files", + (Int==Int64 ? "mingw64" : "mingw32"), + "bin", + cplusplus ? "g++.exe" : "gcc.exe") + compiler_cmd = `$path` + end + if cc !== nothing + compiler_cmd = Cmd(Base.shell_split(cc)) + path = nothing + elseif !Sys.iswindows() + compilers_cpp = ("g++", "clang++") + compilers_c = ("gcc", "clang") + found_compiler = false + if cplusplus + for compiler in compilers_cpp + if Sys.which(compiler) !== nothing + compiler_cmd = `$compiler` + found_compiler = true + break + end + end + end + if !found_compiler + for compiler in compilers_c + if Sys.which(compiler) !== nothing + compiler_cmd = `$compiler` + found_compiler = true + if cplusplus && !WARNED_CPP_COMPILER[] + @warn "could not find a c++ compiler (g++ or clang++), falling back to $compiler, this might cause link errors" + WARNED_CPP_COMPILER[] = true + end + break + end + end + end + found_compiler || error("could not find a compiler, looked for ", + join(((cplusplus ? compilers_cpp : ())..., compilers_c...), ", ", " and ")) + end + if path !== nothing + compiler_cmd = addenv(compiler_cmd, "PATH" => string(ENV["PATH"], ";", dirname(path))) + end + return compiler_cmd +end + # arguments to forward to julia compilation process julia_args = [] enable_trim::Bool = false @@ -82,6 +135,7 @@ function get_rpath(; relative::Bool = false) end end +cc = get_compiler_cmd() absfile = abspath(file) cflags = JuliaConfig.cflags(; framework=false) cflags = Base.shell_split(cflags) @@ -93,7 +147,6 @@ tmpdir = mktempdir(cleanup=false) img_path = joinpath(tmpdir, "img.a") bc_path = joinpath(tmpdir, "img-bc.a") - function precompile_env() # Pre-compile the environment # (otherwise obscure error messages will occur) @@ -121,7 +174,6 @@ function compile_products(enable_trim::Bool) println(stderr, "\nFailed to compile $file") exit(1) end - end function link_products() @@ -137,11 +189,11 @@ function link_products() julia_libs = Base.shell_split(Base.isdebugbuild() ? "-ljulia-debug -ljulia-internal-debug" : "-ljulia -ljulia-internal") try if output_type == "--output-lib" - cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` elseif output_type == "--output-sysimage" - cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` else - cmd2 = `cc $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` end verbose && println("Running: $cmd2") run(cmd2) diff --git a/doc/src/devdocs/sysimg.md b/doc/src/devdocs/sysimg.md index 2cbba2744d4a1..e8202736e57e1 100644 --- a/doc/src/devdocs/sysimg.md +++ b/doc/src/devdocs/sysimg.md @@ -176,7 +176,7 @@ debug info, respectively, and so will make debugging more difficult. We have identified many small changes to Base that significantly increase the set of programs that can be reliably trimmed. Unfortunately some of those changes would be considered breaking, and so are only applied when trimming is requested (this is done by an external build script, -currently maintained inside the test suite as `contrib/juliac-buildscript.jl`). +currently maintained inside the test suite as `contrib/juliac/juliac-buildscript.jl`). Therefore in many cases trimming will require you to opt in to new variants of Base and some standard libraries. diff --git a/test/trimming/Makefile b/test/trimming/Makefile index e3c7536bbc92c..c3145765655e7 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -29,7 +29,7 @@ CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal # get the JuliaC build script -JULIAC_BUILDSCRIPT := $(shell $(JULIA) -e 'print(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "juliac-buildscript.jl"))') +JULIAC_BUILDSCRIPT := $(shell $(JULIA) -e 'print(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "juliac", "juliac-buildscript.jl"))') #============================================================================= From 3ed13ea7ab3d73f408b12a70ad29565c97bf5562 Mon Sep 17 00:00:00 2001 From: Simeon David Schaub Date: Tue, 1 Jul 2025 16:58:39 +0200 Subject: [PATCH 467/662] fix null comparisons for non-standard address spaces (#58837) Co-authored-by: Jameson Nash --- src/cgutils.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 6cdfbc7add6c1..7c2f62ac3350c 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1601,10 +1601,17 @@ static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name, ctx.builder.SetInsertPoint(ifok); } +// ctx.builder.CreateIsNotNull(v) lowers incorrectly in non-standard +// address spaces where null is not zero +// TODO: adapt to https://github.com/llvm/llvm-project/pull/131557 once merged static Value *null_pointer_cmp(jl_codectx_t &ctx, Value *v) { ++EmittedNullchecks; - return ctx.builder.CreateIsNotNull(v); + Type *T = v->getType(); + return ctx.builder.CreateICmpNE( + v, + ctx.builder.CreateAddrSpaceCast( + Constant::getNullValue(ctx.builder.getPtrTy(0)), T)); } From 0ea036bc72a0ed7d7ee598dd9d26a02d14f148f1 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:10:00 -0400 Subject: [PATCH 468/662] debuginfo: Memoize object symbol lookup (#58851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes https://github.com/JuliaLang/julia/pull/58355. Resolves https://github.com/JuliaLang/julia/issues/58326. On this PR: ```julia julia> @btime lgamma(2.0) ┌ Warning: `lgamma(x::Real)` is deprecated, use `(logabsgamma(x))[1]` instead. │ caller = var"##core#283"() at execution.jl:598 └ @ Core ~/.julia/packages/BenchmarkTools/1i1mY/src/execution.jl:598 47.730 μs (105 allocations: 13.24 KiB) ``` On `nightly`: ```julia julia> @btime lgamma(2.0) ┌ Warning: `lgamma(x::Real)` is deprecated, use `(logabsgamma(x))[1]` instead. │ caller = var"##core#283"() at execution.jl:598 └ @ Core ~/.julia/packages/BenchmarkTools/1i1mY/src/execution.jl:598 26.856 ms (89 allocations: 11.32 KiB) ``` --- src/debug-registry.h | 7 ++- src/debuginfo.cpp | 116 ++++++++++++++++++++++++------------------- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/src/debug-registry.h b/src/debug-registry.h index 72189c60d3d40..00e3445200361 100644 --- a/src/debug-registry.h +++ b/src/debug-registry.h @@ -12,7 +12,8 @@ typedef struct { const llvm::object::ObjectFile *obj; llvm::DIContext *ctx; int64_t slide; -} objfileentry_t; + std::map> *symbolmap; +} jl_object_file_entry_t; // Central registry for resolving function addresses to `jl_code_instance_t`s and // originating `ObjectFile`s (for the DWARF debug info). @@ -121,7 +122,7 @@ class JITDebugInfoRegistry using rev_map = std::map>; typedef rev_map objectmap_t; - typedef rev_map objfilemap_t; + typedef rev_map objfilemap_t; objectmap_t objectmap{}; rev_map> cimap{}; @@ -152,4 +153,6 @@ class JITDebugInfoRegistry void add_image_info(image_info_t info) JL_NOTSAFEPOINT; bool get_image_info(uint64_t base, image_info_t *info) const JL_NOTSAFEPOINT; Locked::LockT get_objfile_map() JL_NOTSAFEPOINT; + + std::shared_mutex symbol_mutex; }; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index edfd2d05c716d..4adfd3398e13d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -717,7 +717,8 @@ static inline void ignoreError(T &err) JL_NOTSAFEPOINT #endif } -static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool inimage, +static void get_function_name_and_base(llvm::object::SectionRef Section, std::map> *symbolmap, + size_t pointer, int64_t slide, bool inimage, void **saddr, char **name, bool untrusted_dladdr) JL_NOTSAFEPOINT { bool needs_saddr = saddr && (!*saddr || untrusted_dladdr); @@ -743,59 +744,73 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t #endif } if (Section.getObject() && (needs_saddr || needs_name)) { - size_t distance = (size_t)-1; - object::SymbolRef sym_found; - for (auto sym : Section.getObject()->symbols()) { - if (!Section.containsSymbol(sym)) - continue; - auto addr = sym.getAddress(); - if (!addr) - continue; - size_t symptr = addr.get(); - if (symptr > pointer + slide) - continue; - size_t new_dist = pointer + slide - symptr; - if (new_dist > distance) - continue; - distance = new_dist; - sym_found = sym; - } - if (distance != (size_t)-1) { - if (needs_saddr) { - uintptr_t addr = cantFail(sym_found.getAddress()); - *saddr = (void*)(addr - slide); - needs_saddr = false; + uintptr_t addr = 0; + StringRef nameref{}; + { + std::shared_lock read_lock(getJITDebugRegistry().symbol_mutex); + if (symbolmap->empty()) { + read_lock.unlock(); + { + // symbol map hasn't been generated yet, so fill it in now + std::unique_lock write_lock(getJITDebugRegistry().symbol_mutex); + if (symbolmap->empty()) { + for (auto sym : Section.getObject()->symbols()) { + if (!Section.containsSymbol(sym)) + continue; + + auto maybe_addr = sym.getAddress(); + if (!maybe_addr) + continue; + size_t addr = maybe_addr.get(); + + auto maybe_nameref = sym.getName(); + StringRef nameref{}; + if (maybe_nameref) + nameref = maybe_nameref.get(); + + symbolmap->emplace(addr, nameref); + } + } + } + read_lock.lock(); + } + auto fit = symbolmap->lower_bound(pointer + slide); + if (fit != symbolmap->end()) { + addr = fit->first; + nameref = fit->second; } - if (needs_name) { - if (auto name_or_err = sym_found.getName()) { - auto nameref = name_or_err.get(); - const char globalPrefix = // == DataLayout::getGlobalPrefix + } + std::string namerefstr = nameref.str(); + if (needs_saddr && addr != 0) { + *saddr = (void*)(addr - slide); + needs_saddr = false; + } + if (needs_name && !nameref.empty()) { + const char globalPrefix = // == DataLayout::getGlobalPrefix #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - '_'; + '_'; #elif defined(_OS_DARWIN_) - '_'; + '_'; #else - '\0'; + '\0'; #endif - if (globalPrefix) { - if (nameref[0] == globalPrefix) - nameref = nameref.drop_front(); + if (globalPrefix) { + if (nameref[0] == globalPrefix) + nameref = nameref.drop_front(); #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - else if (nameref[0] == '@') // X86_VectorCall - nameref = nameref.drop_front(); + else if (nameref[0] == '@') // X86_VectorCall + nameref = nameref.drop_front(); #endif - // else VectorCall, Assembly, Internal, etc. - } + // else VectorCall, Assembly, Internal, etc. + } #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - nameref = nameref.split('@').first; + nameref = nameref.split('@').first; #endif - size_t len = nameref.size(); - *name = (char*)realloc_s(*name, len + 1); - memcpy(*name, nameref.data(), len); - (*name)[len] = 0; - needs_name = false; - } - } + size_t len = nameref.size(); + *name = (char*)realloc_s(*name, len + 1); + memcpy(*name, nameref.data(), len); + (*name)[len] = 0; + needs_name = false; } } #ifdef _OS_WINDOWS_ @@ -819,7 +834,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t #endif } -static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT +static jl_object_file_entry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT { int isdarwin = 0, islinux = 0, iswindows = 0; #if defined(_OS_DARWIN_) @@ -832,7 +847,7 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA (void)iswindows; // GOAL: Read debuginfo from file - objfileentry_t entry{nullptr, nullptr, 0}; + jl_object_file_entry_t entry{nullptr, nullptr, 0, nullptr}; auto success = getJITDebugRegistry().get_objfile_map()->emplace(fbase, entry); if (!success.second) // Return cached value @@ -1009,7 +1024,8 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA auto binary = errorobj->takeBinary(); binary.first.release(); binary.second.release(); - entry = {debugobj, context, slide}; + + entry = {debugobj, context, slide, new std::map>()}; // update cache (*getJITDebugRegistry().get_objfile_map())[fbase] = entry; } @@ -1139,7 +1155,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * jl_copy_str(filename, dlinfo.dli_fname); fname = dlinfo.dli_fname; #endif // ifdef _OS_WINDOWS_ - auto entry = find_object_file(fbase, fname); + jl_object_file_entry_t entry = find_object_file(fbase, fname); *slide = entry.slide; *context = entry.ctx; if (entry.obj) @@ -1147,7 +1163,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * // Assume we only need base address for sysimg for now if (!inimage || 0 == image_info.fptrs.nptrs) saddr = nullptr; - get_function_name_and_base(*Section, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); + get_function_name_and_base(*Section, entry.symbolmap, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); return true; } From 056e68b1a62c4cc839b87e13ca555982fe0e1aba Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 1 Jul 2025 16:35:40 -0400 Subject: [PATCH 469/662] bpart: Skip inserting image backedges while we're generating a pkgimage (#58843) Should speed up deeply nested precompiles by skipping unnecessary work here. PR is against #58830 to avoid conflicts, but semantically independent. --- base/invalidation.jl | 4 ++++ src/module.c | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 0d814b3f31f67..9a3d1ddd912e6 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -206,6 +206,10 @@ function scan_new_method!(method::Method, image_backedges_only::Bool) end function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) + if image_backedges_only && Base.generating_output(true) + # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion + return + end for method in internal_methods if isa(method, Method) scan_new_method!(method, image_backedges_only) diff --git a/src/module.c b/src/module.c index 7c83d9329ab50..b0e3efb92fd5d 100644 --- a/src/module.c +++ b/src/module.c @@ -1669,8 +1669,14 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, // Until the first such replacement, we can fast-path validation. // For these purposes, we consider the `Main` module to be a non-sysimg module. // This is legal, because we special case the `Main` in check_safe_import_from. - if (jl_object_in_image((jl_value_t*)b) && b->globalref->mod != jl_main_module && jl_atomic_load_relaxed(&jl_first_image_replacement_world) == ~(size_t)0) + if (jl_object_in_image((jl_value_t*)b) && b->globalref->mod != jl_main_module && jl_atomic_load_relaxed(&jl_first_image_replacement_world) == ~(size_t)0) { + // During incremental compilation replacement of image bindings is forbidden; + // We use this to avoid inserting backedges while loading pkgimages. + // `check_safe_newbinding` checks an equivalent condition on `b->globalref->mod`, + // but doesn't quite query `jl_object_in_image`, so assert here to be extra sure. + assert(!(jl_options.incremental && jl_generating_output())); jl_atomic_store_relaxed(&jl_first_image_replacement_world, new_world); + } assert(jl_atomic_load_relaxed(&b->partitions) == old_bpart); jl_binding_partition_t *new_bpart = new_binding_partition(); From 92aacab2ed35671fa0762fc1a65df8b254fbd302 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 1 Jul 2025 19:24:58 -0300 Subject: [PATCH 470/662] Re-add old function name for backward compatibility in init (#58860) While julia has no C-API backwards compatibility guarantees this is simple enough to add. Fixes #58859 --- src/jl_exported_funcs.inc | 1 + src/jlapi.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index ea6f4c38f7f70..6ecc227d0c1a5 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -238,6 +238,7 @@ XX(jl_init_) \ XX(jl_init_options) \ XX(jl_init_restored_module) \ + XX(jl_init_with_image) \ XX(jl_init_with_image_file) \ XX(jl_init_with_image_handle) \ XX(jl_install_sigint_handler) \ diff --git a/src/jlapi.c b/src/jlapi.c index 0d8ddd10d9ea1..6559adc94c1b9 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -120,6 +120,13 @@ JL_DLLEXPORT void jl_init_with_image_file(const char *julia_bindir, jl_exception_clear(); } +// Deprecated function, kept for backward compatibility +JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, + const char *image_path) +{ + jl_init_with_image_file(julia_bindir, image_path); +} + /** * @brief Initialize the Julia runtime. * From 9fd74b8addf09c4ece03452fd571acfdea4d85d8 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 2 Jul 2025 08:50:30 -0400 Subject: [PATCH 471/662] trimming: Add `_uv_hook_close` support (#58871) Resolves https://github.com/JuliaLang/julia/issues/58862. Since this hook is called internally by the runtime, `--trim` was not aware of the callee edge required here. --- base/libuv.jl | 9 ++++++++- src/staticdata.c | 3 ++- test/trimming/basic_jll.jl | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/libuv.jl b/base/libuv.jl index 8f066971a639d..5e9bdfaf1e75c 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -39,8 +39,15 @@ macro handle_as(hand, typ) end end -@nospecializeinfer associate_julia_struct(handle::Ptr{Cvoid}, @nospecialize(jlobj)) = +function _uv_hook_close end + +function associate_julia_struct(handle::Ptr{Cvoid}, jlobj::T) where T + # This `cfunction` is not used anywhere, but it triggers compilation of this + # MethodInstance for `--trim` so that it will be available when dispatched to + # by `jl_uv_call_close_callback()` + _ = @cfunction(Base._uv_hook_close, Cvoid, (Ref{T},)) ccall(:jl_uv_associate_julia_struct, Cvoid, (Ptr{Cvoid}, Any), handle, jlobj) +end disassociate_julia_struct(uv) = disassociate_julia_struct(uv.handle) disassociate_julia_struct(handle::Ptr{Cvoid}) = handle != C_NULL && ccall(:jl_uv_disassociate_julia_struct, Cvoid, (Ptr{Cvoid},), handle) diff --git a/src/staticdata.c b/src/staticdata.c index 5d61b35c90122..8c38416c7e2b3 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -732,7 +732,8 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ !strcmp(jl_symbol_name(b->globalref->name), "__init__") || // ... or point to Base functions accessed by the runtime (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || - !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { + !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook") || + !strcmp(jl_symbol_name(b->globalref->name), "_uv_hook_close"))))) { jl_queue_for_serialization(s, b); } } diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index 2a63797b4d13a..6ac6b2797d4e3 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -18,6 +18,8 @@ function @main(args::Vector{String})::Cint println(Core.stdout, ver) @assert ver == build_ver + sleep(0.01) + # Add an indirection via `@cfunction` / 1-arg ccall cfunc = @cfunction(print_string, Cvoid, (Ptr{Cvoid},)) fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) From e631972e88603df228d2d9aad0141ece0d3049f4 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 2 Jul 2025 09:03:43 -0400 Subject: [PATCH 472/662] Don't `@inbounds` AbstractArray's iterate method; optimize `checkbounds` instead (#58793) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split off from #58785, this simplifies `iterate` and removes the `@inbounds` call that was added in https://github.com/JuliaLang/julia/pull/58635. It achieves the same (or better!) performance, however, by targeting optimizations in `checkbounds` and — in particular — the construction of a linear `eachindex` (against which the bounds are checked). --------- Co-authored-by: Mosè Giordano --- base/abstractarray.jl | 9 +++++---- base/array.jl | 4 ---- base/genericmemory.jl | 9 --------- base/range.jl | 3 ++- base/strings/basic.jl | 2 +- base/subarray.jl | 10 ++++++++++ test/boundscheck_exec.jl | 16 ++++++++++++++++ 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 94bf3170feb38..df0dc88efe347 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -387,6 +387,7 @@ function eachindex(A::AbstractArray, B::AbstractArray...) @inline eachindex(IndexStyle(A,B...), A, B...) end +eachindex(::IndexLinear, A::Union{Array, Memory}) = unchecked_oneto(length(A)) eachindex(::IndexLinear, A::AbstractArray) = (@inline; oneto(length(A))) eachindex(::IndexLinear, A::AbstractVector) = (@inline; axes1(A)) function eachindex(::IndexLinear, A::AbstractArray, B::AbstractArray...) @@ -1237,15 +1238,15 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) iterate_starting_state(A) = iterate_starting_state(A, IndexStyle(A)) iterate_starting_state(A, ::IndexLinear) = firstindex(A) iterate_starting_state(A, ::IndexStyle) = (eachindex(A),) -iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate(A, state) -function _iterate(A::AbstractArray, state::Tuple) +@inline iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate(A, state) +@inline function _iterate(A::AbstractArray, state::Tuple) y = iterate(state...) y === nothing && return nothing A[y[1]], (state[1], tail(y)...) end -function _iterate(A::AbstractArray, state::Integer) +@inline function _iterate(A::AbstractArray, state::Integer) checkbounds(Bool, A, state) || return nothing - @inbounds(A[state]), state + one(state) + A[state], state + one(state) end isempty(a::AbstractArray) = (length(a) == 0) diff --git a/base/array.jl b/base/array.jl index 61a3c39ca2bb7..cacef7c4acd5d 100644 --- a/base/array.jl +++ b/base/array.jl @@ -902,10 +902,6 @@ function grow_to!(dest, itr, st) return dest end -## Iteration ## - -iterate(A::Array, i=1) = (@inline; _iterate_array(A, i)) - ## Indexing: getindex ## """ diff --git a/base/genericmemory.jl b/base/genericmemory.jl index d25d213a3546e..b180462115f41 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -223,15 +223,6 @@ Memory{T}(x::AbstractArray{S,1}) where {T,S} = copyto_axcheck!(Memory{T}(undef, ## copying iterators to containers -## Iteration ## - -function _iterate_array(A::Union{Memory, Array}, i::Int) - @inline - checkbounds(Bool, A, i) ? (A[i], i + 1) : nothing -end - -iterate(A::Memory, i=1) = (@inline; _iterate_array(A, i)) - ## Indexing: getindex ## # Faster contiguous indexing using copyto! for AbstractUnitRange and Colon diff --git a/base/range.jl b/base/range.jl index 9d2b9fd736b22..3e1cd77eb914b 100644 --- a/base/range.jl +++ b/base/range.jl @@ -687,7 +687,7 @@ end ## interface implementations length(r::AbstractRange) = error("length implementation missing") # catch mistakes -size(r::AbstractRange) = (length(r),) +size(r::AbstractRange) = (@inline; (length(r),)) isempty(r::StepRange) = # steprange_last(r.start, r.step, r.stop) == r.stop @@ -802,6 +802,7 @@ let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, # slightly more accurate length and checked_length in extreme cases # (near typemax) for types with known `unsigned` functions function length(r::OrdinalRange{T}) where T<:bigints + @inline s = step(r) diff = last(r) - first(r) isempty(r) && return zero(diff) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 6619a2b25574e..a436a5494fa79 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -797,7 +797,7 @@ size(s::CodeUnits) = (length(s),) elsize(s::Type{<:CodeUnits{T}}) where {T} = sizeof(T) @propagate_inbounds getindex(s::CodeUnits, i::Int) = codeunit(s.s, i) IndexStyle(::Type{<:CodeUnits}) = IndexLinear() -@inline iterate(s::CodeUnits, i=1) = checkbounds(Bool, s, i) ? (@inbounds s[i], i + 1) : nothing +checkbounds(::Type{Bool}, s::CodeUnits, i::Integer) = checkbounds(Bool, s.s, i) write(io::IO, s::CodeUnits) = write(io, s.s) diff --git a/base/subarray.jl b/base/subarray.jl index eacaddc068f1f..3a0be7d82b981 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -522,6 +522,16 @@ function _indices_sub(i1::AbstractArray, I...) (axes(i1)..., _indices_sub(I...)...) end +axes1(::SubArray{<:Any,0}) = OneTo(1) +axes1(S::SubArray) = (@inline; _axes1_sub(S.indices...)) +_axes1_sub() = () +_axes1_sub(::Real, I...) = (@inline; _axes1_sub(I...)) +_axes1_sub(::AbstractArray{<:Any,0}, I...) = _axes1_sub(I...) +function _axes1_sub(i1::AbstractArray, I...) + @inline + axes1(i1) +end + has_offset_axes(S::SubArray) = has_offset_axes(S.indices...) function replace_in_print_matrix(S::SubArray{<:Any,2,<:AbstractMatrix}, i::Integer, j::Integer, s::AbstractString) diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index e3d5e77c05384..2822176929f4a 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -353,4 +353,20 @@ if bc_opt == bc_default @test (@allocated no_alias_prove(5)) == 0 end +@testset "automatic boundscheck elision for iteration on some important types" begin + if bc_opt != bc_on + @test !contains(sprint(code_llvm, iterate, (Memory{UInt8}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (Vector{UInt8}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (Matrix{UInt8}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (Array{UInt8,3}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 1, Vector{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 2, Matrix{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}}, true}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 2, Matrix{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (Base.CodeUnits{UInt8,String}, Int)), "unreachable") + end +end + end From 2f57a311d51825d47d2d8fe50f2b8f42644345de Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 2 Jul 2025 09:11:07 -0400 Subject: [PATCH 473/662] aotcompile: Fix early-exit if CI not found for `cfunction` (#58722) As written, this was accidentally skipping all the subsequent `cfuncs` that need adapters. --- src/aotcompile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 9bbc70ce0b001..f8057e6a013e8 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -652,7 +652,7 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } } Function *f = codeinst ? aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, "", false) : unspec; - return assign_fptr(f); + assign_fptr(f); } } From 17312feb72976c6ea0003bf3e3c63ea8e01fd4d1 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Wed, 2 Jul 2025 10:14:17 -0400 Subject: [PATCH 474/662] zero-index get/setindex(::ReinterpretArray) require a length of 1 (#58814) fix https://github.com/JuliaLang/julia/issues/58232 o3 helped me understand the existing implementations but code is mine --------- Co-authored-by: Matt Bauman --- NEWS.md | 2 ++ base/reinterpretarray.jl | 4 ++-- test/reinterpretarray.jl | 13 +++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 55b5a09d5b525..879f68c47bb6a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,6 +17,8 @@ is considered a bug fix ([#47102]) - The `hash` algorithm and its values have changed. Most `hash` specializations will remain correct and require no action. Types that reimplement the core hashing logic independently, such as some third-party string packages do, may require a migration to the new algorithm. ([#57509]) +* Indexless `getindex` and `setindex!` (i.e. `A[]`) on `ReinterpretArray` now correctly throw a `BoundsError` when there is more than one element. ([#58814]) + Compiler/Runtime improvements ----------------------------- diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index d7f6a15afa0a7..07ad7325e6aed 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -391,7 +391,7 @@ check_ptr_indexable(a::Array, sz) = sizeof(eltype(a)) !== sz check_ptr_indexable(a::Memory, sz) = true check_ptr_indexable(a::AbstractArray, sz) = false -@propagate_inbounds getindex(a::ReinterpretArray) = a[firstindex(a)] +@propagate_inbounds getindex(a::ReshapedReinterpretArray{T,0}) where {T} = a[firstindex(a)] @propagate_inbounds isassigned(a::ReinterpretArray, inds::Integer...) = checkbounds(Bool, a, inds...) && (check_ptr_indexable(a) || _isassigned_ra(a, inds...)) @propagate_inbounds isassigned(a::ReinterpretArray, inds::SCartesianIndex2) = isassigned(a.parent, inds.j) @@ -541,7 +541,7 @@ end setindex!(a, v, firstindex(a)) end -@propagate_inbounds setindex!(a::ReinterpretArray, v) = setindex!(a, v, firstindex(a)) +@propagate_inbounds setindex!(a::ReshapedReinterpretArray{T,0}, v) where {T} = setindex!(a, v, firstindex(a)) @propagate_inbounds function setindex!(a::ReinterpretArray{T,N,S}, v, inds::Vararg{Int, N}) where {T,N,S} check_writable(a) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index e6381329e4ec6..65befefa710a7 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -21,6 +21,7 @@ A = Int64[1, 2, 3, 4] Ars = Int64[1 3; 2 4] B = Complex{Int64}[5+6im, 7+8im, 9+10im] Av = [Int32[1,2], Int32[3,4]] +C = view([1,1], [1,2]) test_many_wrappers(Ars, (identity, tslow)) do Ar @test @inferred(ndims(reinterpret(reshape, Complex{Int64}, Ar))) == 1 @@ -36,6 +37,10 @@ test_many_wrappers(B, (identity, tslow)) do _B @test @inferred(size(reinterpret(reshape, Int128, _B))) == (3,) end +test_many_wrappers(C) do Cr + @test reinterpret(reshape, Tuple{Int8, Int}, Cr) == fill((1,1)) +end + @test_throws ArgumentError("cannot reinterpret `Int64` as `Vector{Int64}`, type `Vector{Int64}` is not a bits type") reinterpret(Vector{Int64}, A) @test_throws ArgumentError("cannot reinterpret `Vector{Int32}` as `Int32`, type `Vector{Int32}` is not a bits type") reinterpret(Int32, Av) @test_throws ArgumentError("cannot reinterpret a zero-dimensional `Int64` array to `Int32` which is of a different size") reinterpret(Int32, reshape([Int64(0)])) @@ -272,7 +277,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 2) @test all(a .=== reinterpret(Float64, [Int64(2)])[1]) @test all(r .=== Int64(2)) - for badinds in (0, 16, (0,1), (1,0), (6,3), (5,4)) + for badinds in ((), 0, 16, (0,1), (1,0), (6,3), (5,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -2 end @@ -285,7 +290,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 3) @test all(a .=== reinterpret(Float64, [(Int32(3), Int32(3))])[1]) @test all(r .=== Int32(3)) - for badinds in (0, 31, (0,1), (1,0), (11,3), (10,4)) + for badinds in ((), 0, 31, (0,1), (1,0), (11,3), (10,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -3 end @@ -298,7 +303,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 4) @test all(a[1:2:5,:] .=== reinterpret(Float64, [Int64(4)])[1]) @test all(r .=== Int64(4)) - for badinds in (0, 10, (0,1), (1,0), (4,3), (3,4)) + for badinds in ((), 0, 10, (0,1), (1,0), (4,3), (3,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -4 end @@ -311,7 +316,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 5) @test all(a[1:2:5,:] .=== reinterpret(Float64, [(Int32(5), Int32(5))])[1]) @test all(r .=== Int32(5)) - for badinds in (0, 19, (0,1), (1,0), (7,3), (6,4)) + for badinds in ((), 0, 19, (0,1), (1,0), (7,3), (6,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -5 end From d49773cc8a9acbbf7efc1cc5f9489f5b65266f97 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Wed, 2 Jul 2025 12:28:26 -0400 Subject: [PATCH 475/662] Add `Base.isprecompilable` (#58805) Alternative to https://github.com/JuliaLang/julia/pull/58146. We want to compile a subset of the possible specializations of a function. To this end, we have a number of manually written `precompile` statements. Creating this list is, unfortunately, error-prone, and the list is also liable to going stale. Thus we'd like to validate each `precompile` statement in the list. The simple answer is, of course, to actually run the `precompile`s, and we naturally do so, but this takes time. We would like a relatively quick way to check the validity of a `precompile` statement. This is a dev-loop optimization, to allow us to check "is-precompilable" in unit tests. We can't use `hasmethod` as it has both false positives (too loose): ```julia julia> hasmethod(sum, (AbstractVector,)) true julia> precompile(sum, (AbstractVector,)) false julia> Base.isprecompilable(sum, (AbstractVector,)) # <- this PR false ``` and also false negatives (too strict): ```julia julia> bar(@nospecialize(x::AbstractVector{Int})) = 42 bar (generic function with 1 method) julia> hasmethod(bar, (AbstractVector,)) false julia> precompile(bar, (AbstractVector,)) true julia> Base.isprecompilable(bar, (AbstractVector,)) # <- this PR true ``` We can't use `hasmethod && isconcretetype` as it has false negatives (too strict): ```julia julia> has_concrete_method(f, argtypes) = all(isconcretetype, argtypes) && hasmethod(f, argtypes) has_concrete_method (generic function with 1 method) julia> has_concrete_method(bar, (AbstractVector,)) false julia> has_concrete_method(convert, (Type{Int}, Int32)) false julia> precompile(convert, (Type{Int}, Int32)) true julia> Base.isprecompilable(convert, (Type{Int}, Int32)) # <- this PR true ``` `Base.isprecompilable` is essentially `precompile` without the actual compilation. --- base/loading.jl | 14 ++++++++++++++ src/gf.c | 9 +++++++++ src/jl_exported_funcs.inc | 1 + 3 files changed, 24 insertions(+) diff --git a/base/loading.jl b/base/loading.jl index cbe69411fe9d1..e204a8cb01d50 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -4187,6 +4187,20 @@ function expand_compiler_path(tup) end compiler_chi(tup::Tuple) = CacheHeaderIncludes(expand_compiler_path(tup)) +""" + isprecompilable(f, argtypes::Tuple{Vararg{Any}}) + +Check, as far as is possible without actually compiling, if the given +function `f` can be compiled for the argument tuple (of types) `argtypes`. +""" +function isprecompilable(@nospecialize(f), @nospecialize(argtypes::Tuple)) + isprecompilable(Tuple{Core.Typeof(f), argtypes...}) +end + +function isprecompilable(@nospecialize(argt::Type)) + ccall(:jl_is_compilable, Int32, (Any,), argt) != 0 +end + """ precompile(f, argtypes::Tuple{Vararg{Any}}) diff --git a/src/gf.c b/src/gf.c index 50a2e32718163..62ad57b0f906b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3676,6 +3676,15 @@ JL_DLLEXPORT void jl_compile_method_sig(jl_method_t *m, jl_value_t *types, jl_sv jl_compile_method_instance(mi, NULL, world); } +JL_DLLEXPORT int jl_is_compilable(jl_tupletype_t *types) +{ + size_t world = jl_atomic_load_acquire(&jl_world_counter); + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + jl_method_instance_t *mi = jl_get_compile_hint_specialization(types, world, &min_valid, &max_valid, 1); + return mi == NULL ? 0 : 1; +} + JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) { size_t world = jl_atomic_load_acquire(&jl_world_counter); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6ecc227d0c1a5..5db67290dcb36 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -265,6 +265,7 @@ XX(jl_istopmod) \ XX(jl_is_binding_deprecated) \ XX(jl_is_char_signed) \ + XX(jl_is_compilable) \ XX(jl_is_const) \ XX(jl_is_assertsbuild) \ XX(jl_is_debugbuild) \ From 8e524c73804a9615dd68011b2c5741947d19bbb6 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Wed, 2 Jul 2025 10:46:19 -0700 Subject: [PATCH 476/662] Add a `similar` method for `Type{<:CodeUnits}` (#57826) Currently, `similar(::CodeUnits)` works as expected by going through the generic `AbstractArray` method. However, the fallback method hit by `similar(::Type{<:CodeUnits}, dims)` does not work, as it assumes the existence of a constructor that accepts an `UndefInitializer`. This can be made to work by defining a corresponding `similar` method that returns an `Array`. One could make a case that this is a bugfix since it was arguably a bug that this method didn't work given that `CodeUnits` is an `AbstractArray` subtype and the other `similar` methods work. If anybody buys that argument, it could be nice to backport this; it came up in some internal code that uses Arrow.jl and JSON3.jl together. --- base/strings/basic.jl | 2 ++ test/strings/basic.jl | 1 + 2 files changed, 3 insertions(+) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index a436a5494fa79..90a28e48988ae 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -805,6 +805,8 @@ write(io::IO, s::CodeUnits) = write(io, s.s) cconvert(::Type{Ptr{T}}, s::CodeUnits{T}) where {T} = cconvert(Ptr{T}, s.s) cconvert(::Type{Ptr{Int8}}, s::CodeUnits{UInt8}) = cconvert(Ptr{Int8}, s.s) +similar(::Type{<:CodeUnits{T}}, dims::Dims) where {T} = similar(Array{T}, dims) + """ codeunits(s::AbstractString) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 60d24280efe6b..214a14ed2443f 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -1096,6 +1096,7 @@ let s = "∀x∃y", u = codeunits(s) @test_throws Base.CanonicalIndexError (u[1] = 0x00) @test collect(u) == b"∀x∃y" @test Base.elsize(u) == Base.elsize(typeof(u)) == 1 + @test similar(typeof(u), 3) isa Vector{UInt8} end @testset "issue #24388" begin From 2aed6484166a49125f43f830660f3f44e7bc9804 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Wed, 2 Jul 2025 15:41:31 -0400 Subject: [PATCH 477/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=20e3d456127=20to=20109eaea66=20(#58858)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: master Julia branch: master Old commit: e3d456127 New commit: 109eaea66 Julia version: 1.13.0-DEV Pkg version: 1.13.0 Bump invoked by: @KristofferC Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/e3d4561272fc029e9a5f940fe101ba4570fa875d...109eaea66a0adb0ad8fa497e64913eadc2248ad1 ``` $ git log --oneline e3d456127..109eaea66 109eaea66 Various app improvements (#4278) 25c2390ed feat(apps): Add support for multiple apps per package via submodules (#4277) c78b40b35 copy the app project instead of wrapping it (#4276) d2e61025b Fix leading whitespace in REPL commands with comma-separated packages (#4274) e02bcabd7 Registry: Properly pass down `depot` (#4268) e9a055240 fix what project file to look at when package without path but with a subdir is devved by name (#4271) 8b1f0b9ff prompt for confirmation before removing compat entry (#4254) eefbef649 feat(errors): Improve error message for incorrect package UUID (#4270) 4d1c6b0a3 explain no reg installed when no reg installed (#4261) ``` Co-authored-by: KristofferC <1282691+KristofferC@users.noreply.github.com> --- .../Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 | 1 + .../Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 | 1 + .../Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 | 1 - .../Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 create mode 100644 deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 diff --git a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 new file mode 100644 index 0000000000000..227a42db30a37 --- /dev/null +++ b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 @@ -0,0 +1 @@ +7cf1b30c6f32ee26f9cc4fc103d55f9f diff --git a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 new file mode 100644 index 0000000000000..1c5513aa26d5f --- /dev/null +++ b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 @@ -0,0 +1 @@ +9840e88a43e48cdac1ae82e5f50d37324ee837bf6776dac615b1ec94f021852f1124ea94b9cb26a7a6bd463d692707a3d554f8baedb8b04a479b2574ff1edb4e diff --git a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 deleted file mode 100644 index bd80c0cbd8f59..0000000000000 --- a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3a5ec13a2d262404f9112a6a14ebd06d diff --git a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 b/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 deleted file mode 100644 index 1776b08ff133a..0000000000000 --- a/deps/checksums/Pkg-e3d4561272fc029e9a5f940fe101ba4570fa875d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -fe9db6d3bbdb3d4a19e8ecf3833a1f315fdbe48d1437d2b2a12052183012da67380db0fc1842e0112007e4d80748eb314320d45526e9b8ab7fe873e565371605 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 19c3fcc398d62..fca848d3e16c3 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = e3d4561272fc029e9a5f940fe101ba4570fa875d +PKG_SHA1 = 109eaea66a0adb0ad8fa497e64913eadc2248ad1 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 811150ce88f3860ec508b5d92cef52c8db6632ca Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 2 Jul 2025 22:37:24 +0200 Subject: [PATCH 478/662] fix a few tiny JET linter issues (#58869) --- base/loading.jl | 3 ++- base/toml_parser.jl | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index e204a8cb01d50..72a9898ca4784 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -184,8 +184,8 @@ end const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9']) function slug(x::UInt32, p::Int) - y::UInt32 = x sprint(sizehint=p) do io + y = x n = length(slug_chars) for i = 1:p y, d = divrem(y, n) @@ -1061,6 +1061,7 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No for (name, entries) in d entries = entries::Vector{Any} for entry in entries + entry = entry::Dict{String, Any} uuid = get(entry, "uuid", nothing)::Union{Nothing, String} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid diff --git a/base/toml_parser.jl b/base/toml_parser.jl index 26c690253cb8a..bf13fc2b0617a 100644 --- a/base/toml_parser.jl +++ b/base/toml_parser.jl @@ -258,10 +258,10 @@ end mutable struct ParserError <: Exception type::ErrorType - # Arbitrary data to store at the + # Data to store at the # call site to be used when formatting # the error - data + data::Union{Char, Nothing} # These are filled in before returning from parse function str ::Union{String, Nothing} @@ -276,7 +276,7 @@ ParserError(type) = ParserError(type, nothing) # Defining these below can be useful when debugging code that erroneously returns a # ParserError because you get a stacktrace to where the ParserError was created #ParserError(type) = error(type) -#ParserError(type, data) = error(type,data) +#ParserError(type, data) = error(type, data) # Many functions return either a T or a ParserError const Err{T} = Union{T, ParserError} @@ -284,7 +284,7 @@ const Err{T} = Union{T, ParserError} function format_error_message_for_err_type(error::ParserError) msg = err_message[error.type] if error.type == ErrInvalidBareKeyCharacter - c_escaped = escape_string(string(error.data)::String) + c_escaped = escape_string(string(error.data::Char)) msg *= ": '$c_escaped'" end return msg From 903955503f99ea5235ffef6a638e43593fb4a950 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:02:30 -0700 Subject: [PATCH 479/662] Fix data race in jl_new_module__ (#58880) Use an atomic fetch and add to fix a data race in `Module()` identified by tsan: ``` ./usr/bin/julia -t4,0 --gcthreads=1 -e 'Threads.@threads for i=1:100 Module() end' ================== WARNING: ThreadSanitizer: data race (pid=5575) Write of size 4 at 0xffff9bf9bd28 by thread T9: #0 jl_new_module__ /home/user/c/julia/src/module.c:487:22 (libjulia-internal.so.1.13+0x897d4) #1 jl_new_module_ /home/user/c/julia/src/module.c:527:22 (libjulia-internal.so.1.13+0x897d4) #2 jl_f_new_module /home/user/c/julia/src/module.c:649:22 (libjulia-internal.so.1.13+0x8a968) #3 (0xffff76a21164) #4 (0xffff76a1f074) #5 (0xffff76a1f0c4) #6 _jl_invoke /home/user/c/julia/src/gf.c (libjulia-internal.so.1.13+0x5ea04) #7 ijl_apply_generic /home/user/c/julia/src/gf.c:3892:12 (libjulia-internal.so.1.13+0x5ea04) #8 jl_apply /home/user/c/julia/src/julia.h:2343:12 (libjulia-internal.so.1.13+0x9e4c4) #9 start_task /home/user/c/julia/src/task.c:1249:19 (libjulia-internal.so.1.13+0x9e4c4) Previous write of size 4 at 0xffff9bf9bd28 by thread T10: #0 jl_new_module__ /home/user/c/julia/src/module.c:487:22 (libjulia-internal.so.1.13+0x897d4) #1 jl_new_module_ /home/user/c/julia/src/module.c:527:22 (libjulia-internal.so.1.13+0x897d4) #2 jl_f_new_module /home/user/c/julia/src/module.c:649:22 (libjulia-internal.so.1.13+0x8a968) #3 (0xffff76a21164) #4 (0xffff76a1f074) #5 (0xffff76a1f0c4) #6 _jl_invoke /home/user/c/julia/src/gf.c (libjulia-internal.so.1.13+0x5ea04) #7 ijl_apply_generic /home/user/c/julia/src/gf.c:3892:12 (libjulia-internal.so.1.13+0x5ea04) #8 jl_apply /home/user/c/julia/src/julia.h:2343:12 (libjulia-internal.so.1.13+0x9e4c4) #9 start_task /home/user/c/julia/src/task.c:1249:19 (libjulia-internal.so.1.13+0x9e4c4) Location is global 'jl_new_module__.mcounter' of size 4 at 0xffff9bf9bd28 (libjulia-internal.so.1.13+0x3dbd28) ``` --- src/module.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index b0e3efb92fd5d..69ed05eacf05f 100644 --- a/src/module.c +++ b/src/module.c @@ -482,9 +482,10 @@ static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) m->parent = parent ? parent : m; m->istopmod = 0; m->uuid = uuid_zero; - static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing + static _Atomic(unsigned int) mcounter; // simple counter backup, in case hrtime is not incrementing + unsigned int count = jl_atomic_fetch_add_relaxed(&mcounter, 1); // TODO: this is used for ir decompression and is liable to hash collisions so use more of the bits - m->build_id.lo = bitmix(jl_hrtime() + (++mcounter), jl_rand()); + m->build_id.lo = bitmix(jl_hrtime() + count, jl_rand()); if (!m->build_id.lo) m->build_id.lo++; // build id 0 is invalid m->build_id.hi = ~(uint64_t)0; From e853a4f8bac99c6458cf3e09d95d9bea67b687e7 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Wed, 2 Jul 2025 23:06:00 -0400 Subject: [PATCH 480/662] fix trailing indices stackoverflow in reinterpreted array (#58293) would fix https://github.com/JuliaLang/julia/issues/57170, fix https://github.com/JuliaLang/julia/issues/54623 @nanosoldier `runbenchmarks("array", vs=":master")` --- base/reinterpretarray.jl | 8 ++++---- test/reinterpretarray.jl | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 07ad7325e6aed..a0522877fae68 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -314,13 +314,13 @@ end _maybe_reshape(::IndexSCartesian2, A::ReshapedReinterpretArray, I...) = A # fallbacks -function _getindex(::IndexSCartesian2, A::AbstractArray{T,N}, I::Vararg{Int, N}) where {T,N} +function _getindex(::IndexSCartesian2, A::AbstractArray, I::Vararg{Int, N}) where {N} @_propagate_inbounds_meta - getindex(A, I...) + _getindex(IndexCartesian(), A, I...) end -function _setindex!(::IndexSCartesian2, A::AbstractArray{T,N}, v, I::Vararg{Int, N}) where {T,N} +function _setindex!(::IndexSCartesian2, A::AbstractArray, v, I::Vararg{Int, N}) where {N} @_propagate_inbounds_meta - setindex!(A, v, I...) + _setindex!(IndexCartesian(), A, v, I...) end # fallbacks for array types that use "pass-through" indexing (e.g., `IndexStyle(A) = IndexStyle(parent(A))`) # but which don't handle SCartesianIndex2 diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index 65befefa710a7..ee0ffc4a96ff9 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -325,6 +325,23 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ @test r[goodinds...] == -5 end end + +let a = rand(ComplexF32, 5) + r = reinterpret(reshape, Float32, a) + ref = Array(r) + + @test r[1, :, 1] == ref[1, :] + @test r[1, :, 1, 1, 1] == ref[1, :] + @test r[1, :, UInt8(1)] == ref[1, :] + + r[2, :, 1] .= 0f0 + ref[2, :] .= 0f0 + @test r[2, :, 1] == ref[2, :] + + @test r[4] == ref[4] + @test_throws BoundsError r[1, :, 2] +end + let ar = [(1,2), (3,4)] arr = reinterpret(reshape, Int, ar) @test @inferred(IndexStyle(arr)) == Base.IndexSCartesian2{2}() @@ -612,3 +629,9 @@ let R = reinterpret(reshape, Float32, ComplexF32[1.0f0+2.0f0*im, 4.0f0+3.0f0*im] @test !isassigned(R, 5) @test Array(R)::Matrix{Float32} == [1.0f0 4.0f0; 2.0f0 3.0f0] end + +@testset "issue #54623" begin + x = 0xabcdef01234567 + @test reinterpret(reshape, UInt8, fill(x)) == [0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x00] + @test reinterpret(reshape, UInt8, [x]) == [0x67; 0x45; 0x23; 0x01; 0xef; 0xcd; 0xab; 0x00;;] +end From a4a1a7eee69211b499104da186c4ffe140bd2487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 3 Jul 2025 05:51:52 +0200 Subject: [PATCH 481/662] Add missing module qualifier (#58877) A very simple fix addressing the following bug: ```julia Validation: Error During Test at REPL[61]:1 Got exception outside of a @test #=ERROR showing exception stack=# UndefVarError: `get_ci_mi` not defined in `Base.StackTraces` Suggestion: check for spelling errors or missing imports. Hint: a global variable of this name also exists in Base. - Also declared public in Compiler (loaded but not imported in Main). Stacktrace: [1] show_custom_spec_sig(io::IOContext{IOBuffer}, owner::Any, linfo::Core.CodeInstance, frame::Base.StackTraces.StackFrame) @ Base.StackTraces ./stacktraces.jl:293 [2] show_spec_linfo(io::IOContext{IOBuffer}, frame::Base.StackTraces.StackFrame) @ Base.StackTraces ./stacktraces.jl:278 [3] print_stackframe(io::IOContext{IOBuffer}, i::Int64, frame::Base.StackTraces.StackFrame, n::Int64, ndigits_max::Int64, modulecolor::Symbol; prefix::Nothing) @ Base ./errorshow.jl:786 ``` AFAIK this occurs when printing a stacktrace from a `CodeInstance` that has a non-default owner. --- Compiler/test/AbstractInterpreter.jl | 13 +++++++++++++ base/stacktraces.jl | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Compiler/test/AbstractInterpreter.jl b/Compiler/test/AbstractInterpreter.jl index 6ca5154c571e7..12da527225a7e 100644 --- a/Compiler/test/AbstractInterpreter.jl +++ b/Compiler/test/AbstractInterpreter.jl @@ -548,4 +548,17 @@ let interp = InvokeInterp() mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} ci = Compiler.typeinf_ext_toplevel(interp, mi, source_mode) @test invoke(f, ci, args...) == 2 + + f = error + args = "test" + mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} + ci = Compiler.typeinf_ext_toplevel(interp, mi, source_mode) + result = nothing + try + invoke(f, ci, args...) + catch e + result = sprint(Base.show_backtrace, catch_backtrace()) + end + @test isa(result, String) + @test contains(result, "[1] error(::Char, ::Char, ::Char, ::Char)") end diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 653419d2148fb..58681c99d889b 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -290,7 +290,7 @@ end # Can be extended by compiler packages to customize backtrace display of custom code instance frames function show_custom_spec_sig(io::IO, @nospecialize(owner), linfo::CodeInstance, frame::StackFrame) - mi = get_ci_mi(linfo) + mi = Base.get_ci_mi(linfo) return show_spec_sig(io, mi.def, mi.specTypes) end From 41570e9800c872fc7ad8cf1c74fbbed4236e88c4 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 3 Jul 2025 04:07:16 -0400 Subject: [PATCH 482/662] OpenSSL: Update to 3.5.1 (#58876) Update the stdlib OpenSSL to 3.5.1. This is a candidate for backporting to Julia 1.12 if there is another beta release. --- deps/checksums/openssl | 76 ++++++++++++++--------------- deps/openssl.version | 2 +- stdlib/OpenSSL_jll/Project.toml | 2 +- stdlib/OpenSSL_jll/test/runtests.jl | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/deps/checksums/openssl b/deps/checksums/openssl index 3b41bfa69231d..cca21ccd8c5a5 100644 --- a/deps/checksums/openssl +++ b/deps/checksums/openssl @@ -1,38 +1,38 @@ -OpenSSL.v3.5.0+0.aarch64-apple-darwin.tar.gz/md5/b8dc9909528f769bd9ac56cf2681f387 -OpenSSL.v3.5.0+0.aarch64-apple-darwin.tar.gz/sha512/0d9ea24d8f856c31c8b88afa1de317d13aff1f1f60b309e06e77eea91d195526ec91ed2d077f0dbb75370c17b8875c24d3066e6872bbef04312616e99d0aff3d -OpenSSL.v3.5.0+0.aarch64-linux-gnu.tar.gz/md5/7cf5baeacf4d882b547c229758a9fa9b -OpenSSL.v3.5.0+0.aarch64-linux-gnu.tar.gz/sha512/726ceee82379e667a65abe27c482d3b57e611c630d82b1314f6d385f0f2e8256835ef707c2e015f9204d563d7ee469bed2dee88d245af81dcde2af3b8331b19c -OpenSSL.v3.5.0+0.aarch64-linux-musl.tar.gz/md5/4601e56eaed365548203752a19f4f8e8 -OpenSSL.v3.5.0+0.aarch64-linux-musl.tar.gz/sha512/da349081850d47b9393665c4365787c26f61471362475c2acd3c8205063d09a785f7b6c836ba6793e880440115b19e85821b4d1938e57dafea0cabb45048a70b -OpenSSL.v3.5.0+0.aarch64-unknown-freebsd.tar.gz/md5/6a9e78436727e67af2f537170e18445e -OpenSSL.v3.5.0+0.aarch64-unknown-freebsd.tar.gz/sha512/4dc2f7a39f17255871773d10ed1b74de5c908af0f7a4bd3f94fd71bc12480fd4cdee0bd859a154328218935f004eee20359dacc353e366c47ed890229a579fc4 -OpenSSL.v3.5.0+0.armv6l-linux-gnueabihf.tar.gz/md5/5c751092c27910a48cab31f87700fe19 -OpenSSL.v3.5.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/b44e2356f719549dd831745963b8c74346be173d176ca15ab2ee6f4a1ec7e105086d89115cb76831a3251eb67bf7c5ff5cba3a03fd4614a3501af235a8e03beb -OpenSSL.v3.5.0+0.armv6l-linux-musleabihf.tar.gz/md5/fc05f9645ff000b21e46951f16833fb0 -OpenSSL.v3.5.0+0.armv6l-linux-musleabihf.tar.gz/sha512/8c960294fe542ab9d9ae7dc283c0c30621f348ff8011a9a47f38c1460234b3b128011426c3e5d0cb6c9b02fbee261b7b264d0b0c55bdf3be2a2cd5bdd210d71d -OpenSSL.v3.5.0+0.armv7l-linux-gnueabihf.tar.gz/md5/8928d47a0f549d15240eb934caddf599 -OpenSSL.v3.5.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/4b5dbfb3a4ea4ebe6510cbe63da2de0bb3762a0fc98946acbb059e9a92791ac65a3519577250dcb571fa07f29be182f165a5d4fa05fc96b60270441adab30e74 -OpenSSL.v3.5.0+0.armv7l-linux-musleabihf.tar.gz/md5/1e4c11043d05bea0fcdbf92525152c51 -OpenSSL.v3.5.0+0.armv7l-linux-musleabihf.tar.gz/sha512/e9514cd0c3a8c3659ff87d505490ca3011a65800222b21e4f932bc2a80fb38bb11de1d13925c3a6313f6bea1c2baf35b38b3db18193ac11ec42eb434edee3418 -OpenSSL.v3.5.0+0.i686-linux-gnu.tar.gz/md5/ee699f302edd1f7677baa565ae631c74 -OpenSSL.v3.5.0+0.i686-linux-gnu.tar.gz/sha512/16dd396b192b4ca23d1fad54d130a92ef43a36259482cd3b276001d084711ef8674dcd167c9f832f5989d6197889af69d2ae6bcef3e6b9f538066bf347c89584 -OpenSSL.v3.5.0+0.i686-linux-musl.tar.gz/md5/e6ffea118acb68d39ccb13df51e15125 -OpenSSL.v3.5.0+0.i686-linux-musl.tar.gz/sha512/44335dcaf144d388bd47dd80db08862f4cff1d5a90861f34001127df74d0d16babedbe0ffd02ab398bddd17ecda605f433a940b3cc5159947cb54810a564b0df -OpenSSL.v3.5.0+0.i686-w64-mingw32.tar.gz/md5/8ef4284ac47a6b45f8c5b01d203ae668 -OpenSSL.v3.5.0+0.i686-w64-mingw32.tar.gz/sha512/d7ea8c94d54a139631f2710cb2c47c0387b694e60dc7afddbca3c6705e17d25ec8958a84b4424edd1ea27d6d1c78457fbacd92f7634345f4ccc1a81cf242c28f -OpenSSL.v3.5.0+0.powerpc64le-linux-gnu.tar.gz/md5/21674471f2a3352ede9aef3454381edd -OpenSSL.v3.5.0+0.powerpc64le-linux-gnu.tar.gz/sha512/5615d438db7c3e97dc422b260b3152cd89a2412b7b9b5d7cea36b0ce471fbd3f1a2e8a9d77f399e257f5c38b8b5dfc256acfbdbe2645ba47b89c177dadd066e9 -OpenSSL.v3.5.0+0.riscv64-linux-gnu.tar.gz/md5/7761384fd5991eb56286f24c9a0fbdba -OpenSSL.v3.5.0+0.riscv64-linux-gnu.tar.gz/sha512/e63d5f7ddc368f4cdb03c299361faef7274930c622404907c3560eb04e6110f851b9a201b402bb6e52fdafe64988f909c209f659f84ba77957eb45a933c8baf1 -OpenSSL.v3.5.0+0.x86_64-apple-darwin.tar.gz/md5/a970728a9aa6f25d56db7e43e7b0cae2 -OpenSSL.v3.5.0+0.x86_64-apple-darwin.tar.gz/sha512/8ab5b2dd90914e193d1f7689c8560228d03cb6ee79fd43a48ae9339b61274fea0557a2bf3a7ae4ce4d4b51630aede55d6d6e860f263e1ffc0bfd6141367a9514 -OpenSSL.v3.5.0+0.x86_64-linux-gnu.tar.gz/md5/4530c0e1791b0eaec99b68f2967a3c2f -OpenSSL.v3.5.0+0.x86_64-linux-gnu.tar.gz/sha512/ba952738be38f52ebc23f48c52c12c1bec9c8b81416264612da21ca21f23604c8e59bf49f73d4b80256ea17b6b662179620deadb8660be98d8ad5ed57e346394 -OpenSSL.v3.5.0+0.x86_64-linux-musl.tar.gz/md5/eb49cefbb938d80198dbab90e1ad9108 -OpenSSL.v3.5.0+0.x86_64-linux-musl.tar.gz/sha512/f038e9bd950e4472cdd82b0c39aebbfd60e75cdf24fd8408d39e4db0793813c9d30471d1ca8d112b0bb4049f18f8fb36b4c3069dfce61032dc73cb6568852b77 -OpenSSL.v3.5.0+0.x86_64-unknown-freebsd.tar.gz/md5/25023844dae8c7d326620b1f9e730a07 -OpenSSL.v3.5.0+0.x86_64-unknown-freebsd.tar.gz/sha512/e38f1f7c452903a09b3f0127e377d5e46e538903f9a58076e53dfc53883b2423463d3fdcf13dc961516965b6dbc2d289bfbfa1027f8c3110a61bdee060bccf73 -OpenSSL.v3.5.0+0.x86_64-w64-mingw32.tar.gz/md5/a73f5220598dfc5e71e1eee6b26f7a27 -OpenSSL.v3.5.0+0.x86_64-w64-mingw32.tar.gz/sha512/c028527230b6e9e675b7e22a21997e5d032e1099dd1f3437c6e764b7967fd0196d4cb46d66b36f2f6ddeb8200f445aa8d6a7a61f7be61288ee5e0e510b5800f8 -openssl-3.5.0.tar.gz/md5/51da7d2bdf7f4f508cb024f562eb9b03 -openssl-3.5.0.tar.gz/sha512/39cc80e2843a2ee30f3f5de25cd9d0f759ad8de71b0b39f5a679afaaa74f4eb58d285ae50e29e4a27b139b49343ac91d1f05478f96fb0c6b150f16d7b634676f +OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/md5/b19522093c25c50685002ad48933a835 +OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/sha512/afa6363f8396deac5131f5efbe92d5b60f4d6982d279d63b5847e80ac4717d89e32edcc9bc7a5fbaab95e03908a6e3e9b386a3931effb0a7163b947b38ed2cd5 +OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/md5/60af2cb22b7d5f4fddd94bd196f86ad2 +OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/sha512/3d384f5da4be3af848b47f48f2438dbda8cdb228b8569d01bd4fbd6feea9f494ecafd3cab6e7b0bbed596746aa2614826971133a2b6dea02836c0904ce870760 +OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/md5/5f96641ec5256a547e03cd6028892a50 +OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/sha512/668c08f2a08f9d65b2e5c1ca4db8f74932995d0fa97c4737a2d9cedb3548482f85fddd478fad37325e2d48f76259fd8f7e003d31fc2a9ecfdb88c4748f90e1d6 +OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/md5/377bd17ae448f4394b3100b290602f35 +OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/sha512/f989a15062b47f059086d4dc8fd53af00717ca622ef8c475a11f6e62a29d8ec4a80159d93a683e8729da66c4bda4c46b7647754dc996ed2ff5635cbbdaf702aa +OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/md5/83bcc0b545bea092a0a5de9e64cbcbf1 +OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/c589119945ff6c1341bc229a2e61096c630288f7d483ea9538202915f8ee1a9e26cd53efc479f1e92a83a75aa6c7453ceba446832678ffed340a4bec13fefbfc +OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/md5/3b2e34e506960259dbb40a36fed26ffe +OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/sha512/735f22fe1202818f64f369471292bb8fdf8cf1f3395d01e70ddf8f65efc5430aec54a63fe950e52625f2c8a5dbd59ed0175f088144af8d99c7db1967ed0e5aeb +OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/md5/aedb37bde1b3fad428987745dc1dd095 +OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/1919823df3c0de01c59a5f9cf42b912a1d56fe7de903c4e7cbcd54603760783a99fe34cd1c108e954d5fe41502c1791b803d67742d70abae64d316c3656b7280 +OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/md5/573a752ca28fd62644208a4c0b32eaa4 +OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/sha512/3385b170973a9e50919981e66e591077479ae7561368941f018aca6f42c86b3d01aa1d9896d4d5f6deb69760fa42f535f6aaa076b75a15f207328ba6f0a32660 +OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/md5/fcdb2ab108900c412abf154a6cbd46e7 +OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/sha512/b558e6c23809f702a7388dba7031a9df577e1a2eb1ca86b7cf0dcd9809973dff1c9b56d4a09c315b17dcc9860e7f89604513a2d022117d9145f2bc81befa094b +OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/md5/0192e44a52d9518d712db58019ace62c +OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/sha512/fe9740850e6eb32eb539d16303b39d9ad1d3e8cc2e5a742304013890a0e1e8af93e131a5393c3c817b5723680425947d6954455dd715cc83589fd097c517b5c2 +OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/md5/51b5546301f8c474bcc9c97b625df2c1 +OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/sha512/47874ce005e6944f3a4d482f3edf44bcaa3724338230d68fff22c484c0620fe857a11bdc515ef9154522a2884f64bacadfd1fddb1430a45c7722a6a4799107f6 +OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/md5/1aeaa0660006b4b8c13cd1cb45b2acfc +OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/sha512/dff025feb0d1ae96a7c33f1beff5e6f76d5a85833314470f59d75bf791e90363145ae79f3ed82c5c40e36416b75fa9deb5807911c8133fe11f31b4705737f0bc +OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/md5/160065eb12650c504fd40a25e4bae2ba +OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/sha512/68951cf98c4eb778d372e239d14497405e6161461a623135a5254c3fd65bc3a12fe3df1ecce88909cb05dc29104b5b18caafea115799c5abf2505afe75be3207 +OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/md5/7e5903d1d051de70a93a9b801ce274db +OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/sha512/729b33cc208b8646394bcf0029a276ad41cf2e9d44737dbc1e15dca794cc55a52e2b782b0c72ef57b5580b84a93b25133788424f1b16ef2b784d409bca598150 +OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/md5/9cee745524f41dc21af2f460ac2f1293 +OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/sha512/6949c7f19b7919073542af903019ec0d8fd5172450141a3f69f0c325f0c5cc19618f1959b96380719538c5a1a5a388165a0db8e6eab041d0a5874a627820212b +OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/md5/e55565c84e5cff597ea490e02c559d1a +OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/sha512/c27930401c72b6be94ba7717f7b3be0025b09138e146f3d2a761e805308ee51f4ca871459941448e102f64b0e3c1aa479f39ee77f3234321890fa7105418ed44 +OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/md5/276d97e2d573977727ca8d2113335fac +OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/sha512/71f72a82c590542928660f004f38f84ea335b303e7da53578d712994fff9e84a3b69fc836c08d03dd8310d17c9edc63e5f6975e6d26e67124124763633ab1b59 +OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/md5/cebdbbf8a8a301e332d75c46dcdb1af0 +OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/sha512/0e141d7317ac8f5c43d1ecc9d161b00345f99145af785d7554f750b5787ea69969f785e7a1305059137271d26450835c25dd126bf9e5aef2cdf7dcbbdebb6911 +openssl-3.5.1.tar.gz/md5/562a4e8d14ee5272f677a754b9c1ca5c +openssl-3.5.1.tar.gz/sha512/0fa152ae59ab5ea066319de039dfb1d24cbb247172d7512feb5dd920db3740f219d76b0195ea562f84fe5eae36c23772302eddfbb3509df13761452b4dafb9d3 diff --git a/deps/openssl.version b/deps/openssl.version index 49c463aad1565..2313ae5ffe116 100644 --- a/deps/openssl.version +++ b/deps/openssl.version @@ -3,4 +3,4 @@ OPENSSL_JLL_NAME := OpenSSL ## source build -OPENSSL_VER := 3.5.0 +OPENSSL_VER := 3.5.1 diff --git a/stdlib/OpenSSL_jll/Project.toml b/stdlib/OpenSSL_jll/Project.toml index 28ecf86381213..d11c3b25a6922 100644 --- a/stdlib/OpenSSL_jll/Project.toml +++ b/stdlib/OpenSSL_jll/Project.toml @@ -1,6 +1,6 @@ name = "OpenSSL_jll" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.0+0" +version = "3.5.1+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/OpenSSL_jll/test/runtests.jl b/stdlib/OpenSSL_jll/test/runtests.jl index e5ae938b68311..9ef99d57134e0 100644 --- a/stdlib/OpenSSL_jll/test/runtests.jl +++ b/stdlib/OpenSSL_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, OpenSSL_jll major = ccall((:OPENSSL_version_major, libcrypto), Cuint, ()) minor = ccall((:OPENSSL_version_minor, libcrypto), Cuint, ()) patch = ccall((:OPENSSL_version_patch, libcrypto), Cuint, ()) - @test VersionNumber(major, minor, patch) == v"3.5.0" + @test VersionNumber(major, minor, patch) == v"3.5.1" end From 7a5c4b5214c6a8f835cc78c48e872c0806b0f487 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Thu, 3 Jul 2025 11:03:37 -0400 Subject: [PATCH 483/662] `setindex!(::ReinterpretArray, v)` needs to convert before reinterpreting (#58867) Found in https://github.com/JuliaLang/julia/pull/58814#discussion_r2169155093. Previously, in a very limited situation (a zero-dimensional reinterpret that reinterprets between primitive types that was setindex!'ed with zero indices), we omitted the `convert`. I believe this was an unintentional oversight, and hopefully nobody is depending on this behavior. --- base/reinterpretarray.jl | 2 +- test/reinterpretarray.jl | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index a0522877fae68..e1ee16be367b2 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -535,7 +535,7 @@ end @propagate_inbounds function setindex!(a::NonReshapedReinterpretArray{T,0,S}, v) where {T,S} if isprimitivetype(S) && isprimitivetype(T) - a.parent[] = reinterpret(S, v) + a.parent[] = reinterpret(S, convert(T, v)::T) return a end setindex!(a, v, firstindex(a)) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index ee0ffc4a96ff9..608a86c15d916 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -69,6 +69,44 @@ test_many_wrappers(B) do _B @test reinterpret(reshape, Int64, _B) == [5 7 9; 6 8 10] end +@testset "setindex! converts before reinterpreting" begin + for dims in ((), 1) + z = reinterpret(UInt64, fill(1.0, dims)) + @test z[] == z[1] == 0x3ff0000000000000 + z[] = Int32(1)//Int32(1) + @test z[] == z[1] == 0x0000000000000001 + z[1] = Int32(2)//Int32(1) + @test z[] == z[1] == 0x0000000000000002 + z[1] = 3//1 + @test z[] == z[1] == 0x0000000000000003 + @test_throws InexactError z[] = 3//2 + @test_throws InexactError z[] = 1.5 + @test_throws InexactError z[1] = 3//2 + @test_throws InexactError z[1] = 1.5 + + z = reinterpret(UInt64, fill(Int32(16)//Int32(1), dims)) + @test z[] == z[1] == 0x0000000100000010 + z[] = Int32(1)//Int32(1) + @test z[] == z[1] == 0x0000000000000001 + z[1] = Int32(2)//Int32(1) + @test z[] == z[1] == 0x0000000000000002 + z[1] = 3//1 + @test z[] == z[1] == 0x0000000000000003 + @test_throws InexactError z[] = 3//2 + @test_throws InexactError z[] = 1.5 + @test_throws InexactError z[1] = 3//2 + @test_throws InexactError z[1] = 1.5 + + z = reinterpret(Missing, fill(nothing, dims)) + @test z[] === missing + @test z[1] === missing + @test_throws "cannot convert" z[] = nothing + @test_throws "cannot convert" z[1] = nothing + @test z[] === missing + @test z[1] === missing + end +end + # setindex test_many_wrappers((A, Ars, B)) do (A, Ars, B) _A, Ar, _B = deepcopy(A), deepcopy(Ars), deepcopy(B) From 32211a62c4eb85bd74a510661afe2d978ac38d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 3 Jul 2025 18:02:28 +0200 Subject: [PATCH 484/662] Support `debuginfo` context option in IRShow for `IRCode`/`IncrementalCompact` (#58642) This allows us to get complete source information during printing for `IRCode` and `IncrementalCompact`, same as we do by default with `CodeInfo`. The user previously had to do: ```julia Compiler.IRShow.show_ir(stdout, ir, Compiler.IRShow.default_config(ir; verbose_linetable=true)) ``` and now, they only need to do: ```julia show(IOContext(stdout, :debuginfo => :source), ir) ``` --- Compiler/src/ssair/show.jl | 61 ++++++++++++++++++++++++++++++-------- test/show.jl | 22 +++++++------- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index de58018866274..dff92e4a90655 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -200,7 +200,7 @@ end end """ - Compute line number annotations for an IRCode + Compute line number annotations for an IRCode or CodeInfo. This functions compute three sets of annotations for each IR line. Take the following example (taken from `@code_typed sin(1.0)`): @@ -259,7 +259,7 @@ to catch up and print the intermediate scopes. Which scope is printed is indicat by the indentation of the method name and by an increased thickness of the appropriate line for the scope. """ -function compute_ir_line_annotations(code::IRCode) +function compute_ir_line_annotations(code::Union{IRCode,CodeInfo}) loc_annotations = String[] loc_methods = String[] loc_lineno = String[] @@ -269,7 +269,8 @@ function compute_ir_line_annotations(code::IRCode) last_printed_depth = 0 debuginfo = code.debuginfo def = :var"unknown scope" - for idx in 1:length(code.stmts) + n = isa(code, IRCode) ? length(code.stmts) : length(code.code) + for idx in 1:n buf = IOBuffer() print(buf, "│") stack = buildLineInfoNode(debuginfo, def, idx) @@ -833,7 +834,7 @@ function new_nodes_iter(compact::IncrementalCompact) end # print only line numbers on the left, some of the method names and nesting depth on the right -function inline_linfo_printer(code::IRCode) +function inline_linfo_printer(code::Union{IRCode,CodeInfo}) loc_annotations, loc_methods, loc_lineno = compute_ir_line_annotations(code) max_loc_width = maximum(length, loc_annotations) max_lineno_width = maximum(length, loc_lineno) @@ -902,12 +903,15 @@ function stmts_used(::IO, code::CodeInfo) return used end -function default_config(code::IRCode; verbose_linetable=false) - return IRShowConfig(verbose_linetable ? statementidx_lineinfo_printer(code) - : inline_linfo_printer(code); - bb_color=:normal) +function default_config(code::IRCode; debuginfo = :source_inline) + return IRShowConfig(get_debuginfo_printer(code, debuginfo); bb_color=:normal) +end +default_config(code::CodeInfo; debuginfo = :source) = IRShowConfig(get_debuginfo_printer(code, debuginfo)) +function default_config(io::IO, src) + debuginfo = get(io, :debuginfo, nothing) + debuginfo !== nothing && return default_config(src; debuginfo) + return default_config(src) end -default_config(code::CodeInfo) = IRShowConfig(statementidx_lineinfo_printer(code)) function show_ir_stmts(io::IO, ir::Union{IRCode, CodeInfo, IncrementalCompact}, inds, config::IRShowConfig, sptypes::Vector{VarState}, used::BitSet, cfg::CFG, bb_idx::Int; pop_new_node! = Returns(nothing)) @@ -927,8 +931,7 @@ function finish_show_ir(io::IO, cfg::CFG, config::IRShowConfig) return nothing end -function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(ir); - pop_new_node! = new_nodes_iter(ir)) +function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(io, ir); pop_new_node! = new_nodes_iter(ir)) used = stmts_used(io, ir) cfg = ir.cfg maxssaid = length(ir.stmts) + length(ir.new_nodes) @@ -938,7 +941,7 @@ function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(ir); finish_show_ir(io, cfg, config) end -function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(ci); +function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(io, ci); pop_new_node! = Returns(nothing)) used = stmts_used(io, ci) cfg = compute_basic_blocks(ci.code) @@ -952,7 +955,7 @@ function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(ci); finish_show_ir(io, cfg, config) end -function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=default_config(compact.ir)) +function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=default_config(io, compact.ir)) cfg = compact.ir.cfg @@ -1154,3 +1157,35 @@ const __debuginfo = Dict{Symbol, Any}( ) const default_debuginfo = Ref{Symbol}(:none) debuginfo(sym) = sym === :default ? default_debuginfo[] : sym + +const __debuginfo = Dict{Symbol, Any}( + # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information + :source => src -> statementidx_lineinfo_printer(src), + :source_inline => src -> inline_linfo_printer(src), + # :oneliner => src -> statementidx_lineinfo_printer(PartialLineInfoPrinter, src), + :none => src -> lineinfo_disabled, + ) + +const debuginfo_modes = [:none, :source, :source_inline] +@assert Set(debuginfo_modes) == Set(keys(__debuginfo)) + +function validate_debuginfo_mode(mode::Symbol) + in(mode, debuginfo_modes) && return true + throw(ArgumentError("`debuginfo` must be one of the following: $(join([repr(mode) for mode in debuginfo_modes], ", "))")) +end + +const default_debuginfo_mode = Ref{Symbol}(:none) +function expand_debuginfo_mode(mode::Symbol, default = default_debuginfo_mode[]) + if mode === :default + mode = default + end + validate_debuginfo_mode(mode) + return mode +end + +function get_debuginfo_printer(mode::Symbol) + mode = expand_debuginfo_mode(mode) + return __debuginfo[mode] +end + +get_debuginfo_printer(src, mode::Symbol) = get_debuginfo_printer(mode)(src) diff --git a/test/show.jl b/test/show.jl index 7922215ee789c..60d0538e71a07 100644 --- a/test/show.jl +++ b/test/show.jl @@ -8,6 +8,8 @@ include("testenv.jl") replstr(x, kv::Pair...) = sprint((io,x) -> show(IOContext(io, :limit => true, :displaysize => (24, 80), kv...), MIME("text/plain"), x), x) showstr(x, kv::Pair...) = sprint((io,x) -> show(IOContext(io, :limit => true, :displaysize => (24, 80), kv...), x), x) +const IRShow = Base.Compiler.IRShow + @testset "IOContext" begin io = IOBuffer() ioc = IOContext(io) @@ -2161,7 +2163,7 @@ end function compute_annotations(f, types) src = code_typed(f, types, debuginfo=:source)[1][1] ir = Core.Compiler.inflate_ir(src) - la, lb, ll = Base.IRShow.compute_ir_line_annotations(ir) + la, lb, ll = IRShow.compute_ir_line_annotations(ir) max_loc_method = maximum(length(s) for s in la) return join((strip(string(a, " "^(max_loc_method-length(a)), b)) for (a, b) in zip(la, lb)), '\n') end @@ -2216,6 +2218,8 @@ eval(Meta._parse_string("""function my_fun28173(x) return y end""", "a"^80, 1, 1, :statement)[1]) # use parse to control the line numbers let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1] + @test_throws "must be one of the following" sprint(IRShow.show_ir, src; context = :debuginfo => :_) + @test !contains(sprint(IRShow.show_ir, src; context = :debuginfo => :source_inline), "a"^80) ir = Core.Compiler.inflate_ir(src) src.debuginfo = Core.DebugInfo(src.debuginfo.def) # IRCode printing defaults to incomplete line info printing, so turn it off completely for CodeInfo too let source_slotnames = String["my_fun28173", "x"], @@ -2245,18 +2249,16 @@ let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1] @test pop!(lines2) == "18 │ \$(QuoteNode(3))" @test lines1 == lines2 - # verbose linetable - io = IOBuffer() - Base.IRShow.show_ir(io, ir, Base.IRShow.default_config(ir; verbose_linetable=true)) - seekstart(io) - @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), eachline(io)) == 10 + # debuginfo = :source + output = sprint(Base.IRShow.show_ir, ir, Base.IRShow.default_config(ir; debuginfo=:source)) + @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), split(output, '\n')) == 10 + @test output == sprint(show, ir; context = :debuginfo => :source) + @test output != sprint(show, ir) + @test_throws "must be one of the following" sprint(show, ir; context = :debuginfo => :_) # Test that a bad :invoke doesn't cause an error during printing Core.Compiler.insert_node!(ir, 1, Core.Compiler.NewInstruction(Expr(:invoke, nothing, sin), Any), false) - io = IOBuffer() - Base.IRShow.show_ir(io, ir) - seekstart(io) - @test contains(String(take!(io)), "Expr(:invoke, nothing") + @test contains(string(ir), "Expr(:invoke, nothing") end # Verify that extra instructions at the end of the IR From ed1fd390b59b7554c37bfe8f89b8c1f2ca654ead Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 3 Jul 2025 15:54:27 -0400 Subject: [PATCH 485/662] Add offset in `hvncat` dimension calculation to fix issue with 0-length elements in first dimension (#58881) --- base/abstractarray.jl | 13 ++++++++++++- test/abstractarray.jl | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index df0dc88efe347..adc104dadb90b 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -2567,16 +2567,23 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as end # discover number of rows or columns + # d1 dimension is increased by 1 to appropriately handle 0-length arrays for i ∈ 1:dims[d1] outdims[d1] += cat_size(as[i], d1) end + # adjustment to handle 0-length arrays + first_dim_zero = outdims[d1] == 0 + if first_dim_zero + outdims[d1] = dims[d1] + end + currentdims = zeros(Int, N) blockcount = 0 elementcount = 0 for i ∈ eachindex(as) elementcount += cat_length(as[i]) - currentdims[d1] += cat_size(as[i], d1) + currentdims[d1] += first_dim_zero ? 1 : cat_size(as[i], d1) if currentdims[d1] == outdims[d1] currentdims[d1] = 0 for d ∈ (d2, 3:N...) @@ -2604,6 +2611,10 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as throw(DimensionMismatch("argument $i has too many elements along axis $d1")) end end + # restore 0-length adjustment + if first_dim_zero + outdims[d1] = 0 + end outlen = prod(outdims) elementcount == outlen || diff --git a/test/abstractarray.jl b/test/abstractarray.jl index ad821855e573a..bff6e28f8b19b 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -831,6 +831,9 @@ function test_cat(::Type{TestAbstractArray}) r = rand(Float32, 56, 56, 64, 1); f(r) = cat(r, r, dims=(3,)) @inferred f(r); + + #58866 - ensure proper dimension calculation for 0-dimension elements + @test [zeros(1, 0) zeros(1,0); zeros(0,0) zeros(0, 0)] == Matrix{Float64}(undef, 1, 0) end function test_ind2sub(::Type{TestAbstractArray}) @@ -1743,6 +1746,9 @@ using Base: typed_hvncat @test ["A";;"B";;"C";;"D"] == ["A" "B" "C" "D"] @test ["A";"B";;"C";"D"] == ["A" "C"; "B" "D"] @test [["A";"B"];;"C";"D"] == ["A" "C"; "B" "D"] + + #58866 - ensure proper dimension calculation for 0-dimension elements + @test [zeros(1, 0) zeros(1,0);;; zeros(0,0) zeros(0, 0)] == Array{Float64, 3}(undef, 1, 0, 0) end @testset "stack" begin From 7c804050dd7e639461f366038902eb8969913089 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Thu, 3 Jul 2025 18:25:24 -0400 Subject: [PATCH 486/662] fix `setindex(::ReinterpretArray,...)` for zero-d arrays (#58868) by copying the way getindex works. Found in https://github.com/JuliaLang/julia/pull/58814#discussion_r2178243259 --------- Co-authored-by: Andy Dienes <51664769+adienes@users.noreply.github.com> --- base/reinterpretarray.jl | 4 ++-- test/reinterpretarray.jl | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index e1ee16be367b2..f1cd9c9e82918 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -412,7 +412,7 @@ end # Convert to full indices here, to avoid needing multiple conversions in # the loop in _getindex_ra inds = _to_subscript_indices(a, i) - isempty(inds) ? _getindex_ra(a, 1, ()) : _getindex_ra(a, inds[1], tail(inds)) + isempty(inds) ? _getindex_ra(a, firstindex(a), ()) : _getindex_ra(a, inds[1], tail(inds)) end @propagate_inbounds function getindex(a::ReshapedReinterpretArray{T,N,S}, ind::SCartesianIndex2) where {T,N,S} @@ -556,7 +556,7 @@ end return _setindex_ra!(a, v, i, ()) end inds = _to_subscript_indices(a, i) - _setindex_ra!(a, v, inds[1], tail(inds)) + isempty(inds) ? _setindex_ra!(a, v, firstindex(a), ()) : _setindex_ra!(a, v, inds[1], tail(inds)) end @propagate_inbounds function setindex!(a::ReshapedReinterpretArray{T,N,S}, v, ind::SCartesianIndex2) where {T,N,S} diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index 608a86c15d916..873c3bc32c993 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -160,6 +160,16 @@ test_many_wrappers(A3) do A3_ @test A3[2,1,2] == 400 end +test_many_wrappers(C) do Cr + r = reinterpret(reshape, Tuple{Int, Int}, Cr) + r[] = (2,2) + @test r[] === (2,2) + r[1] = (3,3) + @test r[1] === (3,3) + r[1,1] = (4,4) + @test r[1,1] === (4,4) +end + # same-size reinterpret where one of the types is non-primitive let a = NTuple{4,UInt8}[(0x01,0x02,0x03,0x04)] test_many_wrappers(a, (identity, wrapper, fcviews)) do a_ From b6123e8ddc8ac3ebcd53b606055cdea70e42fd11 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 4 Jul 2025 10:52:30 +0200 Subject: [PATCH 487/662] add back `to_power_type` to `deprecated.jl` since some packages call it (#58886) Co-authored-by: KristofferC --- base/deprecated.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/deprecated.jl b/base/deprecated.jl index e7ea1e15e7b50..0ee6b6b790837 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -557,4 +557,11 @@ true """ isbindingresolved +# Some packages call this function +function to_power_type(x::Number) + T = promote_type(typeof(x), typeof(x*x)) + convert(T, x) +end +to_power_type(x) = oftype(x*x, x) + # END 1.12 deprecations From c0cc1e1022b780a3b7de6ad33593ef12484e46e2 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Fri, 4 Jul 2025 14:09:57 +0200 Subject: [PATCH 488/662] Pkg: Allow configuring can_fancyprint(io::IO) using IOContext (#58887) --- base/precompilation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index c24026aa2a8ef..c51297ee2a791 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -355,7 +355,7 @@ Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", e import Base: StaleCacheKey -can_fancyprint(io::IO) = io isa Base.TTY && (get(ENV, "CI", nothing) != "true") +can_fancyprint(io::IO) = @something(get(io, :force_fancyprint, nothing), (io isa Base.TTY && (get(ENV, "CI", nothing) != "true"))) function printpkgstyle(io, header, msg; color=:green) printstyled(io, header; color, bold=true) From bac6d9a13888fde0f3ad2f0630151d3db5e3a48e Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 4 Jul 2025 08:37:35 -0500 Subject: [PATCH 489/662] Make `Base.donotdelete` public (#55774) I rely on `Base.donotdelete` in [Chairmarks.jl](https://chairmarks.lilithhafner.com) and I'd like it to be public. I imagine that other benchmarking tools also rely on it. It's been around since 1.8 (see also: #55773) and I think we should commit to keeping it functional for the rest of 1.x. --- NEWS.md | 1 + base/public.jl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 879f68c47bb6a..6a3f67ff063bd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,6 +46,7 @@ New library functions * `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). * Exporting function `fieldindex` to get the index of a struct's field ([#58119]). +* `Base.donotdelete` is now public. It prevents deadcode elemination of its arguments ([#55774]). New library features -------------------- diff --git a/base/public.jl b/base/public.jl index 217d91b615848..bc3d76e86eadc 100644 --- a/base/public.jl +++ b/base/public.jl @@ -125,4 +125,5 @@ public notnothing, runtests, text_colors, - depwarn + depwarn, + donotdelete From 4846c3d938b14d978678b92a0a01e1141c5a5d7b Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Fri, 4 Jul 2025 21:13:52 +0530 Subject: [PATCH 490/662] Add link to video in profiling manual (#58896) --- doc/src/manual/profile.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/src/manual/profile.md b/doc/src/manual/profile.md index 49b58ba9671c2..77f9fb44e2a29 100644 --- a/doc/src/manual/profile.md +++ b/doc/src/manual/profile.md @@ -562,8 +562,7 @@ Passing `sample_rate=1.0` will make it record everything (which is slow); Since Julia 1.11, all allocations should have a type reported. -For more details on how to use this tool, please see the following talk from JuliaCon 2022: -https://www.youtube.com/watch?v=BFvpwC8hEWQ +For more details on how to use this tool, please see [the talk from JuliaCon 2022](https://www.youtube.com/watch?v=BFvpwC8hEWQ). ##### Allocation Profiler Example From 69a2a571e8953c00b8ee304a4e15cb2de0585e59 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Sat, 5 Jul 2025 09:42:03 -0500 Subject: [PATCH 491/662] Stop documenting that `permute!` is "in-place"; it isn't and never has been non-allocating (#58902) --- base/combinatorics.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/base/combinatorics.jl b/base/combinatorics.jl index dac217cd4fb41..5180f830ce187 100644 --- a/base/combinatorics.jl +++ b/base/combinatorics.jl @@ -184,13 +184,12 @@ end """ permute!(v, p) -Permute vector `v` in-place, according to permutation `p`. No checking is done -to verify that `p` is a permutation. +Permute vector `v` according to permutation `p`, storing the result back into `v`. +No checking is done to verify that `p` is a permutation. To return a new permutation, use `v[p]`. This is generally faster than `permute!(v, p)`; it is even faster to write into a pre-allocated output array with `u .= @view v[p]`. -(Even though `permute!` overwrites `v` in-place, it internally requires some allocation -to keep track of which elements have been moved.) +(Even though `permute!` overwrites `v` in-place, it internally requires some allocation.) $(_DOCS_ALIASING_WARNING) From d82ccc1a9390a6e6b275d0852f2d86dac3406312 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Sun, 6 Jul 2025 07:29:04 -0400 Subject: [PATCH 492/662] faster iteration over a `Flatten` of heterogenous iterators (#58522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seems to help in many cases. would fix the precise MWE given in https://github.com/JuliaLang/julia/issues/52552, but does not necessarily fix comprehensively all perf issues of all heterogenous flattens. but, may as well be better when it's possible setup: ``` julia> using BenchmarkTools julia> A = rand(Int, 100000); B = 1:100000; julia> function g(it) s = 0 for i in it s += i end s end ``` before: ``` julia> @btime g($(Iterators.flatten((A, B)))) 12.461 ms (698979 allocations: 18.29 MiB) julia> @btime g($(Iterators.flatten(i for i in (A, B)))) 12.393 ms (698979 allocations: 18.29 MiB) julia> @btime g($(Iterators.flatten([A, B]))) 15.115 ms (999494 allocations: 25.93 MiB) julia> @btime g($(Iterators.flatten((A, Iterators.flatten((A, B)))))) 82.585 ms (2997964 allocations: 106.78 MiB) ``` after: ``` julia> @btime g($(Iterators.flatten((A, B)))) 135.958 μs (2 allocations: 64 bytes) julia> @btime g($(Iterators.flatten(i for i in (A, B)))) 149.500 μs (2 allocations: 64 bytes) julia> @btime g($(Iterators.flatten([A, B]))) 17.130 ms (999498 allocations: 25.93 MiB) julia> @btime g($(Iterators.flatten((A, Iterators.flatten((A, B)))))) 13.716 ms (398983 allocations: 10.67 MiB) ``` --- base/iterators.jl | 52 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/base/iterators.jl b/base/iterators.jl index 53d7b28316ee1..d226fbb57aa52 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -1239,20 +1239,48 @@ flatten_length(f, T) = throw(ArgumentError( length(f::Flatten{I}) where {I} = flatten_length(f, eltype(I)) length(f::Flatten{Tuple{}}) = 0 -@propagate_inbounds function iterate(f::Flatten, state=()) - if state !== () - y = iterate(tail(state)...) - y !== nothing && return (y[1], (state[1], state[2], y[2])) +@propagate_inbounds function iterate(fl::Flatten) + it_result = iterate(fl.it) + it_result === nothing && return nothing + + inner_iterator, next_outer_state = it_result + inner_it_result = iterate(inner_iterator) + + while inner_it_result === nothing + it_result = iterate(fl.it, next_outer_state) + it_result === nothing && return nothing + + inner_iterator, next_outer_state = it_result + inner_it_result = iterate(inner_iterator) end - x = (state === () ? iterate(f.it) : iterate(f.it, state[1])) - x === nothing && return nothing - y = iterate(x[1]) - while y === nothing - x = iterate(f.it, x[2]) - x === nothing && return nothing - y = iterate(x[1]) + + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) +end + +@propagate_inbounds function iterate(fl::Flatten, state) + next_outer_state, inner_iterator, next_inner_state = state + + # try to advance the inner iterator + inner_it_result = iterate(inner_iterator, next_inner_state) + if inner_it_result !== nothing + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) + end + + # advance the outer iterator + while true + outer_it_result = iterate(fl.it, next_outer_state) + outer_it_result === nothing && return nothing + + inner_iterator, next_outer_state = outer_it_result + inner_it_result = iterate(inner_iterator) + + if inner_it_result !== nothing + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) + end end - return y[1], (x[2], x[1], y[2]) end reverse(f::Flatten) = Flatten(reverse(itr) for itr in reverse(f.it)) From 3e2f90fbb8f6b0651f2601d7599c55d4e3efd496 Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Mon, 7 Jul 2025 02:16:59 +0200 Subject: [PATCH 493/662] Make `hypot` docs example more type stable (#58918) --- doc/src/manual/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index a83d16ca8c9b7..b2c72cb648d6b 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -178,7 +178,7 @@ julia> function hypot(x, y) return x*sqrt(1 + r*r) end if y == 0 - return x + return float(x) end r = x/y return y*sqrt(1 + r*r) From 958d7584053e1f1e4db8cf30316ce64cdfa8c080 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Mon, 7 Jul 2025 23:49:16 +0900 Subject: [PATCH 494/662] Markdown: Make `Table`/`LaTeX` objects subtypes of `MarkdownElement` (#58916) These objects satisfy the requirements of the `MarkdownElement` interface (such as implementing `Markdown.plain`), so they should be subtypes of `MarkdownElement`. This is convenient when defining functions for `MarkdownElement` in other packages. --- stdlib/Markdown/src/GitHub/table.jl | 2 +- stdlib/Markdown/src/IPython/IPython.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/Markdown/src/GitHub/table.jl b/stdlib/Markdown/src/GitHub/table.jl index 7c174007a75ba..fefa667fc7f93 100644 --- a/stdlib/Markdown/src/GitHub/table.jl +++ b/stdlib/Markdown/src/GitHub/table.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mutable struct Table +mutable struct Table <: MarkdownElement rows::Vector{Vector{Any}} align::Vector{Symbol} end diff --git a/stdlib/Markdown/src/IPython/IPython.jl b/stdlib/Markdown/src/IPython/IPython.jl index 54b628e768a48..cab4abbf65412 100644 --- a/stdlib/Markdown/src/IPython/IPython.jl +++ b/stdlib/Markdown/src/IPython/IPython.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mutable struct LaTeX +mutable struct LaTeX <: MarkdownElement formula::String end From 04138bfdbd1ba029ffdc7612dce14275a36f583f Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:43:42 -0400 Subject: [PATCH 495/662] Support "Functor-like" `code_typed` invocation (#57911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets you easily inspect IR associated with "Functor-like" methods: ```julia julia> (f::Foo)(offset::Float64) = f.x + f.y + offset julia> code_typed((Foo, Float64)) 1-element Vector{Any}: CodeInfo( 1 ─ %1 = Base.getfield(f, :x)::Int64 │ %2 = Base.getfield(f, :y)::Int64 │ %3 = Base.add_int(%1, %2)::Int64 │ %4 = Base.sitofp(Float64, %3)::Float64 │ %5 = Base.add_float(%4, offset)::Float64 └── return %5 ) => Float64 ``` This is just a small convenience over `code_typed_by_type`, but I'm in support of it (even though it technically changes the meaning of, e.g., `code_typed((1, 2))` which without this PR inspects `(::Tuple{Int,Int})(::Vararg{Any})` We should probably update all of our reflection machinery (`code_llvm`, `code_lowered`, `methodinstance`, etc.) to support this "non-arg0" style as well, but I wanted to open this first to make sure folks like it. --- base/reflection.jl | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/base/reflection.jl b/base/reflection.jl index edc1afc8a6aaf..d6e2d2f0de875 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -211,8 +211,31 @@ julia> code_typed(+, (Float64, Float64)) 1 ─ %1 = Base.add_float(x, y)::Float64 └── return %1 ) => Float64 + +julia> code_typed((typeof(-), Float64, Float64)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = Base.sub_float(x, y)::Float64 +└── return %1 +) => Float64 + +julia> code_typed((Type{Int}, UInt8)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = Core.zext_int(Core.Int64, x)::Int64 +└── return %1 +) => Int64 + +julia> code_typed((Returns{Int64},)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = builtin Base.getfield(obj, :value)::Int64 +└── return %1 +) => Int64 ``` """ +function code_typed end + function code_typed(@nospecialize(f), @nospecialize(types=default_tt(f)); kwargs...) if isa(f, Core.OpaqueClosure) return code_typed_opaque_closure(f, types; kwargs...) @@ -221,6 +244,12 @@ function code_typed(@nospecialize(f), @nospecialize(types=default_tt(f)); kwargs return code_typed_by_type(tt; kwargs...) end +# support 'functor'-like queries, such as `(::Foo)(::Int, ::Int)` via `code_typed((Foo, Int, Int))` +function code_typed(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) + tt = to_tuple_type(argtypes) + return code_typed_by_type(tt; kwargs...) +end + # returns argument tuple type which is supposed to be used for `code_typed` and its family; # if there is a single method this functions returns the method argument signature, # otherwise returns `Tuple` that doesn't match with any signature From 144de95c20a600e1c554847da71706c7696cbf99 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:44:01 -0400 Subject: [PATCH 496/662] IRShow: Print arg0 type when necessary to disambiguate `invoke` (#58893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When invoking any "functor-like", such as a closure: ```julia bar(x) = @noinline ((y)->x+y)(x) ``` our IR printing was not showing the arg0 invoked, even when it is required to determine which MethodInstance this is invoking. Before: ```julia julia> @code_typed optimize=true bar(1) CodeInfo( 1 ─ %1 = %new(var"#bar##2#bar##3"{Int64}, x)::var"#bar##2#bar##3"{Int64} │ %2 = invoke %1(x::Int64)::Int64 └── return %2 ) => Int64 ``` After: ```julia julia> @code_typed optimize=true bar(1) CodeInfo( 1 ─ %1 = %new(var"#bar##2#bar##3"{Int64}, x)::var"#bar##2#bar##3"{Int64} │ %2 = invoke (%1::var"#bar##2#bar##3"{Int64})(x::Int64)::Int64 └── return %2 ) => Int64 ``` --- Compiler/src/ssair/show.jl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index dff92e4a90655..6731054ee8f30 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -105,11 +105,30 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co printstyled(io, "dynamic invoke "; color = :yellow) abi = (ci::Core.MethodInstance).specTypes end - show_unquoted(io, stmt.args[2], indent) - print(io, "(") # XXX: this is wrong if `sig` is not a concretetype method # more correct would be to use `fieldtype(sig, i)`, but that would obscure / discard Varargs information in show sig = abi == Tuple ? Core.svec() : Base.unwrap_unionall(abi).parameters::Core.SimpleVector + f = stmt.args[2] + ft = maybe_argextype(f, code, sptypes) + + # We can elide the type for arg0 if it... + skip_ftype = (length(sig) == 0) # doesn't exist... + skip_ftype = skip_ftype || ( + # ... or, f prints as a user-accessible value... + (f isa GlobalRef) && + # ... and matches the value of the singleton type of the invoked MethodInstance + (singleton_type(ft) === singleton_type(sig[1]) !== nothing) + ) + if skip_ftype + show_unquoted(io, f, indent) + else + print(io, "(") + show_unquoted(io, f, indent) + print(io, "::", sig[1], ")") + end + + # Print the remaining arguments (with type annotations from the invoked MethodInstance) + print(io, "(") print_arg(i) = sprint(; context=io) do io show_unquoted(io, stmt.args[i], indent) if (i - 1) <= length(sig) From 9fc33b99bd3efab36da0dd856f2312bd69d3395b Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:45:27 -0400 Subject: [PATCH 497/662] Support "functors" for code reflection utilities (#58891) As a follow-up to https://github.com/JuliaLang/julia/pull/57911, this updates: - `Base.method_instance` - `Base.method_instances` - `Base.code_ircode` - `Base.code_lowered` - `InteractiveUtils.code_llvm` - `InteractiveUtils.code_native` - `InteractiveUtils.code_warntype` to support "functor" invocations. e.g. `code_llvm((Foo, Int, Int))` which corresponds to `(::Foo)(::Int, ::Int)` --- Compiler/test/codegen.jl | 2 +- base/reflection.jl | 33 ++++++++-- stdlib/InteractiveUtils/src/codeview.jl | 78 ++++++++++++++++-------- stdlib/InteractiveUtils/test/runtests.jl | 23 +++++++ test/rebinding.jl | 2 +- test/reflection.jl | 25 ++++++-- 6 files changed, 125 insertions(+), 38 deletions(-) diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 379d06ede9fef..eb5eae29cdc54 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -22,7 +22,7 @@ end # The tests below assume a certain format and safepoint_on_entry=true breaks that. function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) - d = InteractiveUtils._dump_function(f, t, false, false, raw, dump_module, :att, optimize, :none, false, params) + d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, params) sprint(print, d) end diff --git a/base/reflection.jl b/base/reflection.jl index d6e2d2f0de875..06fe1a2064b16 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -16,7 +16,7 @@ The keyword `debuginfo` controls the amount of code metadata present in the outp Note that an error will be thrown if `types` are not concrete types when `generated` is `true` and any of the corresponding methods are an `@generated` method. """ -function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=true, debuginfo::Symbol=:default) +function code_lowered(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); generated::Bool=true, debuginfo::Symbol=:default) if @isdefined(IRShow) debuginfo = IRShow.debuginfo(debuginfo) elseif debuginfo === :default @@ -28,7 +28,7 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool= world = get_world_counter() world == typemax(UInt) && error("code reflection cannot be used from generated functions") ret = CodeInfo[] - for m in method_instances(f, t, world) + for m in method_instances(argtypes, world) if generated && hasgenerator(m) if may_invoke_generator(m) code = ccall(:jl_code_for_staged, Ref{CodeInfo}, (Any, UInt, Ptr{Cvoid}), m, world, C_NULL) @@ -46,12 +46,17 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool= return ret end +function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=true, debuginfo::Symbol=:default) + tt = signature_type(f, t) + return code_lowered(tt; generated, debuginfo) +end + # for backwards compat const uncompressed_ast = uncompressed_ir const _uncompressed_ast = _uncompressed_ir -function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) - tt = signature_type(f, t) +function method_instances(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}), world::UInt) + tt = to_tuple_type(argtypes) results = Core.MethodInstance[] # this make a better error message than the typeassert that follows world == typemax(UInt) && error("code reflection cannot be used from generated functions") @@ -62,15 +67,26 @@ function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) return results end -function method_instance(@nospecialize(f), @nospecialize(t); - world=Base.get_world_counter(), method_table=nothing) +function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) tt = signature_type(f, t) + return method_instances(tt, world) +end + +function method_instance(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); + world=Base.get_world_counter(), method_table=nothing) + tt = to_tuple_type(argtypes) mi = ccall(:jl_method_lookup_by_tt, Any, (Any, Csize_t, Any), tt, world, method_table) return mi::Union{Nothing, MethodInstance} end +function method_instance(@nospecialize(f), @nospecialize(t); + world=Base.get_world_counter(), method_table=nothing) + tt = signature_type(f, t) + return method_instance(tt; world, method_table) +end + default_debug_info_kind() = unsafe_load(cglobal(:jl_default_debug_info_kind, Cint)) # this type mirrors jl_cgparams_t (documented in julia.h) @@ -431,6 +447,11 @@ function code_ircode(@nospecialize(f), @nospecialize(types = default_tt(f)); kwa return code_ircode_by_type(tt; kwargs...) end +function code_ircode(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) + tt = to_tuple_type(argtypes) + return code_ircode_by_type(tt; kwargs...) +end + """ code_ircode_by_type(types::Type{<:Tuple}; ...) diff --git a/stdlib/InteractiveUtils/src/codeview.jl b/stdlib/InteractiveUtils/src/codeview.jl index 1aa83a19285ff..c8f909096c819 100644 --- a/stdlib/InteractiveUtils/src/codeview.jl +++ b/stdlib/InteractiveUtils/src/codeview.jl @@ -20,6 +20,28 @@ const llstyle = Dict{Symbol, Tuple{Bool, Union{Symbol, Int}}}( :funcname => (false, :light_yellow), ) +struct ArgInfo + oc::Union{Core.OpaqueClosure,Nothing} + tt::Type{<:Tuple} + + # Construct from a function object + argtypes + function ArgInfo(@nospecialize(f), @nospecialize(t)) + if isa(f, Core.Builtin) + throw(ArgumentError("argument is not a generic function")) + elseif f isa Core.OpaqueClosure + return new(f, Base.to_tuple_type(t)) + else + return new(nothing, signature_type(f, t)) + end + end + + # Construct from argtypes (incl. arg0) + function ArgInfo(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}})) + tt = Base.to_tuple_type(argtypes) + return new(nothing, tt) + end +end + function printstyled_ll(io::IO, x, s::Symbol, trailing_spaces="") printstyled(io, x, bold=llstyle[s][1], color=llstyle[s][2]) print(io, trailing_spaces) @@ -143,7 +165,7 @@ See the [`@code_warntype`](@ref man-code-warntype) section in the Performance Ti See also: [`@code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_llvm`](@ref), [`code_native`](@ref). """ -function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); +function code_warntype(io::IO, arginfo::ArgInfo; world=Base.get_world_counter(), interp::Base.Compiler.AbstractInterpreter=Base.Compiler.NativeInterpreter(world), debuginfo::Symbol=:default, optimize::Bool=false, kwargs...) @@ -152,13 +174,14 @@ function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_t debuginfo = Base.IRShow.debuginfo(debuginfo) lineprinter = Base.IRShow.__debuginfo[debuginfo] nargs::Int = 0 - if isa(f, Core.OpaqueClosure) - isa(f.source, Method) && (nargs = f.source.nargs) - print_warntype_codeinfo(io, Base.code_typed_opaque_closure(f, tt)[1]..., nargs; + if arginfo.oc !== nothing + (; oc, tt) = arginfo + isa(oc.source, Method) && (nargs = oc.source.nargs) + print_warntype_codeinfo(io, Base.code_typed_opaque_closure(oc, tt)[1]..., nargs; lineprinter, label_dynamic_calls = optimize) return nothing end - tt = Base.signature_type(f, tt) + tt = arginfo.tt matches = findall(tt, Base.Compiler.method_table(interp)) matches === nothing && Base.raise_match_failure(:code_warntype, tt) for match in matches.matches @@ -176,6 +199,8 @@ function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_t end nothing end +code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) = code_warntype(io, ArgInfo(f, tt); kwargs...) +code_warntype(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_warntype(io, ArgInfo(argtypes); kwargs...) code_warntype(args...; kwargs...) = (@nospecialize; code_warntype(stdout, args...; kwargs...)) using Base: CodegenParams @@ -189,33 +214,30 @@ const OC_MISMATCH_WARNING = # Printing code representations in IR and assembly -function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrapper::Bool, +function _dump_function(arginfo::ArgInfo, native::Bool, wrapper::Bool, raw::Bool, dump_module::Bool, syntax::Symbol, optimize::Bool, debuginfo::Symbol, binary::Bool, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") - if isa(f, Core.Builtin) - throw(ArgumentError("argument is not a generic function")) - end warning = "" # get the MethodInstance for the method match - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing world = Base.get_world_counter() - match = Base._which(signature_type(f, t); world) + match = Base._which(arginfo.tt; world) mi = Base.specialize_method(match) # TODO: use jl_is_cacheable_sig instead of isdispatchtuple isdispatchtuple(mi.specTypes) || (warning = GENERIC_SIG_WARNING) else - world = UInt64(f.world) - tt = Base.to_tuple_type(t) - if !isdefined(f.source, :source) + (; oc, tt) = arginfo + world = UInt64(oc.world) + if !isdefined(oc.source, :source) # OC was constructed from inferred source. There's only one # specialization and we can't infer anything more precise either. - world = f.source.primary_world - mi = f.source.specializations::Core.MethodInstance - Base.hasintersect(typeof(f).parameters[1], tt) || (warning = OC_MISMATCH_WARNING) + world = oc.source.primary_world + mi = oc.source.specializations::Core.MethodInstance + Base.hasintersect(typeof(oc).parameters[1], tt) || (warning = OC_MISMATCH_WARNING) else - mi = Base.specialize_method(f.source, Tuple{typeof(f.captures), tt.parameters...}, Core.svec()) + mi = Base.specialize_method(oc.source, Tuple{typeof(oc.captures), tt.parameters...}, Core.svec()) isdispatchtuple(mi.specTypes) || (warning = GENERIC_SIG_WARNING) end end @@ -236,19 +258,19 @@ function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrappe end if isempty(str) # if that failed (or we want metadata), use LLVM to generate more accurate assembly output - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing src = Base.Compiler.typeinf_code(Base.Compiler.NativeInterpreter(world), mi, true) else - src, rt = Base.get_oc_code_rt(nothing, f, tt, true) + src, rt = Base.get_oc_code_rt(nothing, arginfo.oc, arginfo.tt, true) end src isa Core.CodeInfo || error("failed to infer source for $mi") str = _dump_function_native_assembly(mi, src, wrapper, syntax, debuginfo, binary, raw, params) end else - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing src = Base.Compiler.typeinf_code(Base.Compiler.NativeInterpreter(world), mi, true) else - src, rt = Base.get_oc_code_rt(nothing, f, tt, true) + src, rt = Base.get_oc_code_rt(nothing, arginfo.oc, arginfo.tt, true) end src isa Core.CodeInfo || error("failed to infer source for $mi") str = _dump_function_llvm(mi, src, wrapper, !raw, dump_module, optimize, debuginfo, params) @@ -311,16 +333,18 @@ Keyword argument `debuginfo` may be one of source (default) or none, to specify See also: [`@code_llvm`](@ref), [`code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_native`](@ref). """ -function code_llvm(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); +function code_llvm(io::IO, arginfo::ArgInfo; raw::Bool=false, dump_module::Bool=false, optimize::Bool=true, debuginfo::Symbol=:default, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) - d = _dump_function(f, types, false, false, raw, dump_module, :intel, optimize, debuginfo, false, params) + d = _dump_function(arginfo, false, false, raw, dump_module, :intel, optimize, debuginfo, false, params) if highlighting[:llvm] && get(io, :color, false)::Bool print_llvm(io, d) else print(io, d) end end +code_llvm(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_llvm(io, ArgInfo(argtypes); kwargs...) +code_llvm(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); kwargs...) = code_llvm(io, ArgInfo(f, types); kwargs...) code_llvm(args...; kwargs...) = (@nospecialize; code_llvm(stdout, args...; kwargs...)) """ @@ -337,17 +361,19 @@ generic function and type signature to `io`. See also: [`@code_native`](@ref), [`code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_llvm`](@ref). """ -function code_native(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); +function code_native(io::IO, arginfo::ArgInfo; dump_module::Bool=true, syntax::Symbol=:intel, raw::Bool=false, debuginfo::Symbol=:default, binary::Bool=false, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) - d = _dump_function(f, types, true, false, raw, dump_module, syntax, true, debuginfo, binary, params) + d = _dump_function(arginfo, true, false, raw, dump_module, syntax, true, debuginfo, binary, params) if highlighting[:native] && get(io, :color, false)::Bool print_native(io, d) else print(io, d) end end +code_native(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_native(io, ArgInfo(argtypes); kwargs...) +code_native(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); kwargs...) = code_native(io, ArgInfo(f, types); kwargs...) code_native(args...; kwargs...) = (@nospecialize; code_native(stdout, args...; kwargs...)) ## colorized IR and assembly printing diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index ea845c012f529..b233dbffa9099 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -592,7 +592,9 @@ end # module ReflectionTest # Issue #18883, code_llvm/code_native for generated functions @generated f18883() = nothing @test !isempty(sprint(code_llvm, f18883, Tuple{})) +@test !isempty(sprint(code_llvm, (typeof(f18883),))) @test !isempty(sprint(code_native, f18883, Tuple{})) +@test !isempty(sprint(code_native, (typeof(f18883),))) ix86 = r"i[356]86" @@ -865,6 +867,27 @@ let # `default_tt` should work with any function with one method end); true) end +let # specifying calls as argtypes (incl. arg0) should be supported + @test (code_warntype(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_warntype(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) + @test (code_llvm(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_llvm(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) + @test (code_native(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_native(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) +end + @testset "code_llvm on opaque_closure" begin let ci = code_typed(+, (Int, Int))[1][1] ir = Core.Compiler.inflate_ir(ci) diff --git a/test/rebinding.jl b/test/rebinding.jl index 56bde53b7b746..a54a7d833403b 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -285,7 +285,7 @@ module RangeMerge function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) - d = InteractiveUtils._dump_function(f, t, false, false, raw, dump_module, :att, optimize, :none, false, params) + d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, params) sprint(print, d) end diff --git a/test/reflection.jl b/test/reflection.jl index 29774c2bfa069..5a0cd2e8645f7 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -16,6 +16,11 @@ function test_ir_reflection(freflect, f, types) nothing end +function test_ir_reflection(freflect, argtypes) + @test !isempty(freflect(argtypes)) + nothing +end + function test_bin_reflection(freflect, f, types) iob = IOBuffer() freflect(iob, f, types) @@ -27,6 +32,9 @@ end function test_code_reflection(freflect, f, types, tester) tester(freflect, f, types) tester(freflect, f, (types.parameters...,)) + tt = Base.signature_type(f, types) + tester(freflect, tt) + tester(freflect, (tt.parameters...,)) nothing end @@ -43,6 +51,7 @@ end test_code_reflections(test_ir_reflection, code_lowered) test_code_reflections(test_ir_reflection, code_typed) +test_code_reflections(test_ir_reflection, Base.code_ircode) io = IOBuffer() Base.print_statement_costs(io, map, (typeof(sqrt), Tuple{Int})) @@ -682,6 +691,10 @@ end @test Base.code_typed_by_type(Tuple{Type{<:Val}})[2][2] == Val @test Base.code_typed_by_type(Tuple{typeof(sin), Float64})[1][2] === Float64 +# functor-like code_typed(...) +@test Base.code_typed((Type{<:Val},))[2][2] == Val +@test Base.code_typed((typeof(sin), Float64))[1][2] === Float64 + # New reflection methods in 0.6 struct ReflectionExample{T<:AbstractFloat, N} x::Tuple{T, N} @@ -1038,11 +1051,12 @@ _test_at_locals2(1,1,0.5f0) @testset "issue #31687" begin import InteractiveUtils._dump_function + import InteractiveUtils.ArgInfo @noinline f31687_child(i) = f31687_nonexistent(i) f31687_parent() = f31687_child(0) params = Base.CodegenParams() - _dump_function(f31687_parent, Tuple{}, + _dump_function(ArgInfo(f31687_parent, Tuple{}), #=native=#false, #=wrapper=#false, #=raw=#true, #=dump_module=#true, #=syntax=#:att, #=optimize=#false, :none, #=binary=#false) @@ -1131,9 +1145,12 @@ end @test 1+1 == 2 mi1 = Base.method_instance(+, (Int, Int)) @test mi1.def.name == :+ - # Note `jl_method_lookup` doesn't returns CNull if not found - mi2 = @ccall jl_method_lookup(Any[+, 1, 1]::Ptr{Any}, 3::Csize_t, Base.get_world_counter()::Csize_t)::Ref{Core.MethodInstance} - @test mi1 == mi2 + mi2 = Base.method_instance((typeof(+), Int, Int)) + @test mi2.def.name == :+ + # Note `jl_method_lookup` doesn't return CNull if not found + mi3 = @ccall jl_method_lookup(Any[+, 1, 1]::Ptr{Any}, 3::Csize_t, Base.get_world_counter()::Csize_t)::Ref{Core.MethodInstance} + @test mi1 == mi3 + @test mi2 == mi3 end Base.@assume_effects :terminates_locally function issue41694(x::Int) From 8602127a4d5f36b889f1a928056e529cafab3d3e Mon Sep 17 00:00:00 2001 From: Sam Schweigel Date: Mon, 7 Jul 2025 10:30:30 -0700 Subject: [PATCH 498/662] Prevent data races in invalidate_code_for_globalref! --- base/invalidation.jl | 32 ++++++++++++++++---------------- src/module.c | 24 ++++++++++++++++++++++++ src/staticdata.c | 4 ++++ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 9a3d1ddd912e6..51b346816d6c6 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -122,23 +122,23 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core invalidated_any |= invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) end end - if isdefined(b, :backedges) - for edge in b.backedges - if isa(edge, CodeInstance) - ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) - invalidated_any = true - elseif isa(edge, Core.Binding) - isdefined(edge, :partitions) || continue - latest_bpart = edge.partitions - latest_bpart.max_world == typemax(UInt) || continue - is_some_imported(binding_kind(latest_bpart)) || continue - if is_some_binding_imported(binding_kind(latest_bpart)) - partition_restriction(latest_bpart) === b || continue - end - push!(queued_bindings, (edge, latest_bpart, latest_bpart)) - else - invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) + nbackedges = ccall(:jl_binding_backedges_length, Csize_t, (Any,), b) + for i = 1:nbackedges + edge = ccall(:jl_binding_backedges_getindex, Any, (Any, Csize_t), b, i) + if isa(edge, CodeInstance) + ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) + invalidated_any = true + elseif isa(edge, Core.Binding) + isdefined(edge, :partitions) || continue + latest_bpart = edge.partitions + latest_bpart.max_world == typemax(UInt) || continue + is_some_imported(binding_kind(latest_bpart)) || continue + if is_some_binding_imported(binding_kind(latest_bpart)) + partition_restriction(latest_bpart) === b || continue end + push!(queued_bindings, (edge, latest_bpart, latest_bpart)) + else + invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end diff --git a/src/module.c b/src/module.c index 69ed05eacf05f..03d82c66fbf51 100644 --- a/src/module.c +++ b/src/module.c @@ -470,6 +470,25 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_bindin return NULL; } +JL_DLLEXPORT size_t jl_binding_backedges_length(jl_binding_t *b) +{ + JL_LOCK(&b->globalref->mod->lock); + size_t len = 0; + if (b->backedges) + len = jl_array_len(b->backedges); + JL_UNLOCK(&b->globalref->mod->lock); + return len; +} + +JL_DLLEXPORT jl_value_t *jl_binding_backedges_getindex(jl_binding_t *b, size_t i) +{ + JL_LOCK(&b->globalref->mod->lock); + assert(b->backedges); + jl_value_t *ret = jl_array_ptr_ref(b->backedges, i-1); + JL_UNLOCK(&b->globalref->mod->lock); + return ret; +} + static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) { jl_task_t *ct = jl_current_task; @@ -527,7 +546,9 @@ jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default { jl_module_t *m = jl_new_module__(name, parent); JL_GC_PUSH1(&m); + JL_LOCK(&world_counter_lock); jl_add_default_names(m, default_using_core, self_name); + JL_UNLOCK(&world_counter_lock); JL_GC_POP(); return m; } @@ -1620,6 +1641,7 @@ void jl_invalidate_binding_refs(jl_globalref_t *ref, jl_binding_partition_t *inv JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge) { + JL_LOCK(&b->globalref->mod->lock); if (!b->backedges) { b->backedges = jl_alloc_vec_any(0); jl_gc_wb(b, b->backedges); @@ -1627,9 +1649,11 @@ JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge) jl_array_ptr_ref(b->backedges, jl_array_len(b->backedges)-1) == edge) { // Optimization: Deduplicate repeated insertion of the same edge (e.g. during // definition of a method that contains many references to the same global) + JL_UNLOCK(&b->globalref->mod->lock); return; } jl_array_ptr_1d_push(b->backedges, edge); + JL_UNLOCK(&b->globalref->mod->lock); } // Called for all GlobalRefs found in lowered code. Adds backedges for cross-module diff --git a/src/staticdata.c b/src/staticdata.c index 10bd2511bb006..2212215a39325 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3751,6 +3751,7 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t // We need to go through and re-validate any bindings in the same image that // may have imported us. if (b->backedges) { + JL_LOCK(&b->globalref->mod->lock); for (size_t i = 0; i < jl_array_len(b->backedges); i++) { jl_value_t *edge = jl_array_ptr_ref(b->backedges, i); if (!jl_is_binding(edge)) @@ -3758,8 +3759,11 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t jl_binding_t *bedge = (jl_binding_t*)edge; if (!jl_atomic_load_relaxed(&bedge->partitions)) continue; + JL_UNLOCK(&b->globalref->mod->lock); jl_validate_binding_partition(bedge, jl_atomic_load_relaxed(&bedge->partitions), mod_idx, 0, 0); + JL_LOCK(&b->globalref->mod->lock); } + JL_UNLOCK(&b->globalref->mod->lock); } if (bpart->kind & PARTITION_FLAG_EXPORTED) { jl_module_t *mod = b->globalref->mod; From 7a1590fc6e6e6c5836ce6b538cf9db245245ac74 Mon Sep 17 00:00:00 2001 From: Sam Schweigel Date: Mon, 7 Jul 2025 13:29:02 -0700 Subject: [PATCH 499/662] Fix type instability in invalidate_code_for_globalref! --- base/invalidation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/invalidation.jl b/base/invalidation.jl index 51b346816d6c6..0a44449748c2f 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -149,7 +149,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod) if usings_backedges !== nothing for user::Module in usings_backedges::Vector{Any} - user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name) + user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name)::Union{Core.Binding, Nothing} user_binding === nothing && continue isdefined(user_binding, :partitions) || continue latest_bpart = user_binding.partitions From 5741911605a1c56837df2285f62320f0237dbf08 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Mon, 7 Jul 2025 19:21:40 -0500 Subject: [PATCH 500/662] Add the fact that functions ending with `!` may allocate to the FAQ (#58904) I've run into this question several times, that might count as "frequently asked". --- doc/src/manual/faq.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/src/manual/faq.md b/doc/src/manual/faq.md index 188b8b7f79f3a..16c17a92e6b3f 100644 --- a/doc/src/manual/faq.md +++ b/doc/src/manual/faq.md @@ -391,6 +391,12 @@ julia> twothreearr() 3 ``` +### Is a function that ends with `!` allowed to allocate? + +Yes! A function name ending with `!` indicates that the function mutates at +least one of its arguments (typically the first argument). However, it may +still allocate a scratch space to expedite computation or produce that result. + ## Types, type declarations, and constructors ### [What does "type-stable" mean?](@id man-type-stability) From 5c31ec013f75363be10851b4889b854cf2f16db6 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:40:02 -0700 Subject: [PATCH 501/662] Economy mode REPL: run the event loop with jl_uv_flush (#58926) `ios_flush` won't wait for the `jl_static_show` from the previous evaluation to complete, resulting in the output being interleaved with subsequent REPL outputs. Anything that produces a lot of output will trigger it, like `Core.GlobalMethods.defs`. --- src/jlapi.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/jlapi.c b/src/jlapi.c index 6559adc94c1b9..dfe5648afe17e 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -996,8 +996,8 @@ static NOINLINE int true_main(int argc, char *argv[]) while (!ios_eof(ios_stdin)) { char *volatile line = NULL; JL_TRY { - ios_puts("\njulia> ", ios_stdout); - ios_flush(ios_stdout); + jl_printf(JL_STDOUT, "\njulia> "); + jl_uv_flush(JL_STDOUT); line = ios_readline(ios_stdin); jl_value_t *val = (jl_value_t*)jl_eval_string(line); JL_GC_PUSH1(&val); @@ -1013,7 +1013,6 @@ static NOINLINE int true_main(int argc, char *argv[]) jl_printf(JL_STDOUT, "\n"); free(line); line = NULL; - jl_process_events(); } JL_CATCH { if (line) { From 67634fedc32248003435379e76c41ba4766f4d3b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Jul 2025 12:56:58 +0200 Subject: [PATCH 502/662] Fix grammar, typos, and formatting issues in docstrings (#58944) Co-authored-by: Claude --- base/essentials.jl | 2 +- base/int.jl | 2 +- base/libc.jl | 2 +- base/math.jl | 2 +- base/rational.jl | 4 ++-- base/reduce.jl | 6 +++--- base/special/exp.jl | 2 +- base/special/trig.jl | 2 +- base/stacktraces.jl | 2 +- base/stream.jl | 2 +- base/strings/basic.jl | 6 +++--- base/strings/search.jl | 2 +- base/strings/string.jl | 2 +- base/strings/util.jl | 2 +- base/subarray.jl | 2 +- base/sysinfo.jl | 2 +- base/terminfo.jl | 4 ++-- stdlib/Dates/src/Dates.jl | 2 +- stdlib/Dates/src/periods.jl | 2 +- stdlib/InteractiveUtils/src/InteractiveUtils.jl | 2 +- stdlib/Sockets/src/Sockets.jl | 4 ++-- stdlib/TOML/src/TOML.jl | 2 +- stdlib/UUIDs/src/UUIDs.jl | 2 +- 23 files changed, 30 insertions(+), 30 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index dd5e398c7e93e..ed7d7cae1586c 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -435,7 +435,7 @@ Stacktrace: [...] ``` -If `T` is a [`AbstractFloat`](@ref) type, then it will return the +If `T` is an [`AbstractFloat`](@ref) type, then it will return the closest value to `x` representable by `T`. Inf is treated as one ulp greater than `floatmax(T)` for purposes of determining nearest. diff --git a/base/int.jl b/base/int.jl index 6fdcc413c04a6..57ebeda4812ad 100644 --- a/base/int.jl +++ b/base/int.jl @@ -844,7 +844,7 @@ widen(::Type{UInt64}) = UInt128 # |x|<=2^(k-1), |y|<=2^k-1 => |x*y|<=2^(2k-1)-1 widemul(x::Signed,y::Unsigned) = widen(x) * signed(widen(y)) widemul(x::Unsigned,y::Signed) = signed(widen(x)) * widen(y) -# multplication by Bool doesn't require widening +# multiplication by Bool doesn't require widening widemul(x::Bool,y::Bool) = x * y widemul(x::Bool,y::Number) = x * y widemul(x::Number,y::Bool) = x * y diff --git a/base/libc.jl b/base/libc.jl index 49e9d67590597..f575690c370d6 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -40,7 +40,7 @@ Base.cconvert(::Type{Cint}, fd::RawFD) = bitcast(Cint, fd) dup(src::RawFD[, target::RawFD])::RawFD Duplicate the file descriptor `src` so that the duplicate refers to the same OS -resource (e.g. a file or socket). A `target` file descriptor may be optionally +resource (e.g. a file or socket). A `target` file descriptor may optionally be passed to use for the new duplicate. """ dup(x::RawFD) = ccall((@static Sys.iswindows() ? :_dup : :dup), RawFD, (RawFD,), x) diff --git a/base/math.jl b/base/math.jl index b415d60e9bb27..5122676958405 100644 --- a/base/math.jl +++ b/base/math.jl @@ -482,7 +482,7 @@ asin(x::Number) """ acos(x::T) where {T <: Number} -> float(T) -Compute the inverse cosine of `x`, where the output is in radians +Compute the inverse cosine of `x`, where the output is in radians. Return a `T(NaN)` if `isnan(x)`. """ diff --git a/base/rational.jl b/base/rational.jl index 6e1edb457277b..e04f14760d8f4 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -572,13 +572,13 @@ float(::Type{Rational{T}}) where {T<:Integer} = float(T) function gcd(x::Rational, y::Rational) if isinf(x) != isinf(y) - throw(ArgumentError("lcm is not defined between infinite and finite numbers")) + throw(ArgumentError("gcd is not defined between infinite and finite numbers")) end unsafe_rational(gcd(x.num, y.num), lcm(x.den, y.den)) end function lcm(x::Rational, y::Rational) if isinf(x) != isinf(y) - throw(ArgumentError("lcm is not defined")) + throw(ArgumentError("lcm is not defined between infinite and finite numbers")) end return unsafe_rational(lcm(x.num, y.num), gcd(x.den, y.den)) end diff --git a/base/reduce.jl b/base/reduce.jl index 968ccda4db28e..743713360ed35 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -362,7 +362,7 @@ reduce_empty(op::FlipArgs, ::Type{T}) where {T} = reduce_empty(op.f, T) """ Base.mapreduce_empty(f, op, T) -The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref`) or +The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref) or [`mapfoldr`](@ref) with map `f` and reduction `op` over an empty array with element type of `T`. See [`Base.reduce_empty`](@ref) for more information. """ @@ -388,7 +388,7 @@ reduce_empty_iter(op, itr, ::EltypeUnknown) = throw(ArgumentError(""" """ Base.reduce_first(op, x) -The value to be returned when calling [`reduce`](@ref), [`foldl`](@ref`) or +The value to be returned when calling [`reduce`](@ref), [`foldl`](@ref) or [`foldr`](@ref) with reduction `op` over an iterator which contains a single element `x`. This value may also be used to initialise the recursion, so that `reduce(op, [x, y])` may call `op(reduce_first(op, x), y)`. @@ -413,7 +413,7 @@ reduce_first(::typeof(hcat), x) = hcat(x) """ Base.mapreduce_first(f, op, x) -The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref`) or +The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref) or [`mapfoldr`](@ref) with map `f` and reduction `op` over an iterator which contains a single element `x`. This value may also be used to initialise the recursion, so that `mapreduce(f, op, [x, y])` may call `op(mapreduce_first(f, op, x), f(y))`. diff --git a/base/special/exp.jl b/base/special/exp.jl index f0db1f70a354b..f423551b02f29 100644 --- a/base/special/exp.jl +++ b/base/special/exp.jl @@ -491,7 +491,7 @@ end expm1(x) Accurately compute ``e^x-1``. It avoids the loss of precision involved in the direct -evaluation of exp(x)-1 for small values of x. +evaluation of exp(x) - 1 for small values of x. # Examples ```jldoctest julia> expm1(1e-16) diff --git a/base/special/trig.jl b/base/special/trig.jl index 66e4b46d7d489..96a2da5f5b968 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -824,7 +824,7 @@ Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. -See also: [`cispi`](@ref), [`sincosd`](@ref), [`cospi`](@ref). +See also: [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). """ function cospi(x::T) where T<:IEEEFloat x = abs(x) diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 58681c99d889b..8f2866af170f7 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -24,7 +24,7 @@ Stack information representing execution context, with the following fields: - `linfo::Union{Method, Core.MethodInstance, Core.CodeInstance, Core.CodeInfo, Nothing}` - The Method, MethodInstance, CodeInstance, or CodeInfo containing the execution context (if it could be found), \ + The Method, MethodInstance, CodeInstance, or CodeInfo containing the execution context (if it could be found), or nothing (for example, if the inlining was a result of macro expansion). - `file::Symbol` diff --git a/base/stream.jl b/base/stream.jl index ae3d4f3c821e0..d88f164f3ccca 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -318,7 +318,7 @@ end """ open(fd::OS_HANDLE)::IO -Take a raw file descriptor wrap it in a Julia-aware IO type, +Take a raw file descriptor and wrap it in a Julia-aware IO type, and take ownership of the fd handle. Call `open(Libc.dup(fd))` to avoid the ownership capture of the original handle. diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 90a28e48988ae..352d42eb3a40f 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -463,7 +463,7 @@ end * Case `n == 1` - If `i` is in bounds in `s` return the index of the start of the character whose + If `i` is in bounds in `str` return the index of the start of the character whose encoding starts before index `i`. In other words, if `i` is the start of a character, return the start of the previous character; if `i` is not the start of a character, rewind until the start of a character and return that index. @@ -522,7 +522,7 @@ end * Case `n == 1` - If `i` is in bounds in `s` return the index of the start of the character whose + If `i` is in bounds in `str` return the index of the start of the character whose encoding starts after index `i`. In other words, if `i` is the start of a character, return the start of the next character; if `i` is not the start of a character, move forward until the start of a character and return that index. @@ -539,7 +539,7 @@ end * Case `n == 0` - Return `i` only if `i` is a valid index in `s` or is equal to `0`. + Return `i` only if `i` is a valid index in `str` or is equal to `0`. Otherwise `StringIndexError` or `BoundsError` is thrown. # Examples diff --git a/base/strings/search.jl b/base/strings/search.jl index 90e02a0e5792c..a68b5f52b9f0a 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -585,7 +585,7 @@ findlast(ch::AbstractChar, string::AbstractString) = findlast(==(ch), string) overlap::Bool = false, ) findall( - pattern::Vector{UInt8} + pattern::Vector{UInt8}, A::Vector{UInt8}; overlap::Bool = false, ) diff --git a/base/strings/string.jl b/base/strings/string.jl index 8d31a6f52ca8b..53593e3936842 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -83,7 +83,7 @@ end Create a `String` from `m`, changing the interpretation of the contents of `m`. This is done without copying, if possible. Thus, any access to `m` after -calling this function, either to read or to write, is undefined behaviour. +calling this function, either to read or to write, is undefined behavior. """ function unsafe_takestring(m::Memory{UInt8}) isempty(m) ? "" : ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), m, length(m)) diff --git a/base/strings/util.jl b/base/strings/util.jl index 957be7b914c02..c3df790cd81e6 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -3,7 +3,7 @@ """ Base.Chars = Union{AbstractChar,Tuple{Vararg{AbstractChar}},AbstractVector{<:AbstractChar},AbstractSet{<:AbstractChar}} -An alias type for a either single character or a tuple/vector/set of characters, used to describe arguments +An alias type for either a single character or a tuple/vector/set of characters, used to describe arguments of several string-matching functions such as [`startswith`](@ref) and [`strip`](@ref). !!! compat "Julia 1.11" diff --git a/base/subarray.jl b/base/subarray.jl index 3a0be7d82b981..bc68d7c555d13 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -128,7 +128,7 @@ _trimmedshape(i::AbstractArray{<:ScalarIndex}, rest...) = (length(i), _trimmedsh _trimmedshape(i::AbstractArray{<:AbstractCartesianIndex{0}}, rest...) = _trimmedshape(rest...) _trimmedshape(i::AbstractArray{<:AbstractCartesianIndex{N}}, rest...) where {N} = (length(i), ntuple(Returns(1), Val(N - 1))..., _trimmedshape(rest...)...) _trimmedshape() = () -# We can avoid the repeation from `AbstractArray{CartesianIndex{0}}` +# We can avoid the repetition from `AbstractArray{CartesianIndex{0}}` _trimmedpind(i, rest...) = (map(Returns(:), axes(i))..., _trimmedpind(rest...)...) _trimmedpind(i::AbstractRange, rest...) = (i, _trimmedpind(rest...)...) _trimmedpind(i::Union{UnitRange,StepRange,OneTo}, rest...) = ((:), _trimmedpind(rest...)...) diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 77fd628ff2883..5e1a26973331e 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -104,7 +104,7 @@ Standard word size on the current machine, in bits. const WORD_SIZE = Core.sizeof(Int) * 8 """ - Sys.SC_CLK_TCK: + Sys.SC_CLK_TCK::Clong The number of system "clock ticks" per second, corresponding to `sysconf(_SC_CLK_TCK)` on POSIX systems, or `0` if it is unknown. diff --git a/base/terminfo.jl b/base/terminfo.jl index 8ea8387077d36..2c40042424f1d 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -323,8 +323,8 @@ Return a boolean signifying whether the current terminal supports 24-bit colors. Multiple conditions are taken as signifying truecolor support, specifically any of the following: - The `COLORTERM` environment variable is set to `"truecolor"` or `"24bit"` -- The current terminfo sets the [`RGB`[^1] - capability](https://invisible-island.net/ncurses/man/user_caps.5.html#h3-Recognized-Capabilities) +- The current terminfo sets the [`RGB` + capability](https://invisible-island.net/ncurses/man/user_caps.5.html#h3-Recognized-Capabilities)[^1] (or the legacy `Tc` capability[^2]) flag - The current terminfo provides `setrgbf` and `setrgbb` strings[^3] - The current terminfo has a `colors` number greater that `256`, on a unix system diff --git a/stdlib/Dates/src/Dates.jl b/stdlib/Dates/src/Dates.jl index 0e6d0d0ef6986..e63ec0ec2a487 100644 --- a/stdlib/Dates/src/Dates.jl +++ b/stdlib/Dates/src/Dates.jl @@ -5,7 +5,7 @@ The `Dates` module provides `Date`, `DateTime`, `Time` types, and related functions. -The types are not aware of time zones, based on UT seconds +The types are not aware of time zones, are based on UT seconds (86400 seconds a day, avoiding leap seconds), and use the proleptic Gregorian calendar, as specified in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). For time zone functionality, see the TimeZones.jl package. diff --git a/stdlib/Dates/src/periods.jl b/stdlib/Dates/src/periods.jl index da4cb495c5d94..42072f329ba11 100644 --- a/stdlib/Dates/src/periods.jl +++ b/stdlib/Dates/src/periods.jl @@ -122,7 +122,7 @@ coarserperiod(::Type{Month}) = (Year, 12) CompoundPeriod A `CompoundPeriod` is useful for expressing time periods that are not a fixed multiple of -smaller periods. For example, "a year and a day" is not a fixed number of days, but can +smaller periods. For example, "a year and a day" is not a fixed number of days, but can be expressed using a `CompoundPeriod`. In fact, a `CompoundPeriod` is automatically generated by addition of different period types, e.g. `Year(1) + Day(1)` produces a `CompoundPeriod` result. diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 8499027fae6cc..874e8d16490ae 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -200,7 +200,7 @@ end # `methodswith` -- shows a list of methods using the type given """ - methodswith(typ[, module or function]; supertypes::Bool=false]) + methodswith(typ[, module or function]; supertypes::Bool=false) Return an array of methods with an argument of type `typ`. diff --git a/stdlib/Sockets/src/Sockets.jl b/stdlib/Sockets/src/Sockets.jl index f9e0f2f88dd78..58438b152d825 100644 --- a/stdlib/Sockets/src/Sockets.jl +++ b/stdlib/Sockets/src/Sockets.jl @@ -220,7 +220,7 @@ end # Disables dual stack mode. const UV_TCP_IPV6ONLY = 1 -# Disables dual stack mode. Only available when using ipv6 binf +# Disables dual stack mode. Only available when using ipv6 bind const UV_UDP_IPV6ONLY = 1 # Indicates message was truncated because read buffer was too small. The @@ -780,7 +780,7 @@ end """ leave_multicast_group(sock::UDPSocket, group_addr, interface_addr = nothing) -Remove a socket from a particular multicast group defined by `group_addr`. +Remove a socket from a particular multicast group defined by `group_addr`. If `interface_addr` is given, specifies a particular interface for multi-homed systems. Use `join_multicast_group()` to enable reception of a group. """ diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index b37a5ca83c251..4f085aff60a18 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -31,7 +31,7 @@ _readstring(f::AbstractString) = isfile(f) ? read(f, String) : error(repr(f), ": Parser() Constructor for a TOML `Parser`. Note that in most cases one does not need to -explicitly create a `Parser` but instead one directly use use +explicitly create a `Parser` but instead one directly uses [`TOML.parsefile`](@ref) or [`TOML.parse`](@ref). Using an explicit parser will however reuse some internal data structures which can be beneficial for performance if a larger number of small files are parsed. diff --git a/stdlib/UUIDs/src/UUIDs.jl b/stdlib/UUIDs/src/UUIDs.jl index 989e0e16e2e58..881e65b218d87 100644 --- a/stdlib/UUIDs/src/UUIDs.jl +++ b/stdlib/UUIDs/src/UUIDs.jl @@ -2,7 +2,7 @@ """ This module provides universally unique identifiers (UUIDs), -along with functions creating the different variants. +along with functions for creating the different variants. """ module UUIDs From 2349f431b1d0f97f3f06f8adc5d335b89d5f061c Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 9 Jul 2025 16:48:45 -0400 Subject: [PATCH 503/662] Fix nthreadpools size in JLOptions (#58937) --- base/options.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/options.jl b/base/options.jl index bf36c3ba0527d..203a91ca8f641 100644 --- a/base/options.jl +++ b/base/options.jl @@ -9,7 +9,7 @@ struct JLOptions commands::Ptr{Ptr{UInt8}} # (e)eval, (E)print, (L)load image_file::Ptr{UInt8} cpu_target::Ptr{UInt8} - nthreadpools::Int16 + nthreadpools::Int8 nthreads::Int16 nmarkthreads::Int16 nsweepthreads::Int8 From a60815fe4bf268726f4cb1574b680061ca8da690 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 9 Jul 2025 16:51:19 -0400 Subject: [PATCH 504/662] NFC: Remove duplicate `julia-src-%` dependency in makefile (#58947) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 39f34b5372262..9d04f36097878 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ julia-src-release julia-src-debug : julia-src-% : julia-deps julia_flisp.boot.in julia-cli-release julia-cli-debug: julia-cli-% : julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/cli $* -julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% julia-src-% | $(build_private_libdir) +julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$* julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \ From 41613c29ddaaf78a55d14c99de7dfb0e858b7b00 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 9 Jul 2025 16:55:18 -0400 Subject: [PATCH 505/662] Improve error message for missing dependencies in packages (#58878) --- base/loading.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index 72a9898ca4784..7d2f33ede633e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2378,7 +2378,8 @@ function __require(into::Module, mod::Symbol) else manifest_warnings = collect_manifest_warnings() throw(ArgumentError(""" - Package $(where.name) does not have $mod in its dependencies: + Cannot load (`using/import`) module $mod into module $into in package $(where.name) + because package $(where.name) does not have $mod in its dependencies: $manifest_warnings- You may have a partially installed environment. Try `Pkg.instantiate()` to ensure all packages in the environment are installed. - Or, if you have $(where.name) checked out for development and have From 72e2c45554dd6e7f55daea4a5f234b246f6e93e8 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 10 Jul 2025 00:45:06 -0400 Subject: [PATCH 506/662] Make current_terminfo a OncePerProcess (#58854) There seems to be no reason to always load this unconditionally - especially since it's in the critical startup path. If we never print colored output or our IO is not a TTY, we don't need to load this at all. While we're at it, remove the `term_type` argument to `ttyhascolor`, which didn't work as advertised anyway, since it still looked at the current_terminfo. If clients want to do a full TermInfo check, they can do that explicitly. (Written by Claude Code) --- base/client.jl | 4 ---- base/terminfo.jl | 24 ++++++++++++------- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/StyledStrings.version | 2 +- 7 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 diff --git a/base/client.jl b/base/client.jl index ae99654c233a8..03248700e7ca0 100644 --- a/base/client.jl +++ b/base/client.jl @@ -270,10 +270,6 @@ function exec_options(opts) interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY) is_interactive::Bool |= interactiveinput - # load terminfo in for styled printing - term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") - global current_terminfo = load_terminfo(term_env) - # load ~/.julia/config/startup.jl file if startup try diff --git a/base/terminfo.jl b/base/terminfo.jl index 2c40042424f1d..be0dd53b1ac74 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -303,16 +303,24 @@ end """ The terminfo of the current terminal. """ -current_terminfo::TermInfo = TermInfo() +const current_terminfo = OncePerProcess{TermInfo}() do + term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") + terminfo = load_terminfo(term_env) + # Ensure setaf is set for xterm terminals + if !haskey(terminfo, :setaf) && startswith(term_env, "xterm") + # For xterm-like terminals without setaf, add a reasonable default + terminfo.strings[:setaf] = "\e[3%p1%dm" + end + return terminfo +end # Legacy/TTY methods and the `:color` parameter if Sys.iswindows() - ttyhascolor(term_type = nothing) = true + ttyhascolor() = true else - function ttyhascolor(term_type = get(ENV, "TERM", "")) - startswith(term_type, "xterm") || - haskey(current_terminfo, :setaf) + function ttyhascolor() + haskey(current_terminfo(), :setaf) end end @@ -352,9 +360,9 @@ Multiple conditions are taken as signifying truecolor support, specifically any function ttyhastruecolor() # Lasciate ogne speranza, voi ch'intrate get(ENV, "COLORTERM", "") ∈ ("truecolor", "24bit") || - get(current_terminfo, :RGB, false) || get(current_terminfo, :Tc, false) || - (haskey(current_terminfo, :setrgbf) && haskey(current_terminfo, :setrgbb)) || - @static if Sys.isunix() get(current_terminfo, :colors, 0) > 256 else false end || + get(current_terminfo(), :RGB, false) || get(current_terminfo(), :Tc, false) || + (haskey(current_terminfo(), :setrgbf) && haskey(current_terminfo(), :setrgbb)) || + @static if Sys.isunix() get(current_terminfo(), :colors, 0) > 256 else false end || (Sys.iswindows() && Sys.windows_version() ≥ v"10.0.14931") || # See something(tryparse(Int, get(ENV, "VTE_VERSION", "")), 0) >= 3600 || # Per GNOME bug #685759 haskey(ENV, "XTERM_VERSION") || diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 new file mode 100644 index 0000000000000..0285925aaaa2a --- /dev/null +++ b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 @@ -0,0 +1 @@ +1762267d32633457c7c64067c1c9d871 diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 new file mode 100644 index 0000000000000..f7d817c2f1e84 --- /dev/null +++ b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 @@ -0,0 +1 @@ +72ca1b52b8f0c4e3d3ca58526cd34a3f1486c4fe373930bf513570f056700cddc65a4666d5b7767305b115ec0352639c4caaf03b9378911d596f6add3cbb762b diff --git a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 deleted file mode 100644 index 46d5cacf788df..0000000000000 --- a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -1cb6007a66d3f74cbe5b27ee449aa9c8 diff --git a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 b/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 deleted file mode 100644 index 724b2d311c123..0000000000000 --- a/deps/checksums/StyledStrings-3fe829fcf611b5fefaefb64df7e61f2ae82db117.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -1fa95646fdf4cc7ea282bd355fded9464e7572792912942ea1c45f6ed126eead2333fdeed92e7db3efbcd6c3a171a04e5c9562dab2685bb39947136284ae1da3 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 55a4a08c17ea0..8600b0875b045 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = main -STYLEDSTRINGS_SHA1 = 3fe829fcf611b5fefaefb64df7e61f2ae82db117 +STYLEDSTRINGS_SHA1 = 1aafc2f3abb6a977ee36a87206f9ce6446a8ae86 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 From 34f9032502f4d590f7889ae51093a452485f6714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8D=86=E9=95=BF=E9=80=AF?= Date: Thu, 10 Jul 2025 15:44:10 +0800 Subject: [PATCH 507/662] chore: remove redundant words in comment (#58955) --- Compiler/src/abstractinterpretation.jl | 2 +- base/staticdata.jl | 2 +- stdlib/Logging/test/runtests.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 924b83c4d2142..100cdd97e511a 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -4414,7 +4414,7 @@ function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt # "causes" since we ignored those in the comparison newtyp = tmerge(𝕃ᵢ, newtyp, LimitedAccuracy(Bottom, oldtyp.causes)) end - # if this `Conditional` is from from `@isdefined condt.slot`, refine its `undef` information + # if this `Conditional` is from `@isdefined condt.slot`, refine its `undef` information newundef = condt.isdefined ? !then_or_else : vtype.undef return StateUpdate(SlotNumber(condt.slot), VarState(newtyp, newundef), #=conditional=#true) end diff --git a/base/staticdata.jl b/base/staticdata.jl index e1180b69e8f5c..ee63f901f9bb8 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -37,7 +37,7 @@ function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visi @ccall jl_promote_ci_to_current(codeinst::Any, validation_world::UInt)::Cvoid minvalid = codeinst.min_world maxvalid = codeinst.max_world - # Finally, if this CI is still valid in some world age and and belongs to an external method(specialization), + # Finally, if this CI is still valid in some world age and belongs to an external method(specialization), # poke it that mi's cache if maxvalid ≥ minvalid && external caller = get_ci_mi(codeinst) diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 3e92b7d9e2697..6d926f4dd0340 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -334,7 +334,7 @@ end end ends = count(entry_end, line) - starts == 1 && ends == 1 && error("Interleaved logs: Log entry started and and another ended on one line") + starts == 1 && ends == 1 && error("Interleaved logs: Log entry started and another ended on one line") ends > 1 && error("Interleaved logs: Multiple log entries ended on one line") if ends == 1 startswith(line, entry_end) || error("Interleaved logs: Log entry ended in the middle of a line") From f65fe61ba7c2654505b17139d2dc7478939a221b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 10 Jul 2025 10:19:19 +0200 Subject: [PATCH 508/662] add a precompile workload to TOML (#58949) --- stdlib/TOML/src/TOML.jl | 2 ++ stdlib/TOML/src/precompile.jl | 38 +++++++++++++++++++++++++++++++++ stdlib/TOML/src/print.jl | 40 ++++++++++++++++++----------------- 3 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 stdlib/TOML/src/precompile.jl diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index 4f085aff60a18..d8cb0b063189a 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -148,4 +148,6 @@ Internals.reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothi Internals.parse(p::Parser) = Internals.parse(p._p) Internals.tryparse(p::Parser) = Internals.tryparse(p._p) +include("precompile.jl") + end diff --git a/stdlib/TOML/src/precompile.jl b/stdlib/TOML/src/precompile.jl new file mode 100644 index 0000000000000..f3f45478ad28b --- /dev/null +++ b/stdlib/TOML/src/precompile.jl @@ -0,0 +1,38 @@ +if Base.generating_output() +let + # Test TOML content + test_toml = """ + title = "Example" + with_quotes = \"quoted\" + number = 42 + float = 3.14 + boolean = true + date = 2023-01-01 + datetime = 2023-01-01T12:00:00Z + time = 12:00:00 + array = [1, 2, 3] + inline = {a=1, b=1.0, c=[1,2,3], d="foo"} + + [nested] + key = "value" + """ + + test_dict = TOML.parse(test_toml) + + TOML.parse(test_toml) + + io_stream = IOBuffer(test_toml) + TOML.parse(io_stream) + + mktemp() do path, io + write(io, test_toml) + close(io) + TOML.parsefile(path) + end + + mktemp() do path, io + TOML.print(io, test_dict) + close(io) + end +end +end diff --git a/stdlib/TOML/src/print.jl b/stdlib/TOML/src/print.jl index c6c046b9b40c6..741fd96e548a8 100644 --- a/stdlib/TOML/src/print.jl +++ b/stdlib/TOML/src/print.jl @@ -33,7 +33,6 @@ function print_toml_escaped(io::IO, s::AbstractString) end end -const MbyFunc = Union{Function, Nothing} const TOMLValue = Union{AbstractVector, AbstractDict, Bool, Integer, AbstractFloat, AbstractString, Dates.DateTime, Dates.Time, Dates.Date, Base.TOML.DateTime, Base.TOML.Time, Base.TOML.Date} @@ -59,8 +58,8 @@ function printkey(io::IO, keys::Vector{String}) end end -function to_toml_value(f::MbyFunc, value) - if f === nothing +function to_toml_value(@nospecialize(f::Function), value) + if f === identity error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`") end toml_value = f(value) @@ -75,12 +74,12 @@ end ########## # Fallback -function printvalue(f::MbyFunc, io::IO, value, sorted::Bool) +function printvalue(f::Function, io::IO, value, sorted::Bool) toml_value = to_toml_value(f, value) @invokelatest printvalue(f, io, toml_value, sorted) end -function printvalue(f::MbyFunc, io::IO, value::AbstractVector, sorted::Bool) +function printvalue(f::Function, io::IO, value::AbstractVector, sorted::Bool) Base.print(io, "[") for (i, x) in enumerate(value) i != 1 && Base.print(io, ", ") @@ -89,7 +88,7 @@ function printvalue(f::MbyFunc, io::IO, value::AbstractVector, sorted::Bool) Base.print(io, "]") end -function printvalue(f::MbyFunc, io::IO, value::TOMLValue, sorted::Bool) +function printvalue(f::Function, io::IO, value::TOMLValue, sorted::Bool) value isa Base.TOML.DateTime && (value = Dates.DateTime(value)) value isa Base.TOML.Time && (value = Dates.Time(value)) value isa Base.TOML.Date && (value = Dates.Date(value)) @@ -117,7 +116,7 @@ function print_integer(io::IO, value::Integer) return end -function print_inline_table(f::MbyFunc, io::IO, value::AbstractDict, sorted::Bool) +function print_inline_table(f::Function, io::IO, value::AbstractDict, sorted::Bool) vkeys = collect(keys(value)) if sorted sort!(vkeys) @@ -138,15 +137,14 @@ end # Tables # ########## -is_table(value) = isa(value, AbstractDict) -is_array_of_tables(value) = isa(value, AbstractArray) && - length(value) > 0 && ( - isa(value, AbstractArray{<:AbstractDict}) || - all(v -> isa(v, AbstractDict), value) - ) -is_tabular(value) = is_table(value) || @invokelatest(is_array_of_tables(value)) +is_table(@nospecialize(value)) = isa(value, AbstractDict) +is_array_of_tables(@nospecialize(value)) = + isa(value, AbstractArray) && + length(value) > 0 && (isa(value, AbstractArray{<:AbstractDict}) || + all(v -> isa(v, AbstractDict), value)) +is_tabular(@nospecialize(value)) = is_table(value) || @invokelatest(is_array_of_tables(value)) -function print_table(f::MbyFunc, io::IO, a::AbstractDict, +function print_table(f::Function, io::IO, a::AbstractDict, ks::Vector{String} = String[]; indent::Int = 0, first_block::Bool = true, @@ -228,7 +226,11 @@ end # API # ####### -print(f::MbyFunc, io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print_table(f, io, a; sorted, by, inline_tables) -print(f::MbyFunc, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print(f, stdout, a; sorted, by, inline_tables) -print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print_table(nothing, io, a; sorted, by, inline_tables) -print( a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print(nothing, stdout, a; sorted, by, inline_tables) +print(f::Function, io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print_table(f, io, a; sorted, by, inline_tables) +print(f::Function, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print(f, stdout, a; sorted, by, inline_tables) +print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print_table(identity, io, a; sorted, by, inline_tables) +print(a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print(identity, stdout, a; sorted, by, inline_tables) From 7634fdf1d6e910e19c2efc26ecedd3ccecc16fd5 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Thu, 10 Jul 2025 04:27:15 -0400 Subject: [PATCH 509/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?NetworkOptions=20stdlib=20from=20c090626=20to=20532992f=20(#588?= =?UTF-8?q?82)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DilumAluthge <5619885+DilumAluthge@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/NetworkOptions.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 create mode 100644 deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 delete mode 100644 deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 delete mode 100644 deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 new file mode 100644 index 0000000000000..fbb7d5b86627f --- /dev/null +++ b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 @@ -0,0 +1 @@ +afab093d162a62d5a488894f33d3b396 diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 new file mode 100644 index 0000000000000..146d3a3d1bad8 --- /dev/null +++ b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 @@ -0,0 +1 @@ +14b41cc9c93e2f9eeaa9499d65b8c42ee80691cbd533ef6cafabdb6e94c7cf31eee00fb603ca70dfe86930c871419cf17a8f05c0a76bd379a8bbf705b875dfe2 diff --git a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 b/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 deleted file mode 100644 index 87111ac121562..0000000000000 --- a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -b851cab503506c37af6e4c861d81b8ce diff --git a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 b/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 deleted file mode 100644 index 79f9e269ff599..0000000000000 --- a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -4cba5c531e5e7205bb6d7f0179da8b29ca7c4dcf42f27de5f70be7674efc1fa92ea22e134e6584743e2905edbd754838d8a02f6ba7811c7a5b99ab9db3bde596 diff --git a/stdlib/NetworkOptions.version b/stdlib/NetworkOptions.version index 69b56e03ed89e..f7cb50a74d106 100644 --- a/stdlib/NetworkOptions.version +++ b/stdlib/NetworkOptions.version @@ -1,4 +1,4 @@ NETWORKOPTIONS_BRANCH = master -NETWORKOPTIONS_SHA1 = c090626d3feee6d6a5c476346d22d6147c9c6d2d +NETWORKOPTIONS_SHA1 = 532992fcc0f1d02df48374969cbae37e34c01360 NETWORKOPTIONS_GIT_URL := https://github.com/JuliaLang/NetworkOptions.jl.git NETWORKOPTIONS_TAR_URL = https://api.github.com/repos/JuliaLang/NetworkOptions.jl/tarball/$1 From d493a978633282a3de03c20b27617ead48c9f3e8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 10 Jul 2025 11:09:32 -0400 Subject: [PATCH 510/662] remove excessive code from trim script (#58853) Co-authored-by: gbaraldi --- contrib/juliac/juliac-trim-base.jl | 54 ++-------------------------- contrib/juliac/juliac-trim-stdlib.jl | 8 ++--- test/trimming/basic_jll.jl | 10 ++++++ 3 files changed, 16 insertions(+), 56 deletions(-) diff --git a/contrib/juliac/juliac-trim-base.jl b/contrib/juliac/juliac-trim-base.jl index 96fed77969c97..1d4d9558e0bf5 100644 --- a/contrib/juliac/juliac-trim-base.jl +++ b/contrib/juliac/juliac-trim-base.jl @@ -78,38 +78,12 @@ end end end show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) - - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) where {F, F2} = - _mapreduce_dim(f, op, init, A, dims) - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted...; kw...) where {F, F2} = - reduce(op, map(f, A...); kw...) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - mapfoldl_impl(f, op, nt, A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - _mapreduce(f, op, IndexStyle(A), A) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, dims) where {F, F2} = - mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) where {F,F2} = - mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) - - mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F, F2} = - reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) - mapreduce_first(f::F, op::F2, x) where {F,F2} = reduce_first(op, f(x)) - - _mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted) where {F,F2} = _mapreduce(f, op, IndexStyle(A), A) - mapreduce_empty(::typeof(identity), op::F, T) where {F} = reduce_empty(op, T) - mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) - mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) end @eval Base.Sys begin - __init_build() = nothing + __init_build() = nothing # VersionNumber parsing is not supported yet end @eval Base.GMP begin - function __init__() + function __init__() # VersionNumber parsing is not supported yet try ccall((:__gmp_set_memory_functions, libgmp), Cvoid, (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), @@ -135,27 +109,3 @@ end end end end -@eval Base.Sort begin - issorted(itr; - lt::T=isless, by::F=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) where {T,F} = - issorted(itr, ord(lt,by,rev,order)) -end -@eval Base.TOML begin - function try_return_datetime(p, year, month, day, h, m, s, ms) - return DateTime(year, month, day, h, m, s, ms) - end - function try_return_date(p, year, month, day) - return Date(year, month, day) - end - function parse_local_time(l::Parser) - h = @try parse_int(l, false) - h in 0:23 || return ParserError(ErrParsingDateTime) - _, m, s, ms = @try _parse_local_time(l, true) - # TODO: Could potentially parse greater accuracy for the - # fractional seconds here. - return try_return_time(l, h, m, s, ms) - end - function try_return_time(p, h, m, s, ms) - return Time(h, m, s, ms) - end -end diff --git a/contrib/juliac/juliac-trim-stdlib.jl b/contrib/juliac/juliac-trim-stdlib.jl index 3e2501c2c3e50..0cc3f01aa92f8 100644 --- a/contrib/juliac/juliac-trim-stdlib.jl +++ b/contrib/juliac/juliac-trim-stdlib.jl @@ -53,7 +53,7 @@ let Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) if Pkg !== nothing @eval Pkg begin - __init__() = rand() #TODO, methods that do nothing don't get codegened + __init__() = nothing # Assume the Pkg is not actually used end end @@ -61,7 +61,7 @@ let Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) if StyledStrings !== nothing @eval StyledStrings begin - __init__() = rand() + __init__() = nothing # Assume that StyledStrings are not actually used end end @@ -69,7 +69,7 @@ let Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) if Markdown !== nothing @eval Markdown begin - __init__() = rand() + __init__() = nothing # Assume that Markdown is not actually used with StyledStrings end end @@ -77,7 +77,7 @@ let Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) if JuliaSyntaxHighlighting !== nothing @eval JuliaSyntaxHighlighting begin - __init__() = rand() + __init__() = nothing # Assume the JuliaSyntaxHighlighting is not actually used with StyledStrings end end end diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index 6ac6b2797d4e3..a99e3f7692d27 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -25,5 +25,15 @@ function @main(args::Vector{String})::Cint fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) ccall(cfunc, Cvoid, (Ptr{Cvoid},), fptr) + # map/mapreduce should work but relies on inlining and other optimizations + arr = rand(10) + sorted_arr = sort(arr) + tot = sum(sorted_arr) + tot = prod(sorted_arr) + a = any(x -> x > 0, sorted_arr) + b = all(x -> x >= 0, sorted_arr) + c = map(x -> x^2, sorted_arr) + # d = mapreduce(x -> x^2, +, sorted_arr) this doesn't work because of mapreduce_empty_iter having F specialized + # e = reduce(xor, rand(Int, 10)) return 0 end From 6a9da96740c2c49fa8fc9477e6e1376cc6ebf908 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Thu, 10 Jul 2025 22:01:54 +0200 Subject: [PATCH 511/662] Add juliac Artifacts.toml in Makefile (#58936) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d04f36097878..40dd1565f8744 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac/juliac.jl $(build_datarootdir)/julia/juliac/juliac-buildscript.jl $(build_datarootdir)/julia/juliac/juliac-trim-base.jl $(build_datarootdir)/julia/juliac/juliac-trim-stdlib.jl +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac/juliac.jl $(build_datarootdir)/julia/juliac/juliac-buildscript.jl $(build_datarootdir)/julia/juliac/juliac-trim-base.jl $(build_datarootdir)/julia/juliac/juliac-trim-stdlib.jl $(build_datarootdir)/julia/juliac/Artifacts.toml @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps From 3705629e80e4af9bf551c7745a773efefd390ce7 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 10 Jul 2025 16:03:29 -0400 Subject: [PATCH 512/662] staticdata: Don't discard inlineable code that inference may need (#58842) See https://github.com/JuliaLang/julia/issues/58841#issuecomment-3014833096. We were accidentally discarding inferred code during staticdata preparation that we would need immediately afterwards to satisfy inlining requests during code generation for the system image. This was resulting in spurious extra compilation at the first inference after sysimage reload. Additionally it was likely causing various unnecessary dispatch slow paths in the generated inference code. Fixes #58841. --- src/staticdata.c | 13 +++++++++++-- test/precompile.jl | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 2212215a39325..ba6b40e2e5ea3 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -756,6 +756,15 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ } } +static int codeinst_may_be_runnable(jl_code_instance_t *ci, int incremental) { + size_t max_world = jl_atomic_load_relaxed(&ci->max_world); + if (max_world == ~(size_t)0) + return 1; + if (incremental) + return 0; + return jl_atomic_load_relaxed(&ci->min_world) <= jl_typeinf_world && jl_typeinf_world <= max_world; +} + // Anything that requires uniquing or fixing during deserialization needs to be "toplevel" // in serialization (i.e., have its own entry in `serialization_order`). Consequently, // objects that act as containers for other potentially-"problematic" objects must add such "children" @@ -924,8 +933,8 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ else if (def->source == NULL) { // don't delete code from optimized opaque closures that can't be reconstructed (and builtins) } - else if (jl_atomic_load_relaxed(&ci->max_world) != ~(size_t)0 || // delete all code that cannot run - jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant + else if (!codeinst_may_be_runnable(ci, s->incremental) || // delete all code that cannot run + jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant discard = 1; } else if (native_functions && // don't delete any code if making a ji file diff --git a/test/precompile.jl b/test/precompile.jl index afb3dcc154853..522ee49efebb7 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2498,4 +2498,10 @@ let m = only(methods(Base.var"@big_str")) @test m.specializations === Core.svec() || !isdefined(m.specializations, :cache) end +# Issue #58841 - make sure we don't accidentally throw away code for inference +let io = IOBuffer() + run(pipeline(`$(Base.julia_cmd()) --trace-compile=stderr -e 'f() = sin(1.) == 0. ? 1 : 0; exit(f())'`, stderr=io)) + @test isempty(String(take!(io))) +end + finish_precompile_test!() From a58dc526213615ddf4141fece114f24f5c32614e Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Thu, 10 Jul 2025 18:21:44 -0400 Subject: [PATCH 513/662] clear up `isdone` docstring (#58958) I got pretty confused on my first reading of this docstring because for some reason I thought it was saying that `isdone(itr, state) == missing` implied that it was true that `iterate(itr, state) === nothing` (aka that `state` is indeed final). which of course is wrong and doesn't make sense, but it's still how I read it. I think the new docstring is a bit more explicit. --- base/essentials.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index ed7d7cae1586c..82fd3d45c0429 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1225,8 +1225,9 @@ Stateful iterators that want to opt into this feature should define an `isdone` method that returns true/false depending on whether the iterator is done or not. Stateless iterators need not implement this function. -If the result is `missing`, callers may go ahead and compute -`iterate(x, state) === nothing` to compute a definite answer. +If the result is `missing`, then `isdone` cannot determine whether the iterator +state is terminal, and callers must compute `iterate(itr, state) === nothing` +to obtain a definitive answer. See also [`iterate`](@ref), [`isempty`](@ref) """ From 45e0925b67c35cff01b82d730dd850f31eb042c2 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 11 Jul 2025 01:12:37 +0200 Subject: [PATCH 514/662] shield `_artifact_str` function behind a world age barrier (#58957) We already do this for `require` in Base loading, it probably makes sense to do this here as well, as invalidating this function easily adds +1s in load time for a jll. Avoids the big load time penalty from loading IntelOpenMP_jll in https://github.com/JuliaLang/julia/issues/57436#issuecomment-3052258775. Before: ``` julia> @time using ModelingToolkit 6.546844 seconds (16.09 M allocations: 938.530 MiB, 11.13% gc time, 16.35% compilation time: 12% of which was recompilation) ``` After: ``` julia> @time using ModelingToolkit 5.637914 seconds (8.26 M allocations: 533.694 MiB, 11.47% gc time, 3.11% compilation time: 17% of which was recompilation) ``` --------- Co-authored-by: KristofferC Co-authored-by: Cody Tapscott <84105208+topolarity@users.noreply.github.com> --- stdlib/Artifacts/src/Artifacts.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl index f240f6defa0d0..3b20e5feb3da3 100644 --- a/stdlib/Artifacts/src/Artifacts.jl +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -543,6 +543,14 @@ function jointail(dir, tail) end function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, ::Val{LazyArtifacts}) where LazyArtifacts + world = Base._require_world_age[] + if world == typemax(UInt) + world = Base.get_world_counter() + end + return Base.invoke_in_world(world, __artifact_str, __module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, Val(LazyArtifacts))::String +end + +function __artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, ::Val{LazyArtifacts}) where LazyArtifacts pkg = Base.PkgId(__module__) if pkg.uuid !== nothing # Process overrides for this UUID, if we know what it is @@ -565,7 +573,8 @@ function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dic if nameof(LazyArtifacts) in (:Pkg, :Artifacts, :PkgArtifacts) Base.depwarn("using Pkg instead of using LazyArtifacts is deprecated", :var"@artifact_str", force=true) end - return jointail(LazyArtifacts.ensure_artifact_installed(string(name), meta, artifacts_toml; platform), path_tail) + path_base = (@invokelatest LazyArtifacts.ensure_artifact_installed(string(name), meta, artifacts_toml; platform))::String + return jointail(path_base, path_tail) end error("Artifact $(repr(name)) is a lazy artifact; package developers must call `using LazyArtifacts` in $(__module__) before using lazy artifacts.") end From f837bf01dab2f9a553d1751f70b4eb20397fd5f7 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 11 Jul 2025 09:48:34 +0200 Subject: [PATCH 515/662] doc: Fix grammar, typos, and formatting issues across documentation (#58932) Co-authored-by: Claude --- doc/src/base/base.md | 2 +- doc/src/base/collections.md | 2 +- doc/src/base/numbers.md | 2 +- doc/src/base/parallel.md | 2 +- doc/src/base/reflection.md | 2 +- doc/src/base/scopedvalues.md | 2 +- doc/src/devdocs/ast.md | 2 +- doc/src/devdocs/backtraces.md | 2 +- doc/src/devdocs/builtins.md | 2 ++ doc/src/devdocs/callconv.md | 4 ++-- doc/src/devdocs/compiler.md | 2 +- doc/src/devdocs/contributing/formatting.md | 22 ++++++++++---------- doc/src/devdocs/debuggingtips.md | 2 +- doc/src/devdocs/eval.md | 2 +- doc/src/devdocs/gc-sa.md | 4 ++-- doc/src/devdocs/gc.md | 2 +- doc/src/manual/arrays.md | 2 +- doc/src/manual/asynchronous-programming.md | 2 +- doc/src/manual/calling-c-and-fortran-code.md | 8 +++---- doc/src/manual/code-loading.md | 8 +++---- doc/src/manual/command-line-interface.md | 6 +++--- doc/src/manual/distributed-computing.md | 2 +- doc/src/manual/documentation.md | 2 +- 23 files changed, 44 insertions(+), 42 deletions(-) diff --git a/doc/src/base/base.md b/doc/src/base/base.md index f3eb62b3680d5..fafe08a6e5125 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -3,7 +3,7 @@ ## Introduction Julia Base contains a range of functions and macros appropriate for performing -scientific and numerical computing, but is also as broad as those of many general purpose programming +scientific and numerical computing, but is also as broad as those of many general-purpose programming languages. Additional functionality is available from a growing collection of [available packages](https://julialang.org/packages/). Functions are grouped by topic below. diff --git a/doc/src/base/collections.md b/doc/src/base/collections.md index e724930222a13..55cf1ba5dd30d 100644 --- a/doc/src/base/collections.md +++ b/doc/src/base/collections.md @@ -275,7 +275,7 @@ Partially implemented by: * [`Array`](@ref) -## Dequeues +## Deques ```@docs Base.push! diff --git a/doc/src/base/numbers.md b/doc/src/base/numbers.md index aad4e94901054..0bd9d2d4c57d0 100644 --- a/doc/src/base/numbers.md +++ b/doc/src/base/numbers.md @@ -148,7 +148,7 @@ Base.@uint128_str ## [BigFloats and BigInts](@id BigFloats-and-BigInts) -The [`BigFloat`](@ref) and [`BigInt`](@ref) types implements +The [`BigFloat`](@ref) and [`BigInt`](@ref) types implement arbitrary-precision floating point and integer arithmetic, respectively. For [`BigFloat`](@ref) the [GNU MPFR library](https://www.mpfr.org/) is used, and for [`BigInt`](@ref) the [GNU Multiple Precision Arithmetic Library (GMP)] diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index cd5c95f17994a..e382e8edc56ee 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -159,5 +159,5 @@ non-atomic assignment of `ev.task`) In this example, `notify(ev::OneWayEvent)` is allowed to call `schedule(ev.task)` if and only if *it* modifies the state from `OWE_WAITING` to `OWE_NOTIFYING`. This lets us know that the task executing `wait(ev::OneWayEvent)` is now in the `ok` branch and that there cannot be -other tasks that tries to `schedule(ev.task)` since their +other tasks that try to `schedule(ev.task)` since their `@atomicreplace(ev.state, state => OWE_NOTIFYING)` will fail. diff --git a/doc/src/base/reflection.md b/doc/src/base/reflection.md index 3abfac3012d51..16c0d1fadba4e 100644 --- a/doc/src/base/reflection.md +++ b/doc/src/base/reflection.md @@ -142,7 +142,7 @@ For more information see [`@code_lowered`](@ref), [`@code_typed`](@ref), [`@code ### Printing of debug information The aforementioned functions and macros take the keyword argument `debuginfo` that controls the level -debug information printed. +of debug information printed. ```jldoctest; setup = :(using InteractiveUtils), filter = r"int.jl:\d+" julia> InteractiveUtils.@code_typed debuginfo=:source +(1,1) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index d35129642b29b..21d075daf9389 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -25,7 +25,7 @@ provided scoped value taking priority over previous definitions. Let's first look at an example of **lexical** scope. A `let` statement begins a new lexical scope within which the outer definition of `x` is shadowed by -it's inner definition. +its inner definition. ```jldoctest julia> x = 1 diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index 0bd8075a40229..db90a6c87e45d 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -163,7 +163,7 @@ parses as: ``` (if a (block (line 2) b) (elseif (block (line 3) c) (block (line 4) d) - (block (line 6 e)))) + (block (line 6) e))) ``` A `while` loop parses as `(while condition body)`. diff --git a/doc/src/devdocs/backtraces.md b/doc/src/devdocs/backtraces.md index d0533ebe57fcb..7ecfa20f89780 100644 --- a/doc/src/devdocs/backtraces.md +++ b/doc/src/devdocs/backtraces.md @@ -128,4 +128,4 @@ Note that this is only works on Linux. The blog post on [Time Travelling Bug Rep A few terms have been used as shorthand in this guide: * `` refers to the root directory of the Julia source tree; e.g. it should contain folders - such as `base`, `deps`, `src`, `test`, etc..... + such as `base`, `deps`, `src`, `test`, etc. diff --git a/doc/src/devdocs/builtins.md b/doc/src/devdocs/builtins.md index 2f1651b2b2518..ce56a7f9a6b91 100644 --- a/doc/src/devdocs/builtins.md +++ b/doc/src/devdocs/builtins.md @@ -33,7 +33,9 @@ Core.memoryrefsetonce! ## Module bindings +```@docs Core.get_binding_type +``` ## Other diff --git a/doc/src/devdocs/callconv.md b/doc/src/devdocs/callconv.md index 88158cb1eee84..2bf89fb92856b 100644 --- a/doc/src/devdocs/callconv.md +++ b/doc/src/devdocs/callconv.md @@ -18,10 +18,10 @@ signature. * LLVM scalars and vectors are passed by value. * LLVM aggregates (arrays and structs) are passed by reference. -A small return values is returned as LLVM return values. A large return values is returned via +A small return value is returned as LLVM return values. A large return value is returned via the "structure return" (`sret`) convention, where the caller provides a pointer to a return slot. -An argument or return values that is a homogeneous tuple is sometimes represented as an LLVM vector +An argument or return value that is a homogeneous tuple is sometimes represented as an LLVM vector instead of an LLVM array. ## JL Call Convention diff --git a/doc/src/devdocs/compiler.md b/doc/src/devdocs/compiler.md index 8f5f2bb1aa17c..c2ee5f9a375e1 100644 --- a/doc/src/devdocs/compiler.md +++ b/doc/src/devdocs/compiler.md @@ -110,7 +110,7 @@ The general principles are that: - Primitive types get passed in int/float registers. - Tuples of VecElement types get passed in vector registers. - Structs get passed on the stack. -- Return values are handle similarly to arguments, +- Return values are handled similarly to arguments, with a size-cutoff at which they will instead be returned via a hidden sret argument. The total logic for this is implemented by `get_specsig_function` and `deserves_sret`. diff --git a/doc/src/devdocs/contributing/formatting.md b/doc/src/devdocs/contributing/formatting.md index 24459884223d2..e3fc8ec129908 100644 --- a/doc/src/devdocs/contributing/formatting.md +++ b/doc/src/devdocs/contributing/formatting.md @@ -3,20 +3,20 @@ ## General Formatting Guidelines for Julia code contributions - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). - - use whitespace to make the code more readable - - no whitespace at the end of a line (trailing whitespace) - - comments are good, especially when they explain the algorithm - - try to adhere to a 92 character line length limit - - it is generally preferred to use ASCII operators and identifiers over + - Use whitespace to make the code more readable + - No whitespace at the end of a line (trailing whitespace) + - Comments are good, especially when they explain the algorithm + - Try to adhere to a 92 character line length limit + - It is generally preferred to use ASCII operators and identifiers over Unicode equivalents whenever possible - - in docstrings refer to the language as "Julia" and the executable as "`julia`" + - In docstrings refer to the language as "Julia" and the executable as "`julia`" ## General Formatting Guidelines For C code contributions - 4 spaces per indentation level, no tabs - - space between `if` and `(` (`if (x) ...`) - - newline before opening `{` in function definitions + - Space between `if` and `(` (`if (x) ...`) + - Newline before opening `{` in function definitions - `f(void)` for 0-argument function declarations - - newline between `}` and `else` instead of `} else {` - - if one part of an `if..else` chain uses `{ }` then all should - - no whitespace at the end of a line + - Newline between `}` and `else` instead of `} else {` + - If one part of an `if..else` chain uses `{ }` then all should + - No whitespace at the end of a line diff --git a/doc/src/devdocs/debuggingtips.md b/doc/src/devdocs/debuggingtips.md index 0c7ee9d98f046..514045810cd9e 100644 --- a/doc/src/devdocs/debuggingtips.md +++ b/doc/src/devdocs/debuggingtips.md @@ -200,7 +200,7 @@ Expr(:return, Expr(:call, :box, :Float32, Expr(:call, :fptrunc, :Float32, :x)::A ``` Finally, and perhaps most usefully, we can force the function to be recompiled in order to step -through the codegen process. To do this, clear the cached `functionObject` from the `jl_lamdbda_info_t*`: +through the codegen process. To do this, clear the cached `functionObject` from the `jl_lambda_info_t*`: ``` (gdb) p f->linfo->functionObject diff --git a/doc/src/devdocs/eval.md b/doc/src/devdocs/eval.md index 82bedc80116fe..42a5f779a57d6 100644 --- a/doc/src/devdocs/eval.md +++ b/doc/src/devdocs/eval.md @@ -15,7 +15,7 @@ function, and primitive function, before turning into the desired result (hopefu short. * AST - Abstract Syntax Tree The AST is the digital representation of the code structure. In this form + Abstract Syntax Tree. The AST is the digital representation of the code structure. In this form the code has been tokenized for meaning so that it is more suitable for manipulation and execution. diff --git a/doc/src/devdocs/gc-sa.md b/doc/src/devdocs/gc-sa.md index ffbb7451fce5f..89a285fe4ad3f 100644 --- a/doc/src/devdocs/gc-sa.md +++ b/doc/src/devdocs/gc-sa.md @@ -57,7 +57,7 @@ code base to make things work. ## GC Invariants -There is two simple invariants correctness: +There are two simple invariants for correctness: - All `GC_PUSH` calls need to be followed by an appropriate `GC_POP` (in practice we enforce this at the function level) - If a value was previously not rooted at any safepoint, it may no longer be referenced @@ -101,7 +101,7 @@ we place on a given function are indeed correct given the implementation of said ## The analyzer annotations These annotations are found in src/support/analyzer_annotations.h. -The are only active when the analyzer is being used and expand either +They are only active when the analyzer is being used and expand either to nothing (for prototype annotations) or to no-ops (for function like annotations). ### `JL_NOTSAFEPOINT` diff --git a/doc/src/devdocs/gc.md b/doc/src/devdocs/gc.md index a45e8afb271ce..b307a16735e69 100644 --- a/doc/src/devdocs/gc.md +++ b/doc/src/devdocs/gc.md @@ -23,7 +23,7 @@ Julia's pool allocator follows a "tiered" allocation discipline. When requesting - Try to claim a page from `page_pool_lazily_freed`, which contains pages which were empty on the last stop-the-world phase, but not yet madvised by a concurrent sweeper GC thread. -- If it failed claiming a page from `page_pool_lazily_freed`, it will try to claim a page from `the page_pool_clean`, which contains pages which were mmaped on a previous page allocation request but never accessed. +- If it failed claiming a page from `page_pool_lazily_freed`, it will try to claim a page from `page_pool_clean`, which contains pages which were mmaped on a previous page allocation request but never accessed. - If it failed claiming a page from `pool_page_clean` and from `page_pool_lazily_freed`, it will try to claim a page from `page_pool_freed`, which contains pages which have already been madvised by a concurrent sweeper GC thread and whose underlying virtual address can be recycled. diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index 6fa27ba75c90f..ba2d261301b40 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -103,7 +103,7 @@ same type, then that is its `eltype`. If they all have a common [promotion type](@ref conversion-and-promotion) then they get converted to that type using [`convert`](@ref) and that type is the array's `eltype`. Otherwise, a heterogeneous array that can hold anything — a `Vector{Any}` — is constructed; this includes the literal `[]` -where no arguments are given. [Array literal can be typed](@ref man-array-typed-literal) with +where no arguments are given. [Array literals can be typed](@ref man-array-typed-literal) with the syntax `T[A, B, C, ...]` where `T` is a type. ```jldoctest diff --git a/doc/src/manual/asynchronous-programming.md b/doc/src/manual/asynchronous-programming.md index ccdffda9a1acc..eccd924aec6b3 100644 --- a/doc/src/manual/asynchronous-programming.md +++ b/doc/src/manual/asynchronous-programming.md @@ -162,7 +162,7 @@ constructors to explicitly link a set of channels with a set of producer/consume ### More on Channels -A channel can be visualized as a pipe, i.e., it has a write end and a read end : +A channel can be visualized as a pipe, i.e., it has a write end and a read end: * Multiple writers in different tasks can write to the same channel concurrently via [`put!`](@ref) calls. diff --git a/doc/src/manual/calling-c-and-fortran-code.md b/doc/src/manual/calling-c-and-fortran-code.md index aa317468b0f75..bbc3ccdeb6fc0 100644 --- a/doc/src/manual/calling-c-and-fortran-code.md +++ b/doc/src/manual/calling-c-and-fortran-code.md @@ -69,7 +69,7 @@ julia> unsafe_string(path) In practice, especially when providing reusable functionality, one generally wraps `@ccall` uses in Julia functions that set up arguments and then check for errors in whatever manner the -C or Fortran function specifies. And if an error occurs it is thrown as a normal Julia exception. This is especially +C or Fortran function specifies. If an error occurs it is thrown as a normal Julia exception. This is especially important since C and Fortran APIs are notoriously inconsistent about how they indicate error conditions. For example, the `getenv` C library function is wrapped in the following Julia function, which is a simplified version of the actual definition from [`env.jl`](https://github.com/JuliaLang/julia/blob/master/base/env.jl): @@ -224,7 +224,7 @@ julia> A ``` As the example shows, the original Julia array `A` has now been sorted: `[-2.7, 1.3, 3.1, 4.4]`. Note that Julia -[takes care of converting the array to a `Ptr{Cdouble}`](@ref automatic-type-conversion)), computing +[takes care of converting the array to a `Ptr{Cdouble}`](@ref automatic-type-conversion), computing the size of the element type in bytes, and so on. For fun, try inserting a `println("mycompare($a, $b)")` line into `mycompare`, which will allow @@ -357,7 +357,7 @@ an `Int` in Julia). | `unsigned long long` | | `Culonglong` | `UInt64` | | `intmax_t` | | `Cintmax_t` | `Int64` | | `uintmax_t` | | `Cuintmax_t` | `UInt64` | -| `float` | `REAL*4i` | `Cfloat` | `Float32` | +| `float` | `REAL*4` | `Cfloat` | `Float32` | | `double` | `REAL*8` | `Cdouble` | `Float64` | | `complex float` | `COMPLEX*8` | `ComplexF32` | `Complex{Float32}` | | `complex double` | `COMPLEX*16` | `ComplexF64` | `Complex{Float64}` | @@ -1015,7 +1015,7 @@ be a calling convention specifier (the `@ccall` macro currently does not support giving a calling convention). Without any specifier, the platform-default C calling convention is used. Other supported conventions are: `stdcall`, `cdecl`, `fastcall`, and `thiscall` (no-op on 64-bit Windows). For example (from -`base/libc.jl`) we see the same `gethostname``ccall` as above, but with the +`base/libc.jl`) we see the same `gethostname` `ccall` as above, but with the correct signature for Windows: ```julia diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 24e64b0ca068e..567b4699c3cf6 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -36,7 +36,7 @@ An *environment* determines what `import X` and `using X` mean in various code c These can be intermixed to create **a stacked environment**: an ordered set of project environments and package directories, overlaid to make a single composite environment. The precedence and visibility rules then combine to determine which packages are available and where they get loaded from. Julia's load path forms a stacked environment, for example. -These environment each serve a different purpose: +These environments each serve a different purpose: * Project environments provide **reproducibility**. By checking a project environment into version control—e.g. a git repository—along with the rest of the project's source code, you can reproduce the exact state of the project and all of its dependencies. The manifest file, in particular, captures the exact version of every dependency, identified by a cryptographic hash of its source tree, which makes it possible for `Pkg` to retrieve the correct versions and be sure that you are running the exact code that was recorded for all dependencies. * Package directories provide **convenience** when a full carefully-tracked project environment is unnecessary. They are useful when you want to put a set of packages somewhere and be able to directly use them, without needing to create a project environment for them. @@ -123,7 +123,7 @@ This manifest file describes a possible complete dependency graph for the `App` - There are two different packages named `Priv` that the application uses. It uses a private package, which is a root dependency, and a public one, which is an indirect dependency through `Pub`. These are differentiated by their distinct UUIDs, and they have different deps: * The private `Priv` depends on the `Pub` and `Zebra` packages. * The public `Priv` has no dependencies. -- The application also depends on the `Pub` package, which in turn depends on the public `Priv ` and the same `Zebra` package that the private `Priv` package depends on. +- The application also depends on the `Pub` package, which in turn depends on the public `Priv` and the same `Zebra` package that the private `Priv` package depends on. This dependency graph represented as a dictionary, looks like this: @@ -155,7 +155,7 @@ graph[UUID("c07ecb7d-0dc9-4db7-8803-fadaaeaf08e1")][:Priv] and gets `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`, which indicates that in the context of the `Pub` package, `import Priv` refers to the public `Priv` package, rather than the private one which the app depends on directly. This is how the name `Priv` can refer to different packages in the main project than it does in one of its package's dependencies, which allows for duplicate names in the package ecosystem. -What happens if `import Zebra` is evaluated in the main `App` code base? Since `Zebra` does not appear in the project file, the import will fail even though `Zebra` *does* appear in the manifest file. Moreover, if `import Zebra` occurs in the public `Priv` package—the one with UUID `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`—then that would also fail since that `Priv` package has no declared dependencies in the manifest file and therefore cannot load any packages. The `Zebra` package can only be loaded by packages for which it appear as an explicit dependency in the manifest file: the `Pub` package and one of the `Priv` packages. +What happens if `import Zebra` is evaluated in the main `App` code base? Since `Zebra` does not appear in the project file, the import will fail even though `Zebra` *does* appear in the manifest file. Moreover, if `import Zebra` occurs in the public `Priv` package—the one with UUID `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`—then that would also fail since that `Priv` package has no declared dependencies in the manifest file and therefore cannot load any packages. The `Zebra` package can only be loaded by packages for which it appears as an explicit dependency in the manifest file: the `Pub` package and one of the `Priv` packages. **The paths map** of a project environment is extracted from the manifest file. The path of a package `uuid` named `X` is determined by these rules (in order): @@ -191,7 +191,7 @@ paths = Dict( # Priv – the public one: (UUID("2d15fe94-a1f7-436c-a4d8-07a9a496e01c"), :Priv) => # package installed in the system depot: - "/usr/local/julia/packages/Priv/HDkr/src/Priv.jl", + "/usr/local/julia/packages/Priv/HDkrT/src/Priv.jl", # Pub: (UUID("c07ecb7d-0dc9-4db7-8803-fadaaeaf08e1"), :Pub) => # package installed in the user depot: diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 6cbb215a2ded5..46b002b9c2ba4 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -41,7 +41,7 @@ See also [Scripting](@ref man-scripting) for more information on writing Julia s ## The `Main.main` entry point -As of Julia, 1.11, `Base` exports the macro `@main`. This macro expands to the symbol `main`, +As of Julia 1.11, `Base` exports the macro `@main`. This macro expands to the symbol `main`, but at the conclusion of executing a script or expression, `julia` will attempt to execute `Main.main(Base.ARGS)` if such a function `Main.main` has been defined and this behavior was opted into by using the `@main` macro. @@ -148,7 +148,7 @@ atreplinit() do repl # ... end ``` -If [`JULIA_DEPOT_PATH`](@ref JULIA_DEPOT_PATH) is set, the startup file should be located there +If [`JULIA_DEPOT_PATH`](@ref JULIA_DEPOT_PATH) is set, the startup file should be located there: `$JULIA_DEPOT_PATH/config/startup.jl`. ## [Command-line switches for Julia](@id command-line-interface) @@ -177,7 +177,7 @@ The following is a complete list of command-line switches available when launchi |`--pkgimages={yes*\|no\|existing}` |Enable or disable usage of native code caching in the form of pkgimages. The `existing` option allows use of existing pkgimages but disallows creation of new ones| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| -|`-m`, `--module [args]` |Run entry point of `Package` (`@main` function) with `args'| +|`-m`, `--module [args]` |Run entry point of `Package` (`@main` function) with `args`| |`-L`, `--load ` |Load `` immediately on all processors| |`-t`, `--threads {auto\|N[,auto\|M]}` |Enable N[+M] threads; N threads are assigned to the `default` threadpool, and if M is specified, M threads are assigned to the `interactive` threadpool; `auto` tries to infer a useful default number of threads to use but the exact behavior might change in the future. Currently sets N to the number of CPUs assigned to this Julia process based on the OS-specific affinity assignment interface if supported (Linux and Windows) or to the number of CPU threads if not supported (MacOS) or if process affinity is not configured, and sets M to 1.| | `--gcthreads=N[,M]` |Use N threads for the mark phase of GC and M (0 or 1) threads for the concurrent sweeping phase of GC. N is set to the number of compute threads and M is set to 0 if unspecified.| diff --git a/doc/src/manual/distributed-computing.md b/doc/src/manual/distributed-computing.md index 873a94ffb2181..9addd491fcf79 100644 --- a/doc/src/manual/distributed-computing.md +++ b/doc/src/manual/distributed-computing.md @@ -197,7 +197,7 @@ loaded From worker 2: loaded ``` -As usual, this does not bring `DummyModule` into scope on any of the process, which requires +As usual, this does not bring `DummyModule` into scope on any of the processes, which requires [`using`](@ref) or [`import`](@ref). Moreover, when `DummyModule` is brought into scope on one process, it is not on any other: diff --git a/doc/src/manual/documentation.md b/doc/src/manual/documentation.md index 58f2de1fe2680..dff410c74aa2e 100644 --- a/doc/src/manual/documentation.md +++ b/doc/src/manual/documentation.md @@ -34,7 +34,7 @@ The basic syntax is simple: any string appearing just before an object the documented object. Here is a basic example: ```julia -"Tell whether there are too foo items in the array." +"Tell whether there are too many foo items in the array." foo(xs::Array) = ... ``` !!! note "Reminder" From 3847ff1434f7b5f5bd0cdc31af4bffba1b529c80 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 11 Jul 2025 02:46:55 -0700 Subject: [PATCH 516/662] Replace Base.Workqueues with a OncePerThread (#58941) Simplify `workqueue_for`. While not strictly necessary, the acquire load in `getindex(once::OncePerThread{T,F}, tid::Integer)` makes ThreadSanitizer happy. With the existing implementation, we get false positives whenever a thread other than the one that originally allocated the array reads it: ``` ================== WARNING: ThreadSanitizer: data race (pid=6819) Atomic read of size 8 at 0xffff86bec058 by main thread: #0 getproperty Base_compiler.jl:57 (sys.so+0x113b478) #1 julia_pushNOT._1925 task.jl:868 (sys.so+0x113b478) #2 julia_enq_work_1896 task.jl:969 (sys.so+0x5cd218) #3 schedule task.jl:983 (sys.so+0x892294) #4 macro expansion threadingconstructs.jl:522 (sys.so+0x892294) #5 julia_start_profile_listener_60681 Base.jl:355 (sys.so+0x892294) #6 julia___init___60641 Base.jl:392 (sys.so+0x1178dc) #7 jfptr___init___60642 (sys.so+0x118134) #8 _jl_invoke /home/user/c/julia/src/gf.c (libjulia-internal.so.1.13+0x5e9a4) #9 ijl_apply_generic /home/user/c/julia/src/gf.c:3892:12 (libjulia-internal.so.1.13+0x5e9a4) #10 jl_apply /home/user/c/julia/src/julia.h:2343:12 (libjulia-internal.so.1.13+0xbba74) #11 jl_module_run_initializer /home/user/c/julia/src/toplevel.c:68:13 (libjulia-internal.so.1.13+0xbba74) #12 _finish_jl_init_ /home/user/c/julia/src/init.c:632:13 (libjulia-internal.so.1.13+0x9c0fc) #13 ijl_init_ /home/user/c/julia/src/init.c:783:5 (libjulia-internal.so.1.13+0x9bcf4) #14 jl_repl_entrypoint /home/user/c/julia/src/jlapi.c:1125:5 (libjulia-internal.so.1.13+0xf7ec8) #15 jl_load_repl /home/user/c/julia/cli/loader_lib.c:601:12 (libjulia.so.1.13+0x11934) #16 main /home/user/c/julia/cli/loader_exe.c:58:15 (julia+0x10dc20) Previous write of size 8 at 0xffff86bec058 by thread T2: #0 IntrusiveLinkedListSynchronized task.jl:863 (sys.so+0x78d220) #1 macro expansion task.jl:932 (sys.so+0x78d220) #2 macro expansion lock.jl:376 (sys.so+0x78d220) #3 julia_workqueue_for_1933 task.jl:924 (sys.so+0x78d220) #4 julia_wait_2048 task.jl:1204 (sys.so+0x6255ac) #5 julia_task_done_hook_49205 task.jl:839 (sys.so+0x128fdc0) #6 jfptr_task_done_hook_49206 (sys.so+0x902218) #7 _jl_invoke /home/user/c/julia/src/gf.c (libjulia-internal.so.1.13+0x5e9a4) #8 ijl_apply_generic /home/user/c/julia/src/gf.c:3892:12 (libjulia-internal.so.1.13+0x5e9a4) #9 jl_apply /home/user/c/julia/src/julia.h:2343:12 (libjulia-internal.so.1.13+0x9c79c) #10 jl_finish_task /home/user/c/julia/src/task.c:345:13 (libjulia-internal.so.1.13+0x9c79c) #11 jl_threadfun /home/user/c/julia/src/scheduler.c:122:5 (libjulia-internal.so.1.13+0xe7db8) Thread T2 (tid=6824, running) created by main thread at: #0 pthread_create (julia+0x85f88) #1 uv_thread_create_ex /workspace/srcdir/libuv/src/unix/thread.c:172 (libjulia-internal.so.1.13+0x1a8d70) #2 _finish_jl_init_ /home/user/c/julia/src/init.c:618:5 (libjulia-internal.so.1.13+0x9c010) #3 ijl_init_ /home/user/c/julia/src/init.c:783:5 (libjulia-internal.so.1.13+0x9bcf4) #4 jl_repl_entrypoint /home/user/c/julia/src/jlapi.c:1125:5 (libjulia-internal.so.1.13+0xf7ec8) #5 jl_load_repl /home/user/c/julia/cli/loader_lib.c:601:12 (libjulia.so.1.13+0x11934) #6 main /home/user/c/julia/cli/loader_exe.c:58:15 (julia+0x10dc20) SUMMARY: ThreadSanitizer: data race Base_compiler.jl:57 in getproperty ================== ``` --- base/task.jl | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/base/task.jl b/base/task.jl index 7e7e5dfc3076b..1ae4fcc8d22f0 100644 --- a/base/task.jl +++ b/base/task.jl @@ -909,31 +909,10 @@ function list_deletefirst!(W::IntrusiveLinkedListSynchronized{T}, t::T) where T end const StickyWorkqueue = IntrusiveLinkedListSynchronized{Task} -global Workqueues::Vector{StickyWorkqueue} = [StickyWorkqueue()] -const Workqueues_lock = Threads.SpinLock() +const Workqueues = OncePerThread{StickyWorkqueue}(StickyWorkqueue) const Workqueue = Workqueues[1] # default work queue is thread 1 // TODO: deprecate this variable -function workqueue_for(tid::Int) - qs = Workqueues - if length(qs) >= tid && isassigned(qs, tid) - return @inbounds qs[tid] - end - # slow path to allocate it - @assert tid > 0 - l = Workqueues_lock - @lock l begin - qs = Workqueues - if length(qs) < tid - nt = Threads.maxthreadid() - @assert tid <= nt - global Workqueues = qs = copyto!(typeof(qs)(undef, length(qs) + nt - 1), qs) - end - if !isassigned(qs, tid) - @inbounds qs[tid] = StickyWorkqueue() - end - return @inbounds qs[tid] - end -end +workqueue_for(tid::Int) = Workqueues[tid] function enq_work(t::Task) (t._state === task_state_runnable && t.queue === nothing) || error("schedule: Task not runnable") From ff33305061cf5ea910631c4ab83a455f47eed774 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Fri, 11 Jul 2025 04:23:40 -0700 Subject: [PATCH 517/662] Fix `hygienic-scope`s in inner macro expansions (#58965) Changes from https://github.com/JuliaLang/julia/pull/43151, github just didn't want me to re-open it. As discussed on slack, any `hygienic-scope` within an outer `hygienic-scope` can read and write variables in the outer one, so it's not particularly hygienic. The result is that we can't safely nest macro calls unless they know the contents of all inner macro calls. Should fix #48910. Co-authored-by: Michiel Dral --- src/macroexpand.scm | 2 +- test/core.jl | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/macroexpand.scm b/src/macroexpand.scm index f67145317dc7a..2990d98eefe6e 100644 --- a/src/macroexpand.scm +++ b/src/macroexpand.scm @@ -513,7 +513,7 @@ (body (cadr e)) (m (caddr e)) (lno (cdddr e))) - (resolve-expansion-vars-with-new-env body env m lno parent-scope inarg #t))) + (resolve-expansion-vars-with-new-env body '() m lno parent-scope inarg #t))) ((tuple) (cons (car e) (map (lambda (x) diff --git a/test/core.jl b/test/core.jl index a824c0abb674a..4af61233ac458 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2280,6 +2280,31 @@ end x6074 = 6074 @test @X6074() == 6074 +# issues #48910, 54417 +macro X43151_nested() + quote my_value = "from_nested_macro" end +end +macro X43151_parent() + quote + my_value = "from_parent_macro" + @X43151_nested() + my_value + end +end +@test @X43151_parent() == "from_parent_macro" + +macro X43151_nested_escaping() + quote $(esc(:my_value)) = "from_nested_macro" end +end +macro X43151_parent_escaping() + quote + my_value = "from_parent_macro" + @X43151_nested_escaping() + my_value + end +end +@test @X43151_parent_escaping() == "from_nested_macro" + # issue #5536 test5536(a::Union{Real, AbstractArray}...) = "Splatting" test5536(a::Union{Real, AbstractArray}) = "Non-splatting" From d7bdd2184d6c94e553795c68bc4f926f958e8cf0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 11 Jul 2025 13:59:20 -0400 Subject: [PATCH 518/662] remove comment from julia-syntax that is no longer true (#58964) The code this referred to was removed by c6c3d72d1cbddb3d27e0df0e739bb27dd709a413 --- src/julia-syntax.scm | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index ad78221c540c5..a73044a228b5a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -545,10 +545,7 @@ ;; call with keyword args pre-sorted - original method code goes here ,(method-def-expr- mangled sparams - `((|::| ,mangled (call (core typeof) ,mangled)) ,@vars ,@restkw - ;; strip type off function self argument if not needed for a static param. - ;; then it is ok for cl-convert to move this definition above the original def. - ,@not-optional ,@vararg) + `((|::| ,mangled (call (core typeof) ,mangled)) ,@vars ,@restkw ,@not-optional ,@vararg) (insert-after-meta `(block ,@stmts) (cons `(meta nkw ,(+ (length vars) (length restkw))) From 49b08a3424dd8a2a7f4315f6011359e912216b18 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Fri, 11 Jul 2025 18:03:26 -0400 Subject: [PATCH 519/662] expand memoryrefnew capabilities (#58768) The goal here is 2-fold. Firstly, this should let us simplify the boundscheck (not yet implimented), but this also should reduce Julia IR side a bit. --- Compiler/src/optimize.jl | 4 ++- Compiler/src/tfuncs.jl | 5 ++-- Compiler/test/inference.jl | 9 ++++--- base/boot.jl | 2 +- base/essentials.jl | 4 +-- src/builtins.c | 31 ++++++++++++++++++------ src/cgutils.cpp | 39 ++++++++++++++++++++++++++++++ src/codegen.cpp | 17 ++++++++++--- stdlib/REPL/test/precompilation.jl | 3 ++- 9 files changed, 92 insertions(+), 22 deletions(-) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index da4a17c5d6913..27fb1e2168639 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -720,7 +720,9 @@ function iscall_with_boundscheck(@nospecialize(stmt), sv::PostOptAnalysisState) f === nothing && return false if f === getfield nargs = 4 - elseif f === memoryrefnew || f === memoryrefget || f === memoryref_isassigned + elseif f === memoryrefnew + nargs= 3 + elseif f === memoryrefget || f === memoryref_isassigned nargs = 4 elseif f === memoryrefset! nargs = 5 diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index e42afb065687c..fb173759ab122 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -2096,6 +2096,7 @@ end @nospecs function memoryref_tfunc(𝕃::AbstractLattice, ref, idx, boundscheck) memoryref_builtin_common_errorcheck(ref, Const(:not_atomic), boundscheck) || return Bottom hasintersect(widenconst(idx), Int) || return Bottom + hasintersect(widenconst(ref), GenericMemory) && return memoryref_tfunc(𝕃, ref) return ref end add_tfunc(memoryrefnew, 1, 3, memoryref_tfunc, 1) @@ -2107,7 +2108,7 @@ end add_tfunc(memoryrefoffset, 1, 1, memoryrefoffset_tfunc, 5) @nospecs function memoryref_builtin_common_errorcheck(mem, order, boundscheck) - hasintersect(widenconst(mem), GenericMemoryRef) || return false + hasintersect(widenconst(mem), Union{GenericMemory, GenericMemoryRef}) || return false hasintersect(widenconst(order), Symbol) || return false hasintersect(widenconst(unwrapva(boundscheck)), Bool) || return false return true @@ -2203,7 +2204,7 @@ function memoryref_builtin_common_nothrow(argtypes::Vector{Any}) idx = widenconst(argtypes[2]) idx ⊑ Int || return false boundscheck ⊑ Bool || return false - memtype ⊑ GenericMemoryRef || return false + memtype ⊑ Union{GenericMemory, GenericMemoryRef} || return false # If we have @inbounds (last argument is false), we're allowed to assume # we don't throw bounds errors. if isa(boundscheck, Const) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 21d0f079b91d7..58f2231319af9 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -1597,9 +1597,12 @@ let memoryref_tfunc(@nospecialize xs...) = Compiler.memoryref_tfunc(Compiler.fal @test memoryref_tfunc(MemoryRef{Int}, Int, Symbol) == Union{} @test memoryref_tfunc(MemoryRef{Int}, Int, Bool) == MemoryRef{Int} @test memoryref_tfunc(MemoryRef{Int}, Int, Vararg{Bool}) == MemoryRef{Int} - @test memoryref_tfunc(Memory{Int}, Int) == Union{} - @test memoryref_tfunc(Any, Any, Any) == Any # also probably could be GenericMemoryRef - @test memoryref_tfunc(Any, Any) == Any # also probably could be GenericMemoryRef + @test memoryref_tfunc(Memory{Int}, Int) == MemoryRef{Int} + @test memoryref_tfunc(Memory{Int}, Int, Symbol) == Union{} + @test memoryref_tfunc(Memory{Int}, Int, Bool) == MemoryRef{Int} + @test memoryref_tfunc(Memory{Int}, Int, Vararg{Bool}) == MemoryRef{Int} + @test memoryref_tfunc(Any, Any, Any) == GenericMemoryRef + @test memoryref_tfunc(Any, Any) == GenericMemoryRef @test memoryref_tfunc(Any) == GenericMemoryRef @test memoryrefget_tfunc(MemoryRef{Int}, Symbol, Bool) === Int @test memoryrefget_tfunc(MemoryRef{Int}, Any, Any) === Int diff --git a/base/boot.jl b/base/boot.jl index f1ef8bd37a276..d055c47516f91 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -592,7 +592,7 @@ const undef = UndefInitializer() # memoryref is simply convenience wrapper function around memoryrefnew memoryref(mem::GenericMemory) = memoryrefnew(mem) -memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(memoryrefnew(mem), Int(i), @_boundscheck) +memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(mem, Int(i), @_boundscheck) memoryref(ref::GenericMemoryRef, i::Integer) = memoryrefnew(ref, Int(i), @_boundscheck) GenericMemoryRef(mem::GenericMemory) = memoryref(mem) GenericMemoryRef(mem::GenericMemory, i::Integer) = memoryref(mem, i) diff --git a/base/essentials.jl b/base/essentials.jl index 82fd3d45c0429..f27346c4b43cb 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -395,7 +395,7 @@ default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic function getindex(A::GenericMemory, i::Int) @_noub_if_noinbounds_meta (@_boundscheck) && checkbounds(A, i) - memoryrefget(memoryrefnew(memoryrefnew(A), i, false), default_access_order(A), false) + memoryrefget(memoryrefnew(A, i, false), default_access_order(A), false) end getindex(A::GenericMemoryRef) = memoryrefget(A, default_access_order(A), @_boundscheck) @@ -973,7 +973,7 @@ function setindex!(A::Array{Any}, @nospecialize(x), i::Int) memoryrefset!(memoryrefnew(getfield(A, :ref), i, false), x, :not_atomic, false) return A end -setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryrefnew(memoryrefnew(A), i, @_boundscheck), x, :not_atomic, @_boundscheck); A) +setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryrefnew(A, i, @_boundscheck), x, :not_atomic, @_boundscheck); A) setindex!(A::MemoryRef{T}, x) where {T} = (memoryrefset!(A, convert(T, x), :not_atomic, @_boundscheck); A) setindex!(A::MemoryRef{Any}, @nospecialize(x)) = (memoryrefset!(A, x, :not_atomic, @_boundscheck); A) diff --git a/src/builtins.c b/src/builtins.c index 46fceb91f86e7..79953001662fd 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1765,30 +1765,45 @@ JL_CALLABLE(jl_f_memoryrefnew) return (jl_value_t*)jl_new_memoryref(typ, m, m->ptr); } else { - JL_TYPECHK(memoryrefnew, genericmemoryref, args[0]); JL_TYPECHK(memoryrefnew, long, args[1]); if (nargs == 3) JL_TYPECHK(memoryrefnew, bool, args[2]); + size_t i = (size_t) jl_unbox_long(args[1]) - 1; + char *data; + if (jl_is_genericmemory(args[0])) { + jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; + jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(((jl_datatype_t*)jl_typetagof(m))->parameters), 3); + JL_GC_PROMISE_ROOTED(typ); // it is a concrete type + if (i >= m->length) + jl_bounds_error((jl_value_t*)m, args[1]); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->flags.arrayelem_isunion || layout->size == 0) + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)i); + else if (layout->flags.arrayelem_isboxed) + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)m->ptr + sizeof(jl_value_t*)*i); + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)m->ptr + layout->size*i); + } + JL_TYPECHK(memoryrefnew, genericmemoryref, args[0]); jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)args[0]; - size_t i = jl_unbox_long(args[1]) - 1; - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m->mem))->layout; - char *data = (char*)m->ptr_or_offset; + jl_genericmemory_t *mem = m->mem; + data = (char*)m->ptr_or_offset; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout; if (layout->flags.arrayelem_isboxed) { - if (((data - (char*)m->mem->ptr) / sizeof(jl_value_t*)) + i >= m->mem->length) + if (((data - (char*)mem->ptr) / sizeof(jl_value_t*)) + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += sizeof(jl_value_t*) * i; } else if (layout->flags.arrayelem_isunion || layout->size == 0) { - if ((size_t)data + i >= m->mem->length) + if ((size_t)data + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += i; } else { - if (((data - (char*)m->mem->ptr) / layout->size) + i >= m->mem->length) + if (((data - (char*)mem->ptr) / layout->size) + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += layout->size * i; } - return (jl_value_t*)jl_new_memoryref((jl_value_t*)jl_typetagof(m), m->mem, data); + return (jl_value_t*)jl_new_memoryref((jl_value_t*)jl_typetagof(m), mem, data); } } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 7c2f62ac3350c..ef36eeb728689 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4684,6 +4684,45 @@ static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &mem, cons return _emit_memoryref(ctx, boxed(ctx, mem), data, layout, typ); } +static jl_cgval_t emit_memoryref_direct(jl_codectx_t &ctx, const jl_cgval_t &mem, jl_cgval_t idx, jl_value_t *typ, jl_value_t *inbounds, const jl_datatype_layout_t *layout) +{ + bool isboxed = layout->flags.arrayelem_isboxed; + bool isunion = layout->flags.arrayelem_isunion; + bool isghost = layout->size == 0; + Value *boxmem = boxed(ctx, mem); + Value *i = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); + Value *idx0 = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); + bool bc = bounds_check_enabled(ctx, inbounds); + if (bc) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); + Value *mlen = emit_genericmemorylen(ctx, boxmem, typ); + Value *inbound = ctx.builder.CreateICmpULT(idx0, mlen); + setName(ctx.emission_context, inbound, "memoryref_isinbounds"); + ctx.builder.CreateCondBr(inbound, endBB, failBB); + failBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), + { mark_callee_rooted(ctx, boxmem), i }); + ctx.builder.CreateUnreachable(); + endBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(endBB); + } + Value *data; + + if ((!isboxed && isunion) || isghost) { + data = idx0; + + } else { + data = emit_genericmemoryptr(ctx, boxmem, layout, 0); + Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(typ)); + data = ctx.builder.CreateInBoundsGEP(elty, data, idx0); + } + + return _emit_memoryref(ctx, boxmem, data, layout, typ); +} + static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) { if (!ref.inline_roots.empty()) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 88b6e9f1f4513..2c38ea4e5d907 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4138,16 +4138,25 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryrefnew) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); - if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { - mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { + mty_dt = (jl_datatype_t*)jl_field_type_concrete(mty_dt, 1); + const jl_datatype_layout_t *layout = mty_dt->layout; jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); return true; } + if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { + const jl_datatype_layout_t *layout = mty_dt->layout; + jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; + if (nargs == 3) + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); + jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(mty_dt->parameters), jl_svec_len(mty_dt->parameters)); + *ret = emit_memoryref_direct(ctx, ref, argv[2], typ, boundscheck, layout); + return true; + } } else if (f == BUILTIN(memoryrefoffset) && nargs == 1) { diff --git a/stdlib/REPL/test/precompilation.jl b/stdlib/REPL/test/precompilation.jl index 7efcf0b5e8282..fcd6f9aa1ae2b 100644 --- a/stdlib/REPL/test/precompilation.jl +++ b/stdlib/REPL/test/precompilation.jl @@ -33,7 +33,8 @@ if !Sys.iswindows() # given this test checks that startup is snappy, it's best to add workloads to # contrib/generate_precompile.jl rather than increase this number. But if that's not # possible, it'd be helpful to add a comment with the statement and a reason below - expected_precompiles = 0 + expected_precompiles = 1 + # Jameson told me to bump this n_precompiles = count(r"precompile\(", tracecompile_out) From 6d1635108a8532c565e9034a955a708c56bc8604 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Fri, 11 Jul 2025 22:08:30 -0300 Subject: [PATCH 520/662] Add news entry and update docstring for #58727 (#58973) --- NEWS.md | 1 + stdlib/Test/src/Test.jl | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6a3f67ff063bd..5d9bf83467b77 100644 --- a/NEWS.md +++ b/NEWS.md @@ -76,6 +76,7 @@ Standard library changes #### Test * Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). +* Transparent test sets (`@testset let`) now show context when tests error ([#58727]). #### InteractiveUtils diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 8278fed697f7b..43855c64dfd0c 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1675,7 +1675,7 @@ of the `begin/end` case (as if used for each loop iteration). # `@testset let` When `@testset let` is used, the macro starts a *transparent* test set with -the given object added as a context object to any failing test contained +the given object added as a context object to any failing or erroring test contained therein. This is useful when performing a set of related tests on one larger object and it is desirable to print this larger object when any of the individual tests fail. Transparent test sets do not introduce additional levels @@ -1688,6 +1688,9 @@ parent test set (with the context object appended to any failing tests.) !!! compat "Julia 1.10" Multiple `let` assignments are supported since Julia 1.10. +!!! compat "Julia 1.13" + Context is shown when a test errors since Julia 1.13. + # Special implicit world age increment for `@testset begin` World age inside `@testset begin` increments implicitly after every statement. From 338f494fc221e166eb1455afa9f1f68fc50d94c2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 12 Jul 2025 00:32:43 -0400 Subject: [PATCH 521/662] Fix alignment of failed precompile jobs on CI (#58971) --- base/precompilation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index c51297ee2a791..573b769094fb6 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -983,7 +983,7 @@ function _precompilepkgs(pkgs::Vector{String}, delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report failed_deps[pkg_config] = (strict || is_project_dep) ? string(sprint(showerror, err), "\n", strip(errmsg)) : "" !fancyprint && @lock print_lock begin - println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + println(io, " "^12, color_string(" ✗ ", Base.error_color()), name) end else rethrow() From fb59b6d93475b1eee5dd276b5b57d138ca5071a0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 12 Jul 2025 03:03:25 -0400 Subject: [PATCH 522/662] bpart: Tweak `isdefinedglobal` on backdated constant (#58976) In d2cc06193ef4161e4ac161bd4b5b57a51686a89a and prior commits, we made backdated access a conditional error (if depwarns are enabled or in generators). However, we did not touch `isdefinedglobal`. This resulted in the common pattern `isdefinedglobal(m, s) && getglobal(m, s)` to sometimes error. In particular, this could be observed when attempting to print a type from inside a generated function before that type's definition age. Additionally, I think the usage there, which used `invokelatest` on each of the two queries is problematic because it is racy, since the two `invokelatest` calls may be looking at different world ages. This makes two tweaks: 1. Makes `isdefinedglobal` consistent with `getglobal` in that it now returns false if `getglobal` would throw due to the above referenced restriction. 2. Removes the implicit `invokelatest` in _isself in the show code. Instead, it will use the current world. I considered having it use the exception age when used for MethodErrors. However, because this is used for printing it matters more how the object can be accessed *now* rather than how it could have been accessed in the past. --- base/show.jl | 2 +- src/module.c | 10 +++++++--- test/worlds.jl | 8 +++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/base/show.jl b/base/show.jl index f89e72ff3ab53..870b2f47fe4a6 100644 --- a/base/show.jl +++ b/base/show.jl @@ -42,7 +42,7 @@ function _isself(ft::DataType) name = ftname.singletonname ftname.name === name && return false mod = parentmodule(ft) - return invokelatest(isdefinedglobal, mod, name) && ft === typeof(invokelatest(getglobal, mod, name)) + return isdefinedglobal(mod, name) && ft === typeof(getglobal(mod, name)) end function show(io::IO, ::MIME"text/plain", f::Function) diff --git a/src/module.c b/src/module.c index 03d82c66fbf51..465a65c840db9 100644 --- a/src/module.c +++ b/src/module.c @@ -1477,10 +1477,14 @@ JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // u } else { jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); } - if (jl_bkind_is_some_guard(jl_binding_kind(bpart))) + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (jl_bkind_is_some_guard(kind)) return 0; - if (jl_bkind_is_defined_constant(jl_binding_kind(bpart))) { - // N.B.: No backdated check for isdefined + if (jl_bkind_is_defined_constant(kind)) { + if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST)) { + return !(jl_current_task->ptls->in_pure_callback || jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR); + } + // N.B.: No backdated admonition for isdefined return 1; } return jl_atomic_load(&b->value) != NULL; diff --git a/test/worlds.jl b/test/worlds.jl index 5c9510e67d2a1..e1e86882306e4 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -557,7 +557,13 @@ struct FooBackdated FooBackdated() = new(FooBackdated[]) end -@test Base.invoke_in_world(before_backdate_age, isdefined, @__MODULE__, :FooBackdated) +# For depwarn == 1, this throws a warning on access, for depwarn == 2, it throws an error. +# `isdefinedglobal` changes with that, but doesn't error. +if Base.JLOptions().depwarn <= 1 + @test Base.invoke_in_world(before_backdate_age, isdefinedglobal, @__MODULE__, :FooBackdated) +else + @test !Base.invoke_in_world(before_backdate_age, isdefinedglobal, @__MODULE__, :FooBackdated) +end # Test that ambiguous binding intersect the using'd binding's world ranges module AmbigWorldTest From 107e1acf5fe79e99a39a57b2aae7045bb5bfcf72 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 12 Jul 2025 06:47:59 -0400 Subject: [PATCH 523/662] Fix precompilepkgs warn loaded setting (#58978) --- base/precompilation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 573b769094fb6..e737faa210c79 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -920,7 +920,7 @@ function _precompilepkgs(pkgs::Vector{String}, flags, cacheflags = config task = @async begin try - loaded = haskey(Base.loaded_modules, pkg) + loaded = warn_loaded && haskey(Base.loaded_modules, pkg) for dep in deps # wait for deps to finish wait(was_processed[(dep,config)]) end From 3f9ba7ff1608da0bd1c3724e0b8f5db50aa6e7c0 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Sat, 12 Jul 2025 08:27:48 -0400 Subject: [PATCH 524/662] specify that `Iterators.rest` must be given a valid `state` (#58962) ~currently `Iterators.rest(1:2, 3)` creates an infinite loop. after this PR it would be an `ArgumentError`~ docs only now --- base/iterators.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/iterators.jl b/base/iterators.jl index d226fbb57aa52..add4c4c0e7bed 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -625,13 +625,19 @@ end """ rest(iter, state) -An iterator that yields the same elements as `iter`, but starting at the given `state`. +An iterator that yields the same elements as `iter`, but starting at the given `state`, which +must be a state obtainable via a sequence of one or more calls to `iterate(iter[, state])` See also: [`Iterators.drop`](@ref), [`Iterators.peel`](@ref), [`Base.rest`](@ref). # Examples ```jldoctest -julia> collect(Iterators.rest([1,2,3,4], 2)) +julia> iter = [1,2,3,4]; + +julia> val, state = iterate(iter) +(1, 2) + +julia> collect(Iterators.rest(iter, state)) 3-element Vector{Int64}: 2 3 From 01b1348df16d07acc29b991d1e06fefdcca6be51 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 12 Jul 2025 17:03:33 -0400 Subject: [PATCH 525/662] stdlib/Dates: Fix doctest regex to handle DateTime with 0 microseconds (#58981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `now(UTC)` doctest can fail when the DateTime has exactly 0 milliseconds, as the output format omits the fractional seconds entirely (e.g., "2023-01-04T10:52:24" instead of "2023-01-04T10:52:24.000"). Update the regex filter to make the milliseconds portion optional by using `(\\.\\d{3})?` instead of `\\.\\d{3}`. Fixes CI failure: https://buildkite.com/julialang/julia-master/builds/49144#0197fd72-d1c6-44d6-9c59-5f548ab98f04 🤖 Generated with [Claude Code](https://claude.ai/code) Co-authored-by: Keno Fischer Co-authored-by: Claude --- stdlib/Dates/src/conversions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Dates/src/conversions.jl b/stdlib/Dates/src/conversions.jl index a68b2a85e9c87..65df4c06b64db 100644 --- a/stdlib/Dates/src/conversions.jl +++ b/stdlib/Dates/src/conversions.jl @@ -85,7 +85,7 @@ Return a `DateTime` corresponding to the user's system time as UTC/GMT. For other time zones, see the TimeZones.jl package. # Examples -```jldoctest; filter = r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}" => "2023-01-04T10:52:24.864" +```jldoctest; filter = r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?" => "2023-01-04T10:52:24.864" julia> now(UTC) 2023-01-04T10:52:24.864 ``` From 6ddb3d64106b05aaaad5dce98de5171f68148670 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Sun, 13 Jul 2025 03:23:59 +0530 Subject: [PATCH 526/662] Fix unique for range wrappers with zero step (#51004) The current implementation assumes that the vector indexing `r[begin:begin]` is specialized to return a range, which isn't the case by default. As a consequence, ```julia julia> struct MyStepRangeLen{T,R} <: AbstractRange{T} x :: R end julia> MyStepRangeLen(s::StepRangeLen{T}) where {T} = MyStepRangeLen{T,typeof(s)}(s) MyStepRangeLen julia> Base.first(s::MyStepRangeLen) = first(s.x) julia> Base.last(s::MyStepRangeLen) = last(s.x) julia> Base.length(s::MyStepRangeLen) = length(s.x) julia> Base.step(s::MyStepRangeLen) = step(s.x) julia> r = MyStepRangeLen(StepRangeLen(1,0,4)) 1:0:1 julia> unique(r) ERROR: MethodError: Cannot `convert` an object of type Vector{Int64} to an object of type MyStepRangeLen{Int64, Int64, StepRangeLen{Int64, Int64, Int64, Int64}} [...] ``` This PR fixes this by using constructing a `UnitRange` instead of the indexing operation. After this, we obtain ```julia julia> unique(r) 1:1:1 ``` In principle, the `step` should be preserved, but `range(r[begin]::Int, step=step(r), length=length(r))` appears to error at present, as it tries to construct a `StepRange` instead of a `StepRangeLen`. This fix isn't perfect as it assumes that the conversion from a `UnitRange` _is_ defined, which is also not the case by default. For example, the following still won't work: ```julia julia> struct MyRange <: AbstractRange{Int} end julia> Base.first(x::MyRange) = 1 julia> Base.last(x::MyRange) = 1 julia> Base.length(x::MyRange) = 3 julia> Base.step(x::MyRange) = 0 julia> unique(MyRange()) ERROR: MethodError: no method matching MyRange(::UnitRange{Int64}) [...] ``` In fact, if the indexing `MyRange()[begin:begin]` has been specialized but the conversion from a `UnitRange` isn't, then this is actually a regression. I'm unsure if such pathological cases are common, though. The reason the first example works is that the conversion for a range wrapper is defined implicitly if the parent type supports conversion from a `UnitRange`. --- base/set.jl | 2 +- test/ranges.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/base/set.jl b/base/set.jl index 38287efe28bee..3c7dc9487ac8a 100644 --- a/base/set.jl +++ b/base/set.jl @@ -259,7 +259,7 @@ _unique_from(itr, out, seen, i) = unique_from(itr, out, seen, i) return out end -unique(r::AbstractRange) = allunique(r) ? r : oftype(r, r[begin:begin]) +unique(r::AbstractRange) = allunique(r) ? r : oftype(r, r[begin]:r[begin]) """ unique(f, itr) diff --git a/test/ranges.jl b/test/ranges.jl index 40dbf6a42dbd9..4a0c6c9ddd65f 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -699,6 +699,18 @@ end @test Duck(4) ∈ Duck(1):Duck(5) @test Duck(0) ∉ Duck(1):Duck(5) end + @testset "unique" begin + struct MyStepRangeLen{T,R} <: AbstractRange{T} + x :: R + end + MyStepRangeLen(s::StepRangeLen{T}) where {T} = MyStepRangeLen{T,typeof(s)}(s) + Base.first(s::MyStepRangeLen) = first(s.x) + Base.last(s::MyStepRangeLen) = last(s.x) + Base.length(s::MyStepRangeLen) = length(s.x) + Base.step(s::MyStepRangeLen) = step(s.x) + sr = StepRangeLen(1,0,4) + @test unique(MyStepRangeLen(sr)) == unique(sr) + end end @testset "indexing range with empty range (#4309)" begin @test (@inferred (3:6)[5:4]) === 7:6 From 1367b3d7ad79d87a6bc0f0aabb4fd05905636c26 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 13 Jul 2025 20:01:59 -0400 Subject: [PATCH 527/662] Docs: add GC user docs (#58733) Co-authored-by: Andy Dienes <51664769+adienes@users.noreply.github.com> Co-authored-by: Gabriel Baraldi <28694980+gbaraldi@users.noreply.github.com> Co-authored-by: Diogo Netto <61364108+d-netto@users.noreply.github.com> --- doc/make.jl | 1 + doc/src/manual/command-line-interface.md | 4 +- doc/src/manual/memory-management.md | 177 +++++++++++++++++++++++ doc/src/manual/multi-threading.md | 6 +- doc/src/manual/performance-tips.md | 2 + 5 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 doc/src/manual/memory-management.md diff --git a/doc/make.jl b/doc/make.jl index bd74316d9fd7e..2604c801f19aa 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -192,6 +192,7 @@ Manual = [ "manual/code-loading.md", "manual/profile.md", "manual/stacktraces.md", + "manual/memory-management.md", "manual/performance-tips.md", "manual/workflow-tips.md", "manual/style-guide.md", diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 46b002b9c2ba4..b8ae83620da50 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -180,7 +180,7 @@ The following is a complete list of command-line switches available when launchi |`-m`, `--module [args]` |Run entry point of `Package` (`@main` function) with `args`| |`-L`, `--load ` |Load `` immediately on all processors| |`-t`, `--threads {auto\|N[,auto\|M]}` |Enable N[+M] threads; N threads are assigned to the `default` threadpool, and if M is specified, M threads are assigned to the `interactive` threadpool; `auto` tries to infer a useful default number of threads to use but the exact behavior might change in the future. Currently sets N to the number of CPUs assigned to this Julia process based on the OS-specific affinity assignment interface if supported (Linux and Windows) or to the number of CPU threads if not supported (MacOS) or if process affinity is not configured, and sets M to 1.| -| `--gcthreads=N[,M]` |Use N threads for the mark phase of GC and M (0 or 1) threads for the concurrent sweeping phase of GC. N is set to the number of compute threads and M is set to 0 if unspecified.| +| `--gcthreads=N[,M]` |Use N threads for the mark phase of GC and M (0 or 1) threads for the concurrent sweeping phase of GC. N is set to the number of compute threads and M is set to 0 if unspecified. See [Memory Management and Garbage Collection](@ref man-memory-management) for more details.| |`-p`, `--procs {N\|auto}` |Integer value N launches N additional local worker processes; `auto` launches as many workers as the number of local CPU threads (logical cores)| |`--machine-file ` |Run processes on hosts listed in ``| |`-i`, `--interactive` |Interactive mode; REPL runs and `isinteractive()` is true| @@ -206,7 +206,7 @@ The following is a complete list of command-line switches available when launchi |`--track-allocation=@` |Count bytes but only in files that fall under the given file path/directory. The `@` prefix is required to select this option. A `@` with no path will track the current directory.| |`--task-metrics={yes\|no*}` |Enable the collection of per-task metrics| |`--bug-report=KIND` |Launch a bug report session. It can be used to start a REPL, run a script, or evaluate expressions. It first tries to use BugReporting.jl installed in current environment and falls back to the latest compatible BugReporting.jl if not. For more information, see `--bug-report=help`.| -|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than the given value. The value may be specified as a number of bytes, optionally in units of KB, MB, GB, or TB, or as a percentage of physical memory with %.| +|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than the given value. The value may be specified as a number of bytes, optionally in units of KB, MB, GB, or TB, or as a percentage of physical memory with %. See [Memory Management and Garbage Collection](@ref man-memory-management) for more details.| |`--compile={yes*\|no\|all\|min}` |Enable or disable JIT compiler, or request exhaustive or minimal compilation| |`--output-o ` |Generate an object file (including system image data)| |`--output-ji ` |Generate a system image data file (.ji)| diff --git a/doc/src/manual/memory-management.md b/doc/src/manual/memory-management.md new file mode 100644 index 0000000000000..4efa683e3f249 --- /dev/null +++ b/doc/src/manual/memory-management.md @@ -0,0 +1,177 @@ +# [Memory Management and Garbage Collection](@id man-memory-management) + +Julia uses automatic memory management through its built-in garbage collector (GC). This section provides an overview of how Julia manages memory and how you can configure and optimize memory usage for your applications. + +## [Garbage Collection Overview](@id man-gc-overview) + +Julia features a garbage collector with the following characteristics: + +* **Non-moving**: Objects are not relocated in memory during garbage collection +* **Generational**: Younger objects are collected more frequently than older ones +* **Parallel and partially concurrent**: The GC can use multiple threads and run concurrently with your program +* **Mostly precise**: The GC accurately identifies object references for pure Julia code, and it provides conservative scanning APIs for users calling Julia from C + +The garbage collector automatically reclaims memory used by objects that are no longer reachable from your program, freeing you from manual memory management in most cases. + +## [Memory Architecture](@id man-memory-architecture) + +Julia uses a two-tier allocation strategy: + +* **Small objects** (currently ≤ 2032 bytes but may change): Allocated using a fast per-thread pool allocator +* **Large objects** : Allocated directly through the system's `malloc` + +This hybrid approach optimizes for both allocation speed and memory efficiency, with the pool allocator providing fast allocation for the many small objects typical in Julia programs. + +## [System Memory Requirements](@id man-system-memory) + +### Swap Space + +Julia's garbage collector is designed with the expectation that your system has adequate swap space configured. The GC uses heuristics that assume it can allocate memory beyond physical RAM when needed, relying on the operating system's virtual memory management. + +If your system has limited or no swap space, you may experience out-of-memory errors during garbage collection. In such cases, you can use the `--heap-size-hint` option to limit Julia's memory usage. + +### Memory Hints + +You can provide a hint to Julia about the maximum amount of memory to use: + +```bash +julia --heap-size-hint=4G # To set the hint to ~4GB +julia --heap-size-hint=50% # or to 50% of physical memory +``` + +The `--heap-size-hint` option tells the garbage collector to trigger collection more aggressively when approaching the specified limit. This is particularly useful in: + +* Containers with memory limits +* Systems without swap space +* Shared systems where you want to limit Julia's memory footprint + +You can also set this via the `JULIA_HEAP_SIZE_HINT` environment variable: + +```bash +export JULIA_HEAP_SIZE_HINT=2G +julia +``` + +## [Multithreaded Garbage Collection](@id man-gc-multithreading) + +Julia's garbage collector can leverage multiple threads to improve performance on multi-core systems. + +### GC Thread Configuration + +By default, Julia uses multiple threads for garbage collection: + +* **Mark threads**: Used during the mark phase to trace object references (default: 1, which is shared with the compute thread if there is only one, otherwise half the number of compute threads) +* **Sweep threads**: Used for concurrent sweeping of freed memory (default: 0, disabled) + +You can configure GC threading using: + +```bash +julia --gcthreads=4,1 # 4 mark threads, 1 sweep thread +julia --gcthreads=8 # 8 mark threads, 0 sweep threads +``` + +Or via environment variable: + +```bash +export JULIA_NUM_GC_THREADS=4,1 +julia +``` + +### Recommendations + +For compute-intensive workloads: + +* Use multiple mark threads (the default configuration is usually appropriate) +* Consider enabling concurrent sweeping with 1 sweep thread for allocation-heavy workloads + +For memory-intensive workloads: + +* Enable concurrent sweeping to reduce GC pauses +* Monitor GC time using `@time` and adjust thread counts accordingly + +## [Monitoring and Debugging](@id man-gc-monitoring) + +### Basic Memory Monitoring + +Use the `@time` macro to see memory allocation and GC overhead: + +```julia +julia> @time some_computation() + 2.123456 seconds (1.50 M allocations: 58.725 MiB, 17.17% gc time) +``` + +### GC Logging + +Enable detailed GC logging to understand collection patterns: + +```julia +julia> GC.enable_logging(true) +julia> # Run your code +julia> GC.enable_logging(false) +``` + +This logs each garbage collection event with timing and memory statistics. + +### Manual GC Control + +While generally not recommended, you can manually trigger garbage collection: + +```julia +GC.gc() # Force a garbage collection +GC.enable(false) # Disable automatic GC (use with caution!) +GC.enable(true) # Re-enable automatic GC +``` + +**Warning**: Disabling GC can lead to memory exhaustion. Only use this for specific performance measurements or debugging. + +## [Performance Considerations](@id man-gc-performance) + +### Reducing Allocations + +The best way to minimize GC impact is to reduce unnecessary allocations: + +* Use in-place operations when possible (e.g., `x .+= y` instead of `x = x + y`) +* Pre-allocate arrays and reuse them +* Avoid creating temporary objects in tight loops +* Consider using `StaticArrays.jl` for small, fixed-size arrays + +### Memory-Efficient Patterns + +* Avoid global variables that change type +* Use `const` for global constants + +### Profiling Memory Usage + +For detailed guidance on profiling memory allocations and identifying performance bottlenecks, see the [Profiling](@ref man-profiling) section. + +## [Advanced Configuration](@id man-gc-advanced) + +### Integration with System Memory Management + +Julia works best when: + +* The system has adequate swap space (recommended: 2x physical RAM) +* Virtual memory is properly configured +* Other processes leave sufficient memory available +* Container memory limits are set appropriately with `--heap-size-hint` + +## [Troubleshooting Memory Issues](@id man-gc-troubleshooting) + +### High GC Overhead + +If garbage collection is taking too much time: + +1. **Reduce allocation rate**: Focus on algorithmic improvements +2. **Adjust GC threads**: Experiment with different `--gcthreads` settings +3. **Use concurrent sweeping**: Enable background sweeping with `--gcthreads=N,1` +4. **Profile memory patterns**: Identify allocation hotspots and optimize them + +### Memory Leaks + +While Julia's GC prevents most memory leaks, issues can still occur: + +* **Global references**: Avoid holding references to large objects in global variables +* **Closures**: Be careful with closures that capture large amounts of data +* **C interop**: Ensure proper cleanup when interfacing with C libraries + +For more detailed information about Julia's garbage collector internals, see the Garbage Collection section in the Developer Documentation. diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index bdf642b777963..ad4b3e6ef1312 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -84,13 +84,15 @@ julia> Threads.threadid() ### Multiple GC Threads -The Garbage Collector (GC) can use multiple threads. The amount used is either half the number -of compute worker threads or configured by either the `--gcthreads` command line argument or by using the +The Garbage Collector (GC) can use multiple threads. The amount used by default matches the compute +worker threads or can configured by either the `--gcthreads` command line argument or by using the [`JULIA_NUM_GC_THREADS`](@ref JULIA_NUM_GC_THREADS) environment variable. !!! compat "Julia 1.10" The `--gcthreads` command line argument requires at least Julia 1.10. +For more details about garbage collection configuration and performance tuning, see [Memory Management and Garbage Collection](@ref man-memory-management). + ## [Threadpools](@id man-threadpools) When a program's threads are busy with many tasks to run, tasks may experience diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index b08b71f65db05..fa197196dad4f 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -116,6 +116,8 @@ Consequently, in addition to the allocation itself, it's very likely that the code generated for your function is far from optimal. Take such indications seriously and follow the advice below. +For more information about memory management and garbage collection in Julia, see [Memory Management and Garbage Collection](@ref man-memory-management). + In this particular case, the memory allocation is due to the usage of a type-unstable global variable `x`, so if we instead pass `x` as an argument to the function it no longer allocates memory (the remaining allocation reported below is due to running the `@time` macro in global scope) and is significantly faster after the first call: From a355630eeb01b5f83fb75f15a3346a1b020a76ad Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:06:37 -0400 Subject: [PATCH 528/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=20109eaea66=20to=20b85e29428=20(#58991)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 | 1 - .../Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 | 1 - .../Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 | 1 + .../Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 create mode 100644 deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 diff --git a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 deleted file mode 100644 index 227a42db30a37..0000000000000 --- a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7cf1b30c6f32ee26f9cc4fc103d55f9f diff --git a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 b/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 deleted file mode 100644 index 1c5513aa26d5f..0000000000000 --- a/deps/checksums/Pkg-109eaea66a0adb0ad8fa497e64913eadc2248ad1.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9840e88a43e48cdac1ae82e5f50d37324ee837bf6776dac615b1ec94f021852f1124ea94b9cb26a7a6bd463d692707a3d554f8baedb8b04a479b2574ff1edb4e diff --git a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 new file mode 100644 index 0000000000000..d9020239bcd33 --- /dev/null +++ b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 @@ -0,0 +1 @@ +61a22eacd9e91ddfee859436d79e1541 diff --git a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 new file mode 100644 index 0000000000000..8d9fa14f8b32b --- /dev/null +++ b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 @@ -0,0 +1 @@ +63caedfb686a93d9ebd6c511ab833e34de7993b825461632a9b47ec64f25e002aca1ec36fb7845cfb722c6d70d473eac2f92b6b9e0848f15d63848969aacdd43 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index fca848d3e16c3..52f07a6c1726e 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 109eaea66a0adb0ad8fa497e64913eadc2248ad1 +PKG_SHA1 = b85e2942846c6ad3dc1db95fb29ddd6e4f262591 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 8596dbe925eaa81dd99b896d0e83aec2ef281994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 14 Jul 2025 16:56:07 +0200 Subject: [PATCH 529/662] Add one-argument `argtypes` methods to source reflection functions (#58925) Follow-up to https://github.com/JuliaLang/julia/pull/58891#issuecomment-3036419509, extending the feature to `which`, `functionloc`, `edit` and `less`. --- base/methodshow.jl | 1 + base/reflection.jl | 1 + stdlib/InteractiveUtils/src/editless.jl | 3 ++- stdlib/InteractiveUtils/test/runtests.jl | 12 ++++++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index f58a4911494b3..46b915a1dd09e 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -181,6 +181,7 @@ end Return a tuple `(filename,line)` giving the location of a generic `Function` definition. """ functionloc(@nospecialize(f), @nospecialize(types)) = functionloc(which(f,types)) +functionloc(@nospecialize(argtypes::Union{Tuple, Type{<:Tuple}})) = functionloc(which(argtypes)) function functionloc(@nospecialize(f)) mt = methods(f) diff --git a/base/reflection.jl b/base/reflection.jl index 06fe1a2064b16..5ad05b615ddfd 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -960,6 +960,7 @@ Returns the method that would be called by the given type signature (as a tuple function which(@nospecialize(tt#=::Type=#)) return _which(tt).method end +which(@nospecialize(argtypes::Tuple)) = which(to_tuple_type(argtypes)) """ which(module, symbol) diff --git a/stdlib/InteractiveUtils/src/editless.jl b/stdlib/InteractiveUtils/src/editless.jl index 6d1d75f1072ea..e7ac51a2d2092 100644 --- a/stdlib/InteractiveUtils/src/editless.jl +++ b/stdlib/InteractiveUtils/src/editless.jl @@ -269,7 +269,8 @@ function edit(@nospecialize f) end edit(m::Method) = edit(functionloc(m)...) edit(@nospecialize(f), idx::Integer) = edit(methods(f).ms[idx]) -edit(f, t) = (@nospecialize; edit(functionloc(f, t)...)) +edit(f, t) = (@nospecialize; edit(functionloc(f, t)...)) +edit(@nospecialize argtypes::Union{Tuple, Type{<:Tuple}}) = edit(functionloc(argtypes)...) edit(file::Nothing, line::Integer) = error("could not find source file for function") edit(m::Module) = edit(pathof(m)) diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index b233dbffa9099..f2c47e6561043 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -329,6 +329,18 @@ catch err13464 @test startswith(err13464.msg, "expression is not a function call") end +@testset "Single-argument forms" begin + a = which(+, (Int, Int)) + b = which((typeof(+), Int, Int)) + c = which(Tuple{typeof(+), Int, Int}) + @test a == b == c + + a = functionloc(+, (Int, Int)) + b = functionloc((typeof(+), Int, Int)) + c = functionloc(Tuple{typeof(+), Int, Int}) + @test a == b == c +end + # PR 57909 @testset "Support for type annotations as arguments" begin @test (@which (::Vector{Int})[::Int]).name === :getindex From cdca6686574e6a079e7318e7f938460f29567fcb Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 15 Jul 2025 02:02:48 +0900 Subject: [PATCH 530/662] Test: Add compiler hint for `ts` variable definedness in `@testset for` (#58989) Helps the new language server avoid reporting unused variable reports. --- stdlib/Test/src/Test.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 43855c64dfd0c..cafc3040ef511 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1975,6 +1975,7 @@ function testset_forloop(args, testloop, source) # Handle `return` in test body if !first_iteration && !finish_errored pop_testset() + @assert @isdefined(ts) "Assertion to tell the compiler about the definedness of this variable" push!(arr, finish(ts)) end copy!(default_rng(), default_rng_orig) From 95c4870e8059d863d3ae6371bc0cf0d4d30e01c4 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 15 Jul 2025 02:03:20 +0900 Subject: [PATCH 531/662] trimming: explictly add Libdl dep for test/trimming/basic_jll.jl (#58990) --- test/trimming/Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/trimming/Project.toml b/test/trimming/Project.toml index 183536eec9a29..86c75e7d24639 100644 --- a/test/trimming/Project.toml +++ b/test/trimming/Project.toml @@ -1,5 +1,6 @@ [deps] JLLWrappers = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [sources] From cecaf5252796abf7158eb238051b3a37ce7562ab Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 14 Jul 2025 16:37:24 -0400 Subject: [PATCH 532/662] win/msys2: Automatically switch msys2 symlinks mode for LLVM (#58988) As noted in https://github.com/JuliaLang/julia/issues/54981#issuecomment-2336444226, msys2 currently fails to untar an llvm source build. Fix that by setting the appropriate environment variable to switch the symlinks mode. --- deps/llvm.mk | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deps/llvm.mk b/deps/llvm.mk index c87f8036ccc17..7f5989565e6ba 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -6,6 +6,14 @@ include $(SRCDIR)/llvm-options.mk ifneq ($(USE_BINARYBUILDER_LLVM), 1) LLVM_GIT_URL:=https://github.com/JuliaLang/llvm-project.git LLVM_TAR_URL=https://api.github.com/repos/JuliaLang/llvm-project/tarball/$1 +# LLVM's tarball contains symlinks to non-existent targets. This breaks the +# the default msys strategy `deepcopy` symlink strategy. To workaround this, +# switch to `native` which tries native windows symlinks (possible if the +# machine is in developer mode) - or if not, falls back to cygwin-style +# symlinks. We don't particularly care either way - we just need to symlinks +# to succeed. We could guard this by a uname check, but it's harmless elsewhere, +# so let's not incur the additional overhead. +$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native $(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) LLVM_BUILDDIR := $(BUILDDIR)/$(LLVM_SRC_DIR) From 7a8d0e4f1886be37f5fdbffbc11522b687654fca Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 14 Jul 2025 19:04:14 -0400 Subject: [PATCH 533/662] Fix order of MSYS rules (#58999) git-external changes the LLVM_SRC_DIR variable, so the target-specific variable applies to the wrong target if defined before it - didn't notice in local testing because I had accidentally switched the variable globally earlier for testing - but showed up on a fresh build. --- deps/llvm.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 7f5989565e6ba..ec606a98c8484 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -6,6 +6,7 @@ include $(SRCDIR)/llvm-options.mk ifneq ($(USE_BINARYBUILDER_LLVM), 1) LLVM_GIT_URL:=https://github.com/JuliaLang/llvm-project.git LLVM_TAR_URL=https://api.github.com/repos/JuliaLang/llvm-project/tarball/$1 +$(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) # LLVM's tarball contains symlinks to non-existent targets. This breaks the # the default msys strategy `deepcopy` symlink strategy. To workaround this, # switch to `native` which tries native windows symlinks (possible if the @@ -14,7 +15,6 @@ LLVM_TAR_URL=https://api.github.com/repos/JuliaLang/llvm-project/tarball/$1 # to succeed. We could guard this by a uname check, but it's harmless elsewhere, # so let's not incur the additional overhead. $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native -$(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) LLVM_BUILDDIR := $(BUILDDIR)/$(LLVM_SRC_DIR) LLVM_BUILDDIR_withtype := $(LLVM_BUILDDIR)/build_$(LLVM_BUILDTYPE) From f2a8e18a98c9578136d4b8bc84b95ef3aead7c83 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 15 Jul 2025 02:42:19 -0400 Subject: [PATCH 534/662] msys2: Recommend correct cmake package (#59001) msys2 ships 2 different cmake packages, one built natively (with mingw prefix in the package name) and one built against the posix emulation environment. The posix emulation one does not work because it will detect unix-style paths, which it then writes into files that native tools process. Unlike during command invocation (where the msys2 runtime library does path translation), when paths are written to files, they are written verbatim. The practical result of this is that e.g. the LLVM build will fail with a mysterious libz link failure (as e.g. reported in #54981). This is our fault, because our built instructions tell the user to install the wrong one. Fix all that by 1. Correcting the build instructions to install the correct cmake 2. Detecting if the wrong cmake is installed and advising the correct one 3. Fixing an issue where the native CMake did not like our CMAKE_C_COMPILER setting. With all this, CMake runs correctly under msys2 with USE_BINARYBUILDER_LLVM=0. --- Make.inc | 5 +++-- deps/tools/common.mk | 14 ++++++++++++++ doc/src/devdocs/build/windows.md | 6 +++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Make.inc b/Make.inc index 10a3ccaf5e229..a3543a5e276f0 100644 --- a/Make.inc +++ b/Make.inc @@ -431,9 +431,10 @@ export PKG_CONFIG_PATH = $(JULIAHOME)/usr/lib/pkgconfig export PKG_CONFIG_LIBDIR = $(JULIAHOME)/usr/lib/pkgconfig # Figure out OS and architecture -BUILD_OS := $(shell uname) +RAW_BUILD_OS = $(shell uname) +BUILD_OS := $(RAW_BUILD_OS) -ifneq (,$(findstring CYGWIN,$(BUILD_OS))) +ifneq (,$(findstring CYGWIN,$(RAW_BUILD_OS))) XC_HOST ?= $(shell uname -m)-w64-mingw32 endif diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 890eca8d718fa..54cc8f62e9c82 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -38,6 +38,12 @@ CMAKE_CC := "$$(which $(shell echo $(CC_ARG) | cut -d' ' -f1))" CMAKE_CXX := "$$(which $(shell echo $(CXX_ARG) | cut -d' ' -f1))" CMAKE_CC_ARG := $(shell echo $(CC_ARG) | cut -s -d' ' -f2-) CMAKE_CXX_ARG := $(shell echo $(CXX_ARG) | cut -s -d' ' -f2-) +else ifneq (,$(findstring MINGW,$(RAW_BUILD_OS))) +# `cmake` is mingw-native and needs `cygpath -w`, rather than `cygpath -m`, which is the msys2 conversion default +CMAKE_CC := "$(shell echo $(call cygpath_w, $(shell which $(CC_BASE))))" +CMAKE_CXX := "$(shell echo $(call cygpath_w, $(shell which $(CXX_BASE))))" +CMAKE_CC_ARG := $(CC_ARG) +CMAKE_CXX_ARG := $(CXX_ARG) else CMAKE_CC := "$$(which $(CC_BASE))" CMAKE_CXX := "$$(which $(CXX_BASE))" @@ -68,6 +74,14 @@ else $(error Unknown CMake generator '$(CMAKE_GENERATOR)'. Options are 'Ninja' and 'make') endif +# Detect MSYS2 with cygwin CMake rather than MinGW cmake - the former fails to +# properly drive MinGW tools +ifneq (,$(findstring MINGW,$(RAW_BUILD_OS))) +ifneq (,$(shell ldd $(shell which cmake) | grep msys-2.0.dll)) +override CMAKE := echo "ERROR: CMake is Cygwin CMake, not MinGW CMake. Build will fail. Use 'pacman -S mingw-w64-{i686,x86_64}-cmake'."; exit 1; $(CMAKE) +endif +endif + # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libdir=$(build_libdir) shlibdir=$(build_shlibdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index c19c7ea1e9949..4fd9f01a81a0f 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -141,19 +141,19 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. 4. Then install tools required to build julia: ``` - pacman -S cmake diffutils git m4 make patch tar p7zip curl python + pacman -S diffutils git m4 make patch tar p7zip curl python ``` For 64 bit Julia, install the x86_64 version: ``` - pacman -S mingw-w64-x86_64-gcc + pacman -S mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake ``` For 32 bit Julia, install the i686 version: ``` - pacman -S mingw-w64-i686-gcc + pacman -S mingw-w64-i686-gcc mingw-w64-i686-cmake ``` 5. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. From 11eeed32e8fe3cd24c8439fbe2220d3f4183f306 Mon Sep 17 00:00:00 2001 From: Martin Kunz Date: Tue, 15 Jul 2025 22:10:09 +0200 Subject: [PATCH 535/662] feat(REPL): Added `active_module` context to numbered REPL (#59000) --- stdlib/REPL/src/REPL.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 916cc5cec8b57..db6f5a26e90ed 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1883,10 +1883,10 @@ end function set_prompt(repl::LineEditREPL, n::Ref{Int}) julia_prompt = repl.interface.modes[1] - julia_prompt.prompt = function() + julia_prompt.prompt = REPL.contextual_prompt(repl, function() n[] = repl_eval_counter(julia_prompt.hist)+1 string("In [", n[], "]: ") - end + end) nothing end From 9d6344a6673387a4ac0ae04f197ab60055d7a05d Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 16 Jul 2025 08:41:37 -0400 Subject: [PATCH 536/662] optimize `length(::OrdinalRange)` for large bit-ints (#58864) Split from #58793, this coalesces nearly all the branches in `length`, allowing it to inline and generally perform much better while retaining the exact same functionality. --------- Co-authored-by: N5N3 <2642243996@qq.com> --- base/range.jl | 18 ++++++------------ test/ranges.jl | 2 ++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/base/range.jl b/base/range.jl index 3e1cd77eb914b..e9d28daf3ba3b 100644 --- a/base/range.jl +++ b/base/range.jl @@ -802,21 +802,15 @@ let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, # slightly more accurate length and checked_length in extreme cases # (near typemax) for types with known `unsigned` functions function length(r::OrdinalRange{T}) where T<:bigints - @inline s = step(r) diff = last(r) - first(r) isempty(r) && return zero(diff) - # if |s| > 1, diff might have overflowed, but unsigned(diff)÷s should - # therefore still be valid (if the result is representable at all) - # n.b. !(s isa T) - if s isa Unsigned || -1 <= s <= 1 || s == -s - a = div(diff, s) % typeof(diff) - elseif s < 0 - a = div(unsigned(-diff), -s) % typeof(diff) - else - a = div(unsigned(diff), s) % typeof(diff) - end - return a + oneunit(a) + # Compute `(diff ÷ s) + 1` in a manner robust to signed overflow + # by using the absolute values as unsigneds for non-empty ranges. + # Note that `s` may be a different type from T and diff; it may not + # even be a BitInteger that supports `unsigned`. Handle with care. + a = div(unsigned(flipsign(diff, s)), s) % typeof(diff) + return flipsign(a, s) + oneunit(a) end function checked_length(r::OrdinalRange{T}) where T<:bigints s = step(r) diff --git a/test/ranges.jl b/test/ranges.jl index 4a0c6c9ddd65f..d5bc968a12399 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -755,6 +755,8 @@ end @test length(typemin(T):typemax(T)) == T(0) @test length(zero(T):one(T):typemax(T)) == typemin(T) @test length(typemin(T):one(T):typemax(T)) == T(0) + @test length(StepRange{T,BigInt}(zero(T), 1, typemax(T))) == typemin(T) + @test length(StepRange{T,BigInt}(typemin(T), 1, typemax(T))) == T(0) @test_throws OverflowError checked_length(zero(T):typemax(T)) @test_throws OverflowError checked_length(typemin(T):typemax(T)) @test_throws OverflowError checked_length(zero(T):one(T):typemax(T)) From 00351da179b7de6cd8a28f229580c96603387f0b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Jul 2025 14:23:38 -0400 Subject: [PATCH 537/662] Fix LLVM TaskDispatcher implementation issues (#58950) Fixes #58229 (LLVM JITLink stack overflow issue) I tried submitting this promise/future implementation upstream (https://github.com/llvm/llvm-project/compare/main...vtjnash:llvm-project:jn/cowait-jit) so that I would not need to duplicate nearly as much code here to fix this bug, but upstream is currently opposed to fixing this bug and instead insists it is preferable for each downstream project to implement this fix themselves adding extra maintenance burden for us for now. Sigh. --- src/Makefile | 2 +- src/jitlayers.cpp | 23 +- src/llvm-julia-task-dispatcher.h | 465 +++++++++++++++++++++++++++++++ 3 files changed, 474 insertions(+), 16 deletions(-) create mode 100644 src/llvm-julia-task-dispatcher.h diff --git a/src/Makefile b/src/Makefile index 717afa55c6207..a8926a46c9b00 100644 --- a/src/Makefile +++ b/src/Makefile @@ -374,7 +374,7 @@ $(BUILDDIR)/gc-alloc-profiler.o $(BUILDDIR)/gc-alloc-profiler.dbg.obj: $(SRCDIR) $(BUILDDIR)/gc-page-profiler.o $(BUILDDIR)/gc-page-profiler.dbg.obj: $(SRCDIR)/gc-page-profiler.h $(BUILDDIR)/init.o $(BUILDDIR)/init.dbg.obj: $(SRCDIR)/builtin_proto.h $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/builtin_proto.h -$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h +$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-julia-task-dispatcher.h $(BUILDDIR)/jltypes.o $(BUILDDIR)/jltypes.dbg.obj: $(SRCDIR)/builtin_proto.h $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvm-codegen-shared.h $(BUILDDIR)/julia_version.h $(BUILDDIR)/llvm-alloc-helpers.o $(BUILDDIR)/llvm-alloc-helpers.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-alloc-helpers.h diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 9ea98fed68db3..c5588be794201 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -6,6 +6,7 @@ #include #include "llvm/IR/Mangler.h" +#include #include #include #include @@ -50,6 +51,7 @@ using namespace llvm; #include "jitlayers.h" #include "julia_assert.h" #include "processor.h" +#include "llvm-julia-task-dispatcher.h" #if JL_LLVM_VERSION >= 180000 # include @@ -723,17 +725,8 @@ static void jl_compile_codeinst_now(jl_code_instance_t *codeinst) if (!decls.specFunctionObject.empty()) NewDefs.push_back(decls.specFunctionObject); } - // Split batches to avoid stack overflow in the JIT linker. - // FIXME: Patch ORCJITs InPlaceTaskDispatcher to not recurse on task dispatches but - // push the tasks to a queue to be drained later. This avoids the stackoverflow caused by recursion - // in the linker when compiling a large number of functions at once. - SmallVector Addrs; - for (size_t i = 0; i < NewDefs.size(); i += 1000) { - auto end = std::min(i + 1000, NewDefs.size()); - SmallVector batch(NewDefs.begin() + i, NewDefs.begin() + end); - auto AddrsBatch = jl_ExecutionEngine->findSymbols(batch); - Addrs.append(AddrsBatch); - } + auto Addrs = jl_ExecutionEngine->findSymbols(NewDefs); + size_t nextaddr = 0; for (auto &this_code : linkready) { auto it = invokenames.find(this_code); @@ -1901,7 +1894,7 @@ llvm::DataLayout jl_create_datalayout(TargetMachine &TM) { JuliaOJIT::JuliaOJIT() : TM(createTargetMachine()), DL(jl_create_datalayout(*TM)), - ES(cantFail(orc::SelfExecutorProcessControl::Create())), + ES(cantFail(orc::SelfExecutorProcessControl::Create(nullptr, std::make_unique<::JuliaTaskDispatcher>()))), GlobalJD(ES.createBareJITDylib("JuliaGlobals")), JD(ES.createBareJITDylib("JuliaOJIT")), ExternalJD(ES.createBareJITDylib("JuliaExternal")), @@ -2159,7 +2152,7 @@ SmallVector JuliaOJIT::findSymbols(ArrayRef Names) Unmangled[NonOwningSymbolStringPtr(Mangled)] = Unmangled.size(); Exports.add(std::move(Mangled)); } - SymbolMap Syms = cantFail(ES.lookup(orc::makeJITDylibSearchOrder(ArrayRef(&JD)), std::move(Exports))); + SymbolMap Syms = cantFail(::safelookup(ES, orc::makeJITDylibSearchOrder(ArrayRef(&JD)), std::move(Exports))); SmallVector Addrs(Names.size()); for (auto it : Syms) { Addrs[Unmangled.at(orc::NonOwningSymbolStringPtr(it.first))] = it.second.getAddress().getValue(); @@ -2171,7 +2164,7 @@ Expected JuliaOJIT::findSymbol(StringRef Name, bool ExportedS { orc::JITDylib* SearchOrders[3] = {&JD, &GlobalJD, &ExternalJD}; ArrayRef SearchOrder = ArrayRef(&SearchOrders[0], ExportedSymbolsOnly ? 3 : 1); - auto Sym = ES.lookup(SearchOrder, Name); + auto Sym = ::safelookup(ES, SearchOrder, Name); return Sym; } @@ -2184,7 +2177,7 @@ Expected JuliaOJIT::findExternalJDSymbol(StringRef Name, bool { orc::JITDylib* SearchOrders[3] = {&ExternalJD, &GlobalJD, &JD}; ArrayRef SearchOrder = ArrayRef(&SearchOrders[0], ExternalJDOnly ? 1 : 3); - auto Sym = ES.lookup(SearchOrder, getMangledName(Name)); + auto Sym = ::safelookup(ES, SearchOrder, getMangledName(Name)); return Sym; } diff --git a/src/llvm-julia-task-dispatcher.h b/src/llvm-julia-task-dispatcher.h new file mode 100644 index 0000000000000..dd4037378b6b6 --- /dev/null +++ b/src/llvm-julia-task-dispatcher.h @@ -0,0 +1,465 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +namespace { + +using namespace llvm::orc; + +template struct future_value_storage { + // Union disables default construction/destruction semantics, allowing us to + // use placement new/delete for precise control over value lifetime + union { + U value_; + }; + + future_value_storage() {} + ~future_value_storage() {} +}; + +template <> struct future_value_storage { + // No value_ member for void +}; + +struct JuliaTaskDispatcher : public TaskDispatcher { + /// Forward declarations + class future_base; + void dispatch(std::unique_ptr T) override; + void shutdown() override; + void work_until(future_base &F); +private: + /// C++ does not support non-static thread_local variables, so this needs to + /// store both the task and the associated dispatcher queue so that shutdown + /// can wait for the correct tasks to finish. + thread_local static SmallVector, JuliaTaskDispatcher*>> TaskQueue; + std::mutex DispatchMutex; + std::condition_variable WorkFinishedCV; + SmallVector WaitingFutures; + +public: + +/// @name ORC Promise/Future Classes +/// +/// ORC-aware promise/future implementation that integrates with the +/// TaskDispatcher system to allow efficient cooperative multitasking while +/// waiting for results (with certain limitations on what can be awaited). +/// Together they provide building blocks for a full async/await-like runtime +/// for llvm that supports multiple threads. +/// +/// Unlike std::promise/std::future alone, these classes can help dispatch other +/// tasks while waiting, preventing deadlocks and improving overall system +/// throughput. They have a similar API, though with some important differences +/// and some features simply not currently implemented. +/// +/// @{ + +template class promise; +template class future; + +/// Status for future/promise state +enum class FutureStatus : uint8_t { NotReady = 0, Ready = 1 }; + +/// @} + +/// Type-erased base class for futures, generally for scheduler use to avoid +/// needing virtual dispatches +class future_base { +public: + /// Check if the future is now ready with a value (precondition: get_promise() + /// must have been called) + bool ready() const { + if (!valid()) + report_fatal_error("ready() called before get_promise()"); + return state_->status_.load(std::memory_order_acquire) == FutureStatus::Ready; + } + + /// Check if the future is in a valid state (not moved-from and get_promise() called) + bool valid() const { return state_ != nullptr; } + + /// Wait for the future to be ready, helping with task dispatch + void wait(JuliaTaskDispatcher &D) { + // Keep helping with task dispatch until our future is ready + if (!ready()) { + D.work_until(*this); + if (state_->status_.load(std::memory_order_relaxed) != FutureStatus::Ready) + report_fatal_error( + "work_until() returned without this future being ready"); + } + } + +protected: + struct state_base { + std::atomic status_{FutureStatus::NotReady}; + }; + + future_base(state_base *state) : state_(state) {} + future_base() = default; + + /// Only allow deleting the future once it is invalid + ~future_base() { + if (state_) + report_fatal_error("get() must be called before future destruction (ensuring promise::set_value memory is valid)"); + } + + // Move constructor and assignment + future_base(future_base &&other) noexcept : state_(other.state_) { + other.state_ = nullptr; + } + + future_base &operator=(future_base &&other) noexcept { + if (this != &other) { + this->~future_base(); + state_ = other.state_; + other.state_ = nullptr; + } + return *this; + } + + state_base *state_; +}; + +/// TaskDispatcher-aware future class for cooperative await. +/// +/// @tparam T The type of value this future will provide. Use void for futures +/// that +/// signal completion without providing a value. +/// +/// This future implementation is similar to `std::future`, so most code can +/// transition to it easily. However, it differs from `std::future` in a few +/// key ways to be aware of: +/// - No exception support (or the overhead for it). +/// - The future is created before the promise, then the promise is created +/// from the future. +/// - The future is in an invalid state until get_promise() has been called. +/// - Waiting operations (get(&D), wait(&D)) help dispatch other tasks while +/// blocked, requiring an additional argument of which TaskDispatcher object +/// of where all associated work will be scheduled. +/// - While `wait` may be called multiple times and on multiple threads, all of +/// them must have returned before calling `get` on exactly one thread. +/// - Must call get() exactly once before destruction (enforced with +/// `report_fatal_error`) after each call to `get_promise`. Internal state is +/// freed when `get` returns, and allocated when `get_promise` is called. +/// +/// Other notable features, in common with `std::future`: +/// - Supports both value types and void specialization through the same +/// interface. +/// - Thread-safe through atomic operations. +/// - Provides acquire-release ordering with `std::promise::set_value()`. +/// - Concurrent access to any method (including to `ready`) on multiple threads +/// is not allowed. +/// - Holding any locks while calling `get()` is likely to lead to deadlock. +/// +/// @warning Users should avoid borrowing references to futures. References may +/// go out of scope and break the uniqueness contract, which may break the +/// soundness of the types. Always use move semantics or pass by value. + +template class future : public future_base { +public: + future() : future_base(nullptr) {} + future(const future &) = delete; + future &operator=(const future &) = delete; + future(future &&) = default; + future &operator=(future &&) = default; + + /// Get the value, helping with task dispatch while waiting. + /// This will destroy the underlying value, so this must be called exactly + /// once, which returns the future to the initial state. + T get(JuliaTaskDispatcher &D) { + if (!valid()) + report_fatal_error("get() must only be called once, after get_promise()"); + wait(D); + auto state_ = static_cast(this->state_); + this->state_ = nullptr; + return take_value(state_); + } + + /// Get the associated promise (must only be called once) + promise get_promise() { + if (valid()) + report_fatal_error("get_promise() can only be called once"); + auto state_ = new state(); + this->state_ = state_; + return promise(state_); + } + +private: + friend class promise; + + // Template the state struct with EBCO so that future has no wasted + // overhead for the value. The declaration of future_value_storage is far + // above here since GCC doesn't implement it properly when nested. + struct state : future_base::state_base, future_value_storage {}; + + template + typename std::enable_if::value, U>::type take_value(state *state_) { + T result = std::move(state_->value_); + state_->value_.~T(); + delete state_; + return result; + } + + template + typename std::enable_if::value, U>::type take_value(state *state_) { + delete state_; + } +}; + +/// TaskDispatcher-aware promise class that provides values to associated +/// futures. +/// +/// @tparam T The type of value this promise will provide. Use void for promises +/// that +/// signal completion without providing a value. +/// +/// This promise implementation provides the value-setting side of the +/// promise/future pair and integrates with the ORC TaskDispatcher system. Key +/// characteristics: +/// - Created from a future via get_promise() rather than creating the future from the promise. +/// - Must call get_future() on the thread that created it (it can be passed to another thread, but do not borrow a reference and use that to mutate it from another thread). +/// - Must call set_value() exactly once per `get_promise()` call to provide the result. +/// - Thread-safe from set_value to get. +/// - Move-only semantics to prevent accidental copying. +/// +/// The `promise` can usually be passed to another thread in one of two ways: +/// - With move semantics: +/// * `[P = F.get_promise()] () { P.set_value(); }` +/// * `[P = std::move(P)] () { P.set_value(); }` +/// * Advantages: clearer where `P` is owned, automatic deadlock detection +/// on destruction, +/// easier memory management if the future is returned from the function. +/// - By reference: +/// * `[&P] () { P.set_value(); }` +/// * Advantages: simpler memory management if the future is consumed in the +/// same function. +/// * Disadvantages: more difficult memory management if the future is +/// returned from the function, no deadlock detection. +/// +/// @warning Users should avoid borrowing references to promises. References may +/// go out of scope and break the uniqueness contract, which may break the +/// soundness of the types. Always use move semantics or pass by value. +/// +/// @par Error Handling: +/// The promise/future system uses report_fatal_error() for misuse: +/// - Calling set_value() more than once. +/// - Destroying a future without calling get(). +/// - Calling get() more than once on a future. +/// +/// @par Thread Safety: +/// - Each promise/future must only be accessed by one thread, as concurrent +/// calls to the API functions may result in crashes. +/// - Multiple threads can safely access different promise/future pairs. +/// - set_value() and get() operations are atomic and thread-safe. +/// - Move operations should only be performed by a single thread. +template class promise { + friend class future; + +public: + promise() : state_(nullptr) {} + + ~promise() { + // Assert proper promise lifecycle: ensure set_value was called if promise was valid. + // This can catch deadlocks where a promise is created but set_value() is + // never called, though only if the promise is moved from instead of + // borrowed from the frame with the future. + // Empty promises (state_ == nullptr) are allowed to be destroyed without calling set_value. + } + + promise(const promise &) = delete; + promise &operator=(const promise &) = delete; + + promise(promise &&other) noexcept + : state_(other.state_) { + other.state_ = nullptr; + } + + promise &operator=(promise &&other) noexcept { + if (this != &other) { + this->~promise(); + state_ = other.state_; + other.state_ = nullptr; + } + return *this; + } + + + /// Set the value (must only be called once) + // In C++20, this std::conditional weirdness can probably be replaced just + // with requires. It ensures that we don't try to define a method for `void&`, + // but that if the user calls set_value(v) for any value v that they get a + // member function error, instead of no member named 'value_'. + template + void + set_value(const typename std::conditional::value, + std::nullopt_t, T>::type &value) const { + assert(state_ && "set_value() can only be called once"); + new (&state_->value_) T(value); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + template + void set_value(typename std::conditional::value, + std::nullopt_t, T>::type &&value) const { + assert(state_ && "set_value() can only be called once"); + new (&state_->value_) T(std::move(value)); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + template + typename std::enable_if::value, void>::type + set_value(const std::nullopt_t &value) = delete; + + template + typename std::enable_if::value, void>::type + set_value(std::nullopt_t &&value) = delete; + + template + typename std::enable_if::value, void>::type set_value() const { + assert(state_ && "set_value() can only be called once"); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + /// Swap with another promise + void swap(promise &other) noexcept { + using std::swap; + swap(state_, other.state_); + } + +private: + explicit promise(typename future::state *state) + : state_(state) {} + + mutable typename future::state *state_; +}; + +}; // class JuliaTaskDispatcher + +thread_local SmallVector, JuliaTaskDispatcher *>> JuliaTaskDispatcher::TaskQueue; + +void JuliaTaskDispatcher::dispatch(std::unique_ptr T) { + TaskQueue.push_back(std::pair(std::move(T), this)); +} + +void JuliaTaskDispatcher::shutdown() { + // Keep processing until no tasks belonging to this dispatcher remain + while (true) { + // Check if any task belongs to this dispatcher + auto it = std::find_if( + TaskQueue.begin(), TaskQueue.end(), + [this](const auto &TaskPair) { return TaskPair.second == this; }); + + // If no tasks belonging to this dispatcher, we're done + if (it == TaskQueue.end()) + return; + + // Create a future/promise pair to wait for completion of this task + future taskFuture; + // Replace the task with a GenericNamedTask that wraps the original task + // with a notification of completion that this thread can work_until. + auto originalTask = std::move(it->first); + it->first = makeGenericNamedTask( + [originalTask = std::move(originalTask), + taskPromise = taskFuture.get_promise()]() { + originalTask->run(); + taskPromise.set_value(); + }, + "Shutdown task marker"); + + // Wait for the task to complete + taskFuture.get(*this); + } +} + +void JuliaTaskDispatcher::work_until(future_base &F) { + while (!F.ready()) { + // First, process any tasks in our local queue + // Process in LIFO order (most recently added first) to avoid deadlocks + // when tasks have dependencies on each other + while (!TaskQueue.empty()) { + { + auto TaskPair = std::move(TaskQueue.back()); + TaskQueue.pop_back(); + TaskPair.first->run(); + } + + // Notify any threads that might be waiting for work to complete + { + std::lock_guard Lock(DispatchMutex); + bool ShouldNotify = llvm::any_of( + WaitingFutures, [](future_base *F) { return F->ready(); }); + if (ShouldNotify) { + WaitingFutures.clear(); + WorkFinishedCV.notify_all(); + } + } + + // Check if our future is now ready + if (F.ready()) + return; + } + + // If we get here, our queue is empty but the future isn't ready + // We need to wait for other threads to finish work that should complete our + // future + { + std::unique_lock Lock(DispatchMutex); + WaitingFutures.push_back(&F); + WorkFinishedCV.wait(Lock, [&F]() { return F.ready(); }); + } + } +} + +} // End namespace + +namespace std { +template +void swap(::JuliaTaskDispatcher::promise &lhs, ::JuliaTaskDispatcher::promise &rhs) noexcept { + lhs.swap(rhs); +} +} // End namespace std + +// n.b. this actually is sometimes a safepoint +Expected +safelookup(ExecutionSession &ES, + const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, LookupKind K = LookupKind::Static, + SymbolState RequiredState = SymbolState::Ready, + RegisterDependenciesFunction RegisterDependencies = NoDependenciesToRegister) JL_NOTSAFEPOINT { + JuliaTaskDispatcher::future> PromisedFuture; + auto NotifyComplete = [PromisedResult = PromisedFuture.get_promise()](Expected R) { + PromisedResult.set_value(std::move(R)); + }; + ES.lookup(K, SearchOrder, std::move(Symbols), RequiredState, + std::move(NotifyComplete), RegisterDependencies); + return PromisedFuture.get(static_cast(ES.getExecutorProcessControl().getDispatcher())); +} + +Expected +safelookup(ExecutionSession &ES, + const JITDylibSearchOrder &SearchOrder, + SymbolStringPtr Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + SymbolLookupSet Names({Name}); + + if (auto ResultMap = safelookup(ES, SearchOrder, std::move(Names), LookupKind::Static, + RequiredState, NoDependenciesToRegister)) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +Expected +safelookup(ExecutionSession &ES, + ArrayRef SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + return safelookup(ES, makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); +} + +Expected +safelookup(ExecutionSession &ES, + ArrayRef SearchOrder, StringRef Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + return safelookup(ES, SearchOrder, ES.intern(Name), RequiredState); +} From 720e23d1195f1719f74305053e15128d12033ed9 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Wed, 16 Jul 2025 15:30:36 -0600 Subject: [PATCH 538/662] Improve --trace-dispatch coverage: emit in "full-cache" fast path as well. (#59012) This PR moves the `--trace-dispatch` logging inside `jl_lookup_generic_` from only the `cache miss case` to also logging it inside the `no method was found in the associative cache, check the full cache` case. This PR logs the data from inside each of the two slow-path cases. --- src/gf.c | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/gf.c b/src/gf.c index 70f9d3dc4c33a..82769d2794d9f 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3096,7 +3096,8 @@ static void record_dispatch_statement(jl_method_instance_t *mi) s_dispatch = (JL_STREAM*) &f_dispatch; } } - if (!jl_has_free_typevars(mi->specTypes)) { + // NOTE: For builtin functions, the specType is just `Tuple`, which is not useful to print. + if (!jl_has_free_typevars(mi->specTypes) && (jl_datatype_t*)mi->specTypes != jl_tuple_type) { jl_printf(s_dispatch, "precompile("); jl_static_show(s_dispatch, mi->specTypes); jl_printf(s_dispatch, ")\n"); @@ -3106,6 +3107,19 @@ static void record_dispatch_statement(jl_method_instance_t *mi) JL_UNLOCK(&dispatch_statement_out_lock); } +static void record_dispatch_statement_on_first_dispatch(jl_method_instance_t *mfunc) { + uint8_t force_trace_dispatch = jl_atomic_load_relaxed(&jl_force_trace_dispatch_enabled); + if (force_trace_dispatch || jl_options.trace_dispatch != NULL) { + uint8_t miflags = jl_atomic_load_relaxed(&mfunc->flags); + uint8_t was_dispatched = miflags & JL_MI_FLAGS_MASK_DISPATCHED; + if (!was_dispatched) { + miflags |= JL_MI_FLAGS_MASK_DISPATCHED; + jl_atomic_store_relaxed(&mfunc->flags, miflags); + record_dispatch_statement(mfunc); + } + } +} + // If waitcompile is 0, this will return NULL if compiling is on-going in the JIT. This is // useful for the JIT itself, since it just doesn't cause redundant work or missed updates, // but merely causes it to look into the current JIT worklist. @@ -3941,6 +3955,11 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t jl_atomic_store_relaxed(&pick_which[cache_idx[0]], which); jl_atomic_store_release(&call_cache[cache_idx[which & 3]], entry); } + if (entry) { + // mfunc was found in slow path, so log --trace-dispatch + jl_method_instance_t *mfunc = entry->func.linfo; + record_dispatch_statement_on_first_dispatch(mfunc); + } } jl_method_instance_t *mfunc; @@ -3963,23 +3982,15 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t jl_method_error(F, args, nargs, world); // unreachable } - // mfunc is about to be dispatched - uint8_t force_trace_dispatch = jl_atomic_load_relaxed(&jl_force_trace_dispatch_enabled); - if (force_trace_dispatch || jl_options.trace_dispatch != NULL) { - uint8_t miflags = jl_atomic_load_relaxed(&mfunc->flags); - uint8_t was_dispatched = miflags & JL_MI_FLAGS_MASK_DISPATCHED; - if (!was_dispatched) { - miflags |= JL_MI_FLAGS_MASK_DISPATCHED; - jl_atomic_store_relaxed(&mfunc->flags, miflags); - record_dispatch_statement(mfunc); - } - } + // mfunc was found in slow path, so log --trace-dispatch + record_dispatch_statement_on_first_dispatch(mfunc); } #ifdef JL_TRACE if (traceen) jl_printf(JL_STDOUT, " at %s:%d\n", jl_symbol_name(mfunc->def.method->file), mfunc->def.method->line); #endif + return mfunc; } From be59b3b2f480078afad0a34205bf9807b95aa46b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 16 Jul 2025 20:55:51 -0400 Subject: [PATCH 539/662] MozillaCACerts: Update to 2025-07-15 (#59010) --- deps/checksums/cacert-2025-05-20.pem/md5 | 1 - deps/checksums/cacert-2025-05-20.pem/sha512 | 1 - deps/checksums/cacert-2025-07-15.pem/md5 | 1 + deps/checksums/cacert-2025-07-15.pem/sha512 | 1 + deps/libgit2.version | 2 +- stdlib/MozillaCACerts_jll/Project.toml | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/cacert-2025-05-20.pem/md5 delete mode 100644 deps/checksums/cacert-2025-05-20.pem/sha512 create mode 100644 deps/checksums/cacert-2025-07-15.pem/md5 create mode 100644 deps/checksums/cacert-2025-07-15.pem/sha512 diff --git a/deps/checksums/cacert-2025-05-20.pem/md5 b/deps/checksums/cacert-2025-05-20.pem/md5 deleted file mode 100644 index 06a21e784fc78..0000000000000 --- a/deps/checksums/cacert-2025-05-20.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -a4e2b0c77e807b80a4b8a58c411e4a15 diff --git a/deps/checksums/cacert-2025-05-20.pem/sha512 b/deps/checksums/cacert-2025-05-20.pem/sha512 deleted file mode 100644 index 4e5117cbf9f49..0000000000000 --- a/deps/checksums/cacert-2025-05-20.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -97bc802a7c055e6e58384920feb593596bc30bc9493a7550a168f4d7337d34166dc8f350713c468c605f81ed1c3b6380050f04e31b86b4877c9de90ce3512867 diff --git a/deps/checksums/cacert-2025-07-15.pem/md5 b/deps/checksums/cacert-2025-07-15.pem/md5 new file mode 100644 index 0000000000000..084a7e65d544f --- /dev/null +++ b/deps/checksums/cacert-2025-07-15.pem/md5 @@ -0,0 +1 @@ +ce594ef75f07eed66c538f7d4d83eafd diff --git a/deps/checksums/cacert-2025-07-15.pem/sha512 b/deps/checksums/cacert-2025-07-15.pem/sha512 new file mode 100644 index 0000000000000..7fb658508c700 --- /dev/null +++ b/deps/checksums/cacert-2025-07-15.pem/sha512 @@ -0,0 +1 @@ +743e96c112787f9b415310cb762ef9fec845534974e7e27bcec07472c346b1085fcbe89dad1f97699123800b37e29cb97f3fa1ffa6bb005af9743b4c74771fae diff --git a/deps/libgit2.version b/deps/libgit2.version index 71cea3aa9dd7b..90cdd89b83a29 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -11,4 +11,4 @@ LIBGIT2_SHA1=0060d9cf5666f015b1067129bd874c6cc4c9c7ac # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. # Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. -MOZILLA_CACERT_VERSION := 2025-05-20 +MOZILLA_CACERT_VERSION := 2025-07-15 diff --git a/stdlib/MozillaCACerts_jll/Project.toml b/stdlib/MozillaCACerts_jll/Project.toml index 57c5526a6f1f2..e1dbca5814708 100644 --- a/stdlib/MozillaCACerts_jll/Project.toml +++ b/stdlib/MozillaCACerts_jll/Project.toml @@ -1,7 +1,7 @@ name = "MozillaCACerts_jll" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" # Keep in sync with `deps/libgit2.version`. -version = "2025.05.20" +version = "2025.07.15" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From b45b429b93966e992c25a101c896e72738799ca4 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 17 Jul 2025 13:19:33 -0400 Subject: [PATCH 540/662] Fix use-after-free in FileWatching (#59017) We observe an abort on Windows on Revise master CI, where a free'd handle is passed to jl_close_uv. The root cause is that uv_fseventscb_file called uvfinalize earlier, but did not set the handle to NULL, so when the actual finalizer ran later, it would see corrupted state. --- stdlib/FileWatching/src/FileWatching.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/stdlib/FileWatching/src/FileWatching.jl b/stdlib/FileWatching/src/FileWatching.jl index ebfdd9c8fea6b..ddaf36dfd33a4 100644 --- a/stdlib/FileWatching/src/FileWatching.jl +++ b/stdlib/FileWatching/src/FileWatching.jl @@ -516,17 +516,19 @@ end function uvfinalize(uv::Union{FileMonitor, FolderMonitor}) iolock_begin() - if uv.handle != C_NULL - disassociate_julia_struct(uv) # close (and free) without notify - ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), uv.handle) + handle = @atomicswap :monotonic uv.handle = C_NULL + if handle != C_NULL + disassociate_julia_struct(handle) # close (and free) without notify + ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), handle) end iolock_end() end function close(t::Union{FileMonitor, FolderMonitor}) iolock_begin() - if t.handle != C_NULL - ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t.handle) + handle = t.handle + if handle != C_NULL + ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), handle) end iolock_end() end From d150fdf0550c2e5c7ea418f6522038b9140ccf5b Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 17 Jul 2025 16:22:29 -0400 Subject: [PATCH 541/662] Roll up msys2/clang/windows build fixes (#59003) This rolls up everything I had to change to get a successful source build of Julia under msys2. It's a misc collection of msys2, clang and other fixes. With this, I can use the following Make.user: ``` USE_SYSTEM_CSL=1 USE_BINARYBUILDER_LLVM=0 CC=clang CXX=clang++ FC=gfortran ``` The default USE_SYSTEM_CSL is broken due to #56840 With USE_SYSTEM_CSL=1, LLVM is broken due to #57021 Clang is required because gcc can't do an LLVM source build due to known export symbol size limits (ref JuliaPackaging/Yggdrasil#11652). That said, if we address the ABI issues in #56840, the default Make.user should build again (with BB-provided LLVM). --- Make.inc | 4 +++- base/linking.jl | 10 +++++++++- cli/loader.h | 4 ++-- cli/loader_win_utils.c | 8 ++++---- deps/Makefile | 5 +++-- deps/csl.mk | 22 +++++++++++++++++++--- deps/llvm.mk | 10 ++++++++-- deps/p7zip.mk | 2 +- deps/tools/common.mk | 14 ++++++++++++-- deps/zstd.mk | 2 ++ doc/src/devdocs/build/windows.md | 14 +++++++++++--- src/codegen.cpp | 8 ++++---- src/gf.c | 6 +++--- src/jl_uv.c | 3 +++ src/julia_internal.h | 9 +++++++++ src/module.c | 2 +- src/support/dtypes.h | 28 ++++++---------------------- sysimage.mk | 2 +- 18 files changed, 101 insertions(+), 52 deletions(-) diff --git a/Make.inc b/Make.inc index a3543a5e276f0..8701fcdfcfd7c 100644 --- a/Make.inc +++ b/Make.inc @@ -1580,7 +1580,9 @@ ifeq ($(OS), WINNT) HAVE_SSP := 1 OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(BUILDROOT)/src/julia.expmap \ $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -latomic -lole32 -JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-reloc +# N.B.: Unlike in the sysimage, we cannot -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc here, because libstdc++/LLVM are not fully correct under +# enforced visibility at this point. +JLDFLAGS += -Wl,--stack,8388608 ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware endif diff --git a/base/linking.jl b/base/linking.jl index a60e27c3051a8..edb70e8061119 100644 --- a/base/linking.jl +++ b/base/linking.jl @@ -140,7 +140,15 @@ function link_image_cmd(path, out) LIBS = isdebugbuild() ? ("-ljulia-debug", "-ljulia-internal-debug") : ("-ljulia", "-ljulia-internal") @static if Sys.iswindows() - LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") + LIBS = (LIBS..., "-lopenlibm", "-lgcc_s", "-lgcc", "-lmsvcrt") + if isdebugbuild() + LIBS = (LIBS..., "-lssp") + if isfile(joinpath(private_libdir(), "libmingwex.a")) + # In MinGW 11, the ssp implementation was moved from libssp to + # libmingwex with ssp only being a stub. See #59020. + LIBS = (LIBS..., "-lmingwex", "-lkernel32") + end + end end V = verbose_linking() ? "--verbose" : "" diff --git a/cli/loader.h b/cli/loader.h index be5195583b29f..310226c84f815 100644 --- a/cli/loader.h +++ b/cli/loader.h @@ -36,9 +36,9 @@ // Borrow definition from `support/dtypes.h` #ifdef _OS_WINDOWS_ # ifdef JL_LIBRARY_EXPORTS -# define JL_DLLEXPORT __declspec(dllexport) +# define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default"))) # endif -# define JL_DLLIMPORT __declspec(dllimport) +# define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #define JL_HIDDEN #else # define JL_DLLIMPORT __attribute__ ((visibility("default"))) diff --git a/cli/loader_win_utils.c b/cli/loader_win_utils.c index ed585a7a64ff0..34fe277fb2879 100644 --- a/cli/loader_win_utils.c +++ b/cli/loader_win_utils.c @@ -12,6 +12,10 @@ static FILE _stderr = { INVALID_HANDLE_VALUE }; FILE *stdout = &_stdout; FILE *stderr = &_stderr; +void JL_HIDDEN free(void* mem) { + HeapFree(GetProcessHeap(), 0, mem); +} + int JL_HIDDEN fwrite(const char *str, size_t nchars, FILE *out) { DWORD written; if (out->isconsole) { @@ -44,10 +48,6 @@ void JL_HIDDEN *realloc(void * mem, const size_t size) { return HeapReAlloc(GetProcessHeap(), HEAP_GENERATE_EXCEPTIONS, mem, size); } -void JL_HIDDEN free(void* mem) { - HeapFree(GetProcessHeap(), 0, mem); -} - LPWSTR *CommandLineToArgv(LPWSTR lpCmdLine, int *pNumArgs) { LPWSTR out = lpCmdLine; LPWSTR cmd = out; diff --git a/deps/Makefile b/deps/Makefile index 303520fb45e80..392b4fad2b2e2 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -42,9 +42,10 @@ ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0) DEP_LIBS += blastrampoline endif -ifeq ($(USE_SYSTEM_CSL), 0) +# We need to run this whether or not USE_SYSTEM_CSL is set. +# If it is, this target copies the system CSLs into the location our +# build system expects. DEP_LIBS += csl -endif ifeq ($(SANITIZE), 1) DEP_LIBS += sanitizers diff --git a/deps/csl.mk b/deps/csl.mk index 8319a0aeb485f..51368187c55fc 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -2,10 +2,10 @@ STD_LIB_PATH := $(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^programs: =' | sed -e "s/^programs: =//") STD_LIB_PATH += $(PATHSEP)$(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^libraries: =' | sed -e "s/^libraries: =//") ifeq ($(BUILD_OS),WINNT) # the mingw compiler lies about it search directory paths -STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") +STD_LIB_PATH += $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") endif -# Given a colon-separated list of paths in $(2), find the location of the library given in $(1) +# Given a $(PATHSEP)-separated list of paths in $(2), find the location of the library given in $(1) define pathsearch $(firstword $(wildcard $(addsuffix /$(1),$(subst $(PATHSEP), ,$(2))))) endef @@ -54,6 +54,13 @@ $$(build_shlibdir)/$(1): | $$(build_shlibdir) [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_shlibdir)' endef +define copy_csl_static_lib +install-csl: | $$(build_private_libdir) $$(build_private_libdir)/$(1) +$$(build_private_libdir)/$(1): | $$(build_private_libdir) + -@SRC_LIB='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \ + [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_private_libdir)' +endef + # libgfortran has multiple names; we're just going to copy any version we can find # Since we're only looking in the location given by `$(FC)` this should only succeed for one. $(eval $(call copy_csl,$(call versioned_libname,libgfortran,3))) @@ -63,11 +70,20 @@ $(eval $(call copy_csl,$(call versioned_libname,libgfortran,5))) # These are all libraries that we should always have $(eval $(call copy_csl,$(call versioned_libname,libquadmath,0))) $(eval $(call copy_csl,$(call versioned_libname,libstdc++,6))) -$(eval $(call copy_csl,$(call versioned_libname,libssp,0))) $(eval $(call copy_csl,$(call versioned_libname,libatomic,1))) $(eval $(call copy_csl,$(call versioned_libname,libgomp,1))) +# Configurable either a static or dynamic library depending on the system +$(eval $(call copy_csl,$(call versioned_libname,libssp,0))) +$(eval $(call copy_csl_static_lib,libssp.a)) + ifeq ($(OS),WINNT) +# On windows we need the static gcc runtime libraries for linking pkgimages +$(eval $(call copy_csl_static_lib,libgcc.a)) +$(eval $(call copy_csl_static_lib,libgcc_s.a)) +$(eval $(call copy_csl_static_lib,libmsvcrt.a)) +$(eval $(call copy_csl_static_lib,libmingwex.a)) +$(eval $(call copy_csl_static_lib,libkernel32.a)) # Windows has special gcc_s names ifeq ($(ARCH),i686) $(eval $(call copy_csl,$(call versioned_libname,libgcc_s_sjlj,1))) diff --git a/deps/llvm.mk b/deps/llvm.mk index ec606a98c8484..7c4acff9e537b 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -292,6 +292,12 @@ $(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-e echo 1 > $@ $(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_BUILDDIR_withtype)/build-configured +ifeq ($(OS),WINNT) +ifeq ($(USEGCC),1) + echo "LLVM source build is currently known to fail using GCC due to exceeded export table limits. Try clang." + exit 1 +endif +endif cd $(LLVM_BUILDDIR_withtype) && \ $(if $(filter $(CMAKE_GENERATOR),make), \ $(MAKE), \ @@ -306,8 +312,8 @@ endif echo 1 > $@ LLVM_INSTALL = \ - cd $1 && mkdir -p $2$$(build_depsbindir) && \ - cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit $2$$(build_depsbindir)/ && \ + cd $1 && mkdir -p $2$$(build_depsbindir)/lit && \ + cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/{*.py,*.toml,lit/} $2$$(build_depsbindir)/lit/ && \ $$(CMAKE) -DCMAKE_INSTALL_PREFIX="$2$$(build_prefix)" -P cmake_install.cmake ifeq ($(OS), WINNT) LLVM_INSTALL += && cp $2$$(build_shlibdir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_depsbindir) diff --git a/deps/p7zip.mk b/deps/p7zip.mk index b817db31c7cba..cbc850a1d5280 100644 --- a/deps/p7zip.mk +++ b/deps/p7zip.mk @@ -19,7 +19,7 @@ checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured - $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za$(EXE) + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za echo 1 > $@ define P7ZIP_INSTALL diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 54cc8f62e9c82..5e4b9ce0b9b62 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -5,8 +5,15 @@ # apparently not on FreeBSD). Ref PR #22352 CONFIGURE_COMMON = --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) --bindir=$(abspath $(build_depsbindir)) $(CUSTOM_LD_LIBRARY_PATH) + +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) + ifneq ($(XC_HOST),) CONFIGURE_COMMON += --host=$(XC_HOST) +else +# Defeat bad automatic cross compile detection (e.g. clang on mingw) +# CMAKE_COMMON += -DCMAKE_CROSSCOMPILING=0 endif ifeq ($(OS),WINNT) CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608" @@ -15,8 +22,6 @@ CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN) $(SANITIZE_LDFLA endif CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(SANITIZE_OPTS)" CXX="$(CXX) $(SANITIZE_OPTS)" LD="$(LD)" -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(OS),WINNT) CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) endif @@ -61,7 +66,12 @@ endif CMAKE_COMMON += -DCMAKE_LINKER="$$(which $(LD))" -DCMAKE_AR="$$(which $(AR))" -DCMAKE_RANLIB="$$(which $(RANLIB))" ifeq ($(OS),WINNT) +ifeq ($(BUILD_OS),WINNT) +# Don't make CMake think we're cross compiling, but do make sure it knows we're Windows +CMAKE_COMMON += -DCMAKE_HOST_SYSTEM_NAME=Windows +else CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows +endif CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)" endif diff --git a/deps/zstd.mk b/deps/zstd.mk index 5ead77641858a..73cdde2b6b27c 100644 --- a/deps/zstd.mk +++ b/deps/zstd.mk @@ -3,6 +3,8 @@ ifneq ($(USE_BINARYBUILDER_ZSTD), 1) ZSTD_GIT_URL := https://github.com/facebook/zstd.git ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 $(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) +# See note in llvm.mk for source tarballs with symlinks to non-existent targets +$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index 4fd9f01a81a0f..ed35ea10d198b 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -147,13 +147,13 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. For 64 bit Julia, install the x86_64 version: ``` - pacman -S mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake + pacman -S mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-clang ``` For 32 bit Julia, install the i686 version: ``` - pacman -S mingw-w64-i686-gcc mingw-w64-i686-cmake + pacman -S mingw-w64-i686-gcc mingw-w64-i686-cmake mingw-w64-i686-clang ``` 5. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. @@ -171,7 +171,15 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. cd julia ``` - 3. Start the build + 3. If you want to use clang (currently required if building LLVM from source), put the following in your Make.user + ``` + CC=/mingw64/bin/clang + CXX=/mingw64/bin/clang++ + ``` +!!! warning "UCRT Unsupported" + Do not try to use any other clang that MSYS2 may install (which may not have the correct default target) or the "Clang" environment(which defaults to the currently unsupported ucrt). + + 4. Start the build ``` make -j$(nproc) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2c38ea4e5d907..3c37a80e5e088 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -239,13 +239,13 @@ extern void __stack_chk_fail(); #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) extern void ___chkstk_ms(void); #else extern void __chkstk(void); #endif #else -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) #undef _alloca extern void _alloca(void); #else @@ -10050,13 +10050,13 @@ static void init_jit_functions(void) #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) add_named_global("__julia_personality", &__julia_personality); -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) add_named_global("___chkstk_ms", &___chkstk_ms); #else add_named_global("__chkstk", &__chkstk); #endif #else -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) add_named_global("_alloca", &_alloca); #else add_named_global("_chkstk", &_chkstk); diff --git a/src/gf.c b/src/gf.c index 82769d2794d9f..63350928bbaf8 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3369,19 +3369,19 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t return codeinst; } -JL_DLLEXPORT jl_value_t *jl_fptr_const_return(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_const_return(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { return m->rettype_const; } -JL_DLLEXPORT jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { jl_fptr_args_t invoke = jl_atomic_load_relaxed(&m->specptr.fptr1); assert(invoke && "Forgot to set specptr for jl_fptr_args!"); return invoke(f, args, nargs); } -JL_DLLEXPORT jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { jl_svec_t *sparams = jl_get_ci_mi(m)->sparam_vals; assert(sparams != jl_emptysvec); diff --git a/src/jl_uv.c b/src/jl_uv.c index a21b05433b8c6..24da8629faf02 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -8,6 +8,9 @@ #include #include +// Needs to come before windows platform headers +#include "support/dtypes.h" + #ifdef _OS_WINDOWS_ #include #include diff --git a/src/julia_internal.h b/src/julia_internal.h index c87f2cd648098..7df8e6bca499b 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -182,7 +182,16 @@ JL_DLLIMPORT void __tsan_switch_to_fiber(void *fiber, unsigned flags); #endif #endif +#if defined(HAVE_SSP) && defined(_OS_DARWIN_) +// On Darwin, this is provided by libSystem and imported +extern JL_DLLIMPORT uintptr_t __stack_chk_guard; +#elif defined(HAVE_SSP) +// Added by compiler runtime in final link - not DLLIMPORT +extern uintptr_t __stack_chk_guard; +#else +// The system doesn't have it - we define our own extern JL_DLLEXPORT uintptr_t __stack_chk_guard; +#endif // If this is detected in a backtrace of segfault, it means the functions // that use this value must be reworked into their async form with cb arg diff --git a/src/module.c b/src/module.c index 465a65c840db9..59968e2fd6cca 100644 --- a/src/module.c +++ b/src/module.c @@ -1597,7 +1597,7 @@ JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *va jl_checked_assignment(bp, m, var, val); } -JL_DLLEXPORT void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported) +void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported) { // this function is only valid during initialization, so there is no risk of data races her are not too important to use int kind = PARTITION_KIND_CONST | (exported ? PARTITION_FLAG_EXPORTED : 0); diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 6513370da4dae..2ee2d3529c10d 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -25,6 +25,9 @@ #include #include #define WIN32_LEAN_AND_MEAN +/* Clang does not like fvisibility=hidden with windows headers. This adds the visibility attribute there. + Arguably this is a clang bug. */ +#define DECLSPEC_IMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #include #if defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) @@ -37,21 +40,6 @@ typedef intptr_t ssize_t; #endif /* defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) */ -#if !defined(_COMPILER_GCC_) - -#define strtoull _strtoui64 -#define strtoll _strtoi64 -#define strcasecmp _stricmp -#define strncasecmp _strnicmp -#define snprintf _snprintf -#define stat _stat - -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#define STDERR_FILENO 2 - -#endif /* !_COMPILER_GCC_ */ - #endif /* _OS_WINDOWS_ */ @@ -73,13 +61,13 @@ typedef intptr_t ssize_t; #ifdef _OS_WINDOWS_ #define STDCALL __stdcall # ifdef JL_LIBRARY_EXPORTS_INTERNAL -# define JL_DLLEXPORT __declspec(dllexport) +# define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default"))) # endif # ifdef JL_LIBRARY_EXPORTS_CODEGEN -# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) +# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) __attribute__ ((visibility("default"))) # endif #define JL_HIDDEN -#define JL_DLLIMPORT __declspec(dllimport) +#define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #else #define STDCALL #define JL_DLLIMPORT __attribute__ ((visibility("default"))) @@ -123,11 +111,7 @@ typedef intptr_t ssize_t; #define STATIC_INLINE static inline #define FORCE_INLINE static inline __attribute__((always_inline)) -#ifdef _OS_WINDOWS_ -#define EXTERN_INLINE_DECLARE inline -#else #define EXTERN_INLINE_DECLARE inline __attribute__ ((visibility("default"))) -#endif #define EXTERN_INLINE_DEFINE extern inline JL_DLLEXPORT #if defined(_OS_WINDOWS_) && !defined(_COMPILER_GCC_) diff --git a/sysimage.mk b/sysimage.mk index 6dbc1f2abf3e5..bdd69dc6e31c0 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -18,7 +18,7 @@ $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%-o.a @$(call PRINT_LINK, $(CXX) $(LDFLAGS) -shared $(fPIC) -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ \ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) \ $(if $(findstring -debug,$(notdir $@)),-ljulia-internal-debug -ljulia-debug,-ljulia-internal -ljulia) \ - $$([ $(OS) = WINNT ] && echo '' $(LIBM) -lssp --disable-auto-import --disable-runtime-pseudo-reloc)) + $$([ $(OS) = WINNT ] && echo '' $(LIBM) -lssp -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc)) @$(INSTALL_NAME_CMD)$(notdir $@) $@ @$(DSYMUTIL) $@ From e4755dedfa55507c9ba6039d503467b031b52749 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Thu, 17 Jul 2025 22:37:09 +0100 Subject: [PATCH 542/662] Fix tar command (#59026) Scheduled build failing with ``` cd [buildroot]/deps/srccache/ && /usr/bin/tar --no-same-owner -xfz [buildroot]/deps/srccache/libunwind-1.8.2.tar.gz /usr/bin/tar: z: Cannot open: No such file or directory ``` Issue probably introduced in https://github.com/JuliaLang/julia/pull/58796. According to chatgpt this will fix it --- deps/unwind.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/unwind.mk b/deps/unwind.mk index c11338d0162ed..12c5cba543da4 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -20,7 +20,7 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz: | $(SRCCACHE) $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted: $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) -xfz $< + cd $(dir $<) && $(TAR) -xzf $< touch -c $(SRCCACHE)/libunwind-$(UNWIND_VER)/configure # old target echo 1 > $@ From 3b2918785597f9b7b05f7b3e479ea89716ca22bd Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 18 Jul 2025 14:32:10 -0400 Subject: [PATCH 543/662] Add 'sysimage' keyword for `JULIA_CPU_TARGET` to match (or extend) the sysimage target (#58970) --- NEWS.md | 2 ++ base/sysinfo.jl | 18 ++++++++++++ doc/src/base/base.md | 1 + doc/src/manual/environment-variables.md | 24 ++++++++++++---- pkgimage.mk | 2 +- src/aotcompile.cpp | 13 +++++++-- src/jl_exported_funcs.inc | 1 + src/processor.cpp | 37 +++++++++++++++++++++++++ src/processor.h | 7 +++++ test/cmdlineargs.jl | 8 ++++++ 10 files changed, 105 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5d9bf83467b77..018a556698549 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,6 +26,7 @@ Command-line option changes --------------------------- * The option `--sysimage-native-code=no` has been deprecated. +* The `JULIA_CPU_TARGET` environment variable now supports a `sysimage` keyword to match (or extend) the CPU target used to build the current system image ([#58970]). Multi-threading changes ----------------------- @@ -47,6 +48,7 @@ New library functions * `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). * Exporting function `fieldindex` to get the index of a struct's field ([#58119]). * `Base.donotdelete` is now public. It prevents deadcode elemination of its arguments ([#55774]). +* `Sys.sysimage_target()` returns the CPU target string used to build the current system image ([#58970]). New library features -------------------- diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 5e1a26973331e..06e5cd298caf1 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -16,6 +16,7 @@ export BINDIR, JIT, cpu_info, cpu_summary, + sysimage_target, uptime, loadavg, free_memory, @@ -312,6 +313,23 @@ function cpu_info() return cpus end +""" + Sys.sysimage_target() + +Return the CPU target string that was used to build the current system image. + +This function returns the original CPU target specification that was passed to Julia +when the system image was compiled. This can be useful for reproducing the same +system image or for understanding what CPU features were enabled during compilation. + +If the system image was built with the default settings this will return `"native"`. + +See also [`JULIA_CPU_TARGET`](@ref). +""" +function sysimage_target() + return ccall(:jl_get_sysimage_cpu_target, Ref{String}, ()) +end + """ Sys.uptime() diff --git a/doc/src/base/base.md b/doc/src/base/base.md index fafe08a6e5125..7721f6b9f6482 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -397,6 +397,7 @@ Base.Sys.total_memory Base.Sys.free_physical_memory Base.Sys.total_physical_memory Base.Sys.uptime +Base.Sys.sysimage_target Base.Sys.isjsvm Base.Sys.loadavg Base.Sys.isexecutable diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index 8cbb86188febd..636f6711cdbdd 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -479,9 +479,15 @@ stored in memory. Valid values for [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) can be obtained by executing `julia -C help`. +To get the CPU target string that was used to build the current system image, +use [`Sys.sysimage_target()`](@ref). This can be useful for reproducing +the same system image or understanding what CPU features were enabled during compilation. + Setting [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) is important for heterogeneous compute systems where processors of distinct types or features may be present. This is commonly encountered in high performance -computing (HPC) clusters since the component nodes may be using distinct processors. +computing (HPC) clusters since the component nodes may be using distinct processors. In this case, +you may want to use the `sysimage` CPU target to maintain the same configuration as the sysimage. +See below for more details. The CPU target string is a list of strings separated by `;` each string starts with a CPU or architecture name and followed by an optional list of features separated by `,`. @@ -490,13 +496,21 @@ which is at least the architecture the C/C++ runtime is compiled with. Each stri is interpreted by LLVM. A few special features are supported: -1. `clone_all` + +1. `sysimage` + + A special keyword that can be used as a CPU target name, which will be replaced + with the CPU target string that was used to build the current system image. This allows + you to specify CPU targets that build upon or extend the current sysimage's target, which + is particularly helpful for creating package images that are as flexible as the sysimage. + +2. `clone_all` This forces the target to have all functions in sysimg cloned. When used in negative form (i.e. `-clone_all`), this disables full clone that's enabled by default for certain targets. -2. `base([0-9]*)` +3. `base([0-9]*)` This specifies the (0-based) base target index. The base target is the target that the current target is based on, i.e. the functions that are not being cloned @@ -504,11 +518,11 @@ A few special features are supported: fully cloned (as if `clone_all` is specified for it) if it is not the default target (0). The index can only be smaller than the current index. -3. `opt_size` +4. `opt_size` Optimize for size with minimum performance impact. Clang/GCC's `-Os`. -4. `min_size` +5. `min_size` Optimize only for size. Clang's `-Oz`. diff --git a/pkgimage.mk b/pkgimage.mk index 9217573ab623f..ed5e1095c0229 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -26,7 +26,7 @@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') $(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled - @$(call PRINT_JULIA, JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ + @$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') touch $@ diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index f8057e6a013e8..ae5e826a6152f 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -2344,7 +2344,15 @@ void jl_dump_native_impl(void *native_code, "jl_small_typeof"); jl_small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility); jl_small_typeof_copy->setDSOLocal(true); - AT = ArrayType::get(T_psize, 5); + + // Create CPU target string constant + auto cpu_target_str = jl_options.cpu_target ? jl_options.cpu_target : "native"; + auto cpu_target_data = ConstantDataArray::getString(Context, cpu_target_str, true); + auto cpu_target_global = new GlobalVariable(metadataM, cpu_target_data->getType(), true, + GlobalVariable::InternalLinkage, + cpu_target_data, "jl_cpu_target_string"); + + AT = ArrayType::get(T_psize, 6); auto pointers = new GlobalVariable(metadataM, AT, false, GlobalVariable::ExternalLinkage, ConstantArray::get(AT, { @@ -2352,7 +2360,8 @@ void jl_dump_native_impl(void *native_code, ConstantExpr::getBitCast(shards, T_psize), ConstantExpr::getBitCast(ptls, T_psize), ConstantExpr::getBitCast(jl_small_typeof_copy, T_psize), - ConstantExpr::getBitCast(target_ids, T_psize) + ConstantExpr::getBitCast(target_ids, T_psize), + ConstantExpr::getBitCast(cpu_target_global, T_psize) }), "jl_image_pointers"); addComdat(pointers, TheTriple); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 5db67290dcb36..34978e97abd54 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -190,6 +190,7 @@ XX(jl_check_binding_currently_writable) \ XX(jl_get_cpu_name) \ XX(jl_get_cpu_features) \ + XX(jl_get_sysimage_cpu_target) \ XX(jl_cpu_has_fma) \ XX(jl_get_current_task) \ XX(jl_get_default_sysimg_path) \ diff --git a/src/processor.cpp b/src/processor.cpp index 6f95ee7f3790a..52c8f51741405 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -506,6 +506,20 @@ parse_cmdline(const char *option, F &&feature_cb) if (!option) abort(); + // Preprocess the option string to expand "sysimage" keyword + std::string processed_option; + if (strncmp(option, "sysimage", 8) == 0 && (option[8] == '\0' || option[8] == ';')) { + // Replace "sysimage" with the actual sysimage CPU target + jl_value_t *target_str = jl_get_sysimage_cpu_target(); + if (target_str != nullptr) { + processed_option = std::string(jl_string_data(target_str), jl_string_len(target_str)); + if (option[8] == ';') { + processed_option += option + 8; // append the rest after "sysimage" + } + option = processed_option.c_str(); + } + } + llvm::SmallVector, 0> res; TargetData arg{}; auto reset_arg = [&] { @@ -633,6 +647,12 @@ static inline jl_image_t parse_sysimg(jl_image_buf_t image, F &&callback, void * const jl_image_pointers_t *pointers = (const jl_image_pointers_t *)image.pointers; const void *ids = pointers->target_data; + + // Set the sysimage CPU target from the stored string + if (pointers->cpu_target_string) { + jl_set_sysimage_cpu_target(pointers->cpu_target_string); + } + jl_value_t* rejection_reason = nullptr; JL_GC_PUSH1(&rejection_reason); uint32_t target_idx = callback(ctx, ids, &rejection_reason); @@ -1002,6 +1022,9 @@ static std::string jl_get_cpu_features_llvm(void) #endif +// Global variable to store the CPU target string used for the sysimage +static std::string sysimage_cpu_target; + JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) { return jl_cstr_to_string(host_cpu_name().c_str()); @@ -1038,3 +1061,17 @@ extern "C" JL_DLLEXPORT void jl_reflect_feature_names(const FeatureName **fnames *fnames = feature_names; *nf = nfeature_names; } + +extern "C" JL_DLLEXPORT jl_value_t *jl_get_sysimage_cpu_target(void) { + if (sysimage_cpu_target.empty()) { + return jl_cstr_to_string("native"); + } + return jl_cstr_to_string(sysimage_cpu_target.c_str()); +} + +// Function to set the sysimage CPU target (called during initialization) +void jl_set_sysimage_cpu_target(const char *cpu_target) { + if (cpu_target) { + sysimage_cpu_target = cpu_target; + } +} diff --git a/src/processor.h b/src/processor.h index 65b634fd0ba26..b22e12d0aa4c0 100644 --- a/src/processor.h +++ b/src/processor.h @@ -195,6 +195,8 @@ typedef struct { // This contains the number of targets // in addition to the name and feature set of each target. const void *target_data; + // Original CPU target string used to build this sysimage + const char *cpu_target_string; } jl_image_pointers_t; /** @@ -210,10 +212,15 @@ typedef struct { jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target); jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image); +// Internal function to set the sysimage CPU target during initialization +void jl_set_sysimage_cpu_target(const char *cpu_target); + // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); // Return the features of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void); +// Return the CPU target string used to build the current sysimage +JL_DLLEXPORT jl_value_t *jl_get_sysimage_cpu_target(void); // Dump the name and feature set of the host CPU JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits); // Check if the CPU has native FMA instructions; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 4b0c8b6b59a8f..4bf8f62243838 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1028,6 +1028,14 @@ end @test v[2] == "" @test contains(v[3], "More than one command line CPU targets specified") end + + # Testing this more precisely would be very platform and build system dependent and brittle. + withenv("JULIA_CPU_TARGET" => "sysimage") do + v = readchomp(`$julia_path -E "Sys.sysimage_target()"`) + # Local builds will likely be "native" but CI shouldn't be. + invalid_results = Base.get_bool_env("CI", false) ? ("", "native", "sysimage") : ("", "sysimage",) + @test !in(v, invalid_results) + end end # Find the path of libjulia (or libjulia-debug, as the case may be) From bea90a2f503018115b30559b9d6838c05a86b9b4 Mon Sep 17 00:00:00 2001 From: Miles Cranmer Date: Fri, 18 Jul 2025 22:05:35 +0200 Subject: [PATCH 544/662] add `@__FUNCTION__` and `Expr(:thisfunction)` as generic function self-reference (#58940) This PR adds `@__FUNCTION__` to match the naming conventions of existing reflection macros (`@__MODULE__`, `@__FILE__`, etc.). --------- Co-authored-by: Jeff Bezanson --- NEWS.md | 1 + base/exports.jl | 1 + base/runtime_internals.jl | 37 ++++++ doc/src/base/base.md | 1 + doc/src/manual/performance-tips.md | 34 ++++++ src/ast.scm | 1 + src/julia-syntax.scm | 43 ++++++- test/syntax.jl | 179 +++++++++++++++++++++++++++++ 8 files changed, 293 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 018a556698549..38e3eb1fc379c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,6 +6,7 @@ New language features - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) - New `nth` function to access the `n`-th element of a generic iterable. ([#56580]) + - New `@__FUNCTION__` macro to refer to the innermost enclosing function. ([#58909]) - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. ([JuliaLang/JuliaSyntax.jl#525], [#57143]) diff --git a/base/exports.jl b/base/exports.jl index 53f6152ea55f2..2c30f095a3998 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1059,6 +1059,7 @@ export @__DIR__, @__LINE__, @__MODULE__, + @__FUNCTION__, @int128_str, @uint128_str, @big_str, diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 98dd111ccbf68..bb5c09d80db43 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -173,6 +173,43 @@ false """ ispublic(m::Module, s::Symbol) = ccall(:jl_module_public_p, Cint, (Any, Any), m, s) != 0 +""" + @__FUNCTION__ + +Get the innermost enclosing function object. + +!!! note + `@__FUNCTION__` has the same scoping behavior as `return`: when used + inside a closure, it refers to the closure and not the outer function. + Some macros, including [`@spawn`](@ref Threads.@spawn), [`@async`](@ref), etc., + wrap their input in closures. When `@__FUNCTION__` is used within such code, + it will refer to the closure created by the macro rather than the enclosing function. + +# Examples + +`@__FUNCTION__` enables recursive anonymous functions: + +```jldoctest +julia> factorial = (n -> n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1)); + +julia> factorial(5) +120 +``` + +`@__FUNCTION__` can be combined with `nameof` to identify a function's +name from within its body: + +```jldoctest +julia> bar() = nameof(@__FUNCTION__); + +julia> bar() +:bar +``` +""" +macro __FUNCTION__() + Expr(:thisfunction) +end + # TODO: this is vaguely broken because it only works for explicit calls to # `Base.deprecate`, not the @deprecated macro: isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0 diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 7721f6b9f6482..ab4bfdb6b105b 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -481,6 +481,7 @@ Base.moduleroot __module__ __source__ Base.@__MODULE__ +Base.@__FUNCTION__ Base.@__FILE__ Base.@__DIR__ Base.@__LINE__ diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index fa197196dad4f..90a40337e081a 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -919,6 +919,40 @@ In the mean time, some user-contributed packages like [FastClosures](https://github.com/c42f/FastClosures.jl) automate the insertion of `let` statements as in `abmult3`. +#### Use `@__FUNCTION__` for recursive closures + +For recursive closures specifically, the [`@__FUNCTION__`](@ref) macro can avoid both type instability and boxing. + +First, let's see the unoptimized version: + +```julia +function make_fib_unoptimized() + fib(n) = n <= 1 ? 1 : fib(n - 1) + fib(n - 2) # fib is boxed + return fib +end +``` + +The `fib` function is boxed, meaning the return type is inferred as `Any`: + +```julia +@code_warntype make_fib_unoptimized() +``` + +Now, to eliminate this type instability, we can instead use `@__FUNCTION__` to refer to the concrete function object: + +```julia +function make_fib_optimized() + fib(n) = n <= 1 ? 1 : (@__FUNCTION__)(n - 1) + (@__FUNCTION__)(n - 2) + return fib +end +``` + +This gives us a concrete return type: + +```julia +@code_warntype make_fib_optimized() +``` + ### [Types with values-as-parameters](@id man-performance-value-type) diff --git a/src/ast.scm b/src/ast.scm index 15e55fc616041..ea538e0aede4e 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -466,6 +466,7 @@ (define (make-assignment l r) `(= ,l ,r)) (define (assignment? e) (and (pair? e) (eq? (car e) '=))) (define (return? e) (and (pair? e) (eq? (car e) 'return))) +(define (thisfunction? e) (and (pair? e) (eq? (car e) 'thisfunction))) (define (tuple-call? e) (and (length> e 1) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index a73044a228b5a..052e0000ebe87 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -549,7 +549,9 @@ (insert-after-meta `(block ,@stmts) (cons `(meta nkw ,(+ (length vars) (length restkw))) - annotations)) + (if (has-thisfunction? `(block ,@stmts)) + (cons `(meta thisfunction-original ,(arg-name (car not-optional))) annotations) + annotations))) rett) ;; call with no keyword args @@ -2911,6 +2913,7 @@ 'generator (lambda (e) (check-no-return e) + (check-no-thisfunction e) (expand-generator e #f '())) 'flatten @@ -2995,6 +2998,13 @@ (if (has-return? e) (error "\"return\" not allowed inside comprehension or generator"))) +(define (has-thisfunction? e) + (expr-contains-p thisfunction? e (lambda (x) (not (function-def? x))))) + +(define (check-no-thisfunction e) + (if (has-thisfunction? e) + (error "\"@__FUNCTION__\" not allowed inside comprehension or generator"))) + (define (has-break-or-continue? e) (expr-contains-p (lambda (x) (and (pair? x) (memq (car x) '(break continue)))) e @@ -3003,6 +3013,7 @@ (define (lower-comprehension ty expr itrs) (check-no-return expr) + (check-no-thisfunction expr) (if (has-break-or-continue? expr) (error "break or continue outside loop")) (let ((result (make-ssavalue)) @@ -3434,7 +3445,7 @@ vi) tab)) -;; env: list of vinfo (includes any closure #self#; should not include globals) +;; env: list of vinfo (should not include globals) ;; captvars: list of vinfo ;; sp: list of symbol ;; new-sp: list of symbol (static params declared here) @@ -3855,7 +3866,7 @@ f(x) = yt(x) (Set '(quote top core lineinfo line inert local-def unnecessary copyast meta inbounds boundscheck loopinfo decl aliasscope popaliasscope thunk with-static-parameters toplevel-only - global globalref global-if-global assign-const-if-global isglobal thismodule + global globalref global-if-global assign-const-if-global isglobal thismodule thisfunction const atomic null true false ssavalue isdefined toplevel module lambda error gc_preserve_begin gc_preserve_end export public inline noinline purity))) @@ -4093,7 +4104,7 @@ f(x) = yt(x) ((atom? e) e) (else (case (car e) - ((quote top core global globalref thismodule lineinfo line break inert module toplevel null true false meta) e) + ((quote top core global globalref thismodule thisfunction lineinfo line break inert module toplevel null true false meta) e) ((toplevel-only) ;; hack to avoid generating a (method x) expr for struct types (if (eq? (cadr e) 'struct) @@ -5133,6 +5144,30 @@ f(x) = yt(x) ((error) (error (cadr e))) + + ;; thisfunction replaced with first argument name + ((thisfunction) + (let ((first-arg (and (pair? (lam:args lam)) (car (lam:args lam))))) + (if first-arg + (let* ((arg-name (arg-name first-arg)) + ;; Check for thisfunction-original metadata in keyword wrapper functions + (original-name (let ((body (lam:body lam))) + (and (pair? body) (pair? (cdr body)) + (let loop ((stmts (cdr body))) + (if (pair? stmts) + (let ((stmt (car stmts))) + (if (and (pair? stmt) (eq? (car stmt) 'meta) + (pair? (cdr stmt)) (eq? (cadr stmt) 'thisfunction-original) + (pair? (cddr stmt))) + (caddr stmt) + (loop (cdr stmts)))) + #f))))) + (final-name (or original-name arg-name))) + (cond (tail (emit-return tail final-name)) + (value final-name) + (else (emit final-name) #f))) + (error "\"@__FUNCTION__\" can only be used inside a function")))) + (else (error (string "invalid syntax " (deparse e))))))) ;; introduce new slots for assigned arguments diff --git a/test/syntax.jl b/test/syntax.jl index dcd921823d273..25a683a3b9b31 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1526,8 +1526,11 @@ end @test Meta.lower(@__MODULE__, :(return 0 for i=1:2)) == Expr(:error, "\"return\" not allowed inside comprehension or generator") @test Meta.lower(@__MODULE__, :([ return 0 for i=1:2 ])) == Expr(:error, "\"return\" not allowed inside comprehension or generator") @test Meta.lower(@__MODULE__, :(Int[ return 0 for i=1:2 ])) == Expr(:error, "\"return\" not allowed inside comprehension or generator") +@test Meta.lower(@__MODULE__, :([ $(Expr(:thisfunction)) for i=1:2 ])) == Expr(:error, "\"@__FUNCTION__\" not allowed inside comprehension or generator") +@test Meta.lower(@__MODULE__, :($(Expr(:thisfunction)) for i=1:2)) == Expr(:error, "\"@__FUNCTION__\" not allowed inside comprehension or generator") @test [ ()->return 42 for i = 1:1 ][1]() == 42 @test Function[ identity() do x; return 2x; end for i = 1:1 ][1](21) == 42 +@test @eval let f=[ ()->$(Expr(:thisfunction)) for i = 1:1 ][1]; f() === f; end # issue #27155 macro test27155() @@ -4351,3 +4354,179 @@ let f = NoSpecClosure.K(1) @test f(2) == 1 @test typeof(f).parameters == Core.svec() end + +@testset "@__FUNCTION__ and Expr(:thisfunction)" begin + @testset "Basic usage" begin + # @__FUNCTION__ in regular functions + test_function_basic() = @__FUNCTION__ + @test test_function_basic() === test_function_basic + + # Expr(:thisfunction) in regular functions + @eval regular_func() = $(Expr(:thisfunction)) + @test regular_func() === regular_func + end + + @testset "Recursion" begin + # Factorial with @__FUNCTION__ + factorial_function(n) = n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1) + @test factorial_function(5) == 120 + + # Fibonacci with Expr(:thisfunction) + struct RecursiveCallableStruct; end + @eval (::RecursiveCallableStruct)(n) = n <= 1 ? n : $(Expr(:thisfunction))(n-1) + $(Expr(:thisfunction))(n-2) + @test RecursiveCallableStruct()(10) === 55 + + # Anonymous function recursion + @test (n -> n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1))(5) == 120 + end + + @testset "Closures and nested functions" begin + # Prevents boxed closures + function make_closure() + fib(n) = n <= 1 ? 1 : (@__FUNCTION__)(n - 1) + (@__FUNCTION__)(n - 2) + return fib + end + Test.@inferred make_closure() + closure = make_closure() + @test closure(5) == 8 + Test.@inferred closure(5) + + # Complex closure of closures + function f1() + function f2() + function f3() + return @__FUNCTION__ + end + return (@__FUNCTION__), f3() + end + return (@__FUNCTION__), f2()... + end + Test.@inferred f1() + @test f1()[1] === f1 + @test f1()[2] !== f1 + @test f1()[3] !== f1 + @test f1()[3]() === f1()[3] + @test f1()[2]()[2]() === f1()[3] + end + + @testset "Do blocks" begin + function test_do_block() + result = map([1, 2, 3]) do x + return (@__FUNCTION__, x) + end + # All should refer to the same do-block function + @test all(r -> r[1] === result[1][1], result) + # Values should be different + @test [r[2] for r in result] == [1, 2, 3] + # It should be different than `test_do_block` + @test result[1][1] !== test_do_block + end + test_do_block() + end + + @testset "Keyword arguments" begin + # @__FUNCTION__ with kwargs + foo(; n) = n <= 1 ? 1 : n * (@__FUNCTION__)(; n = n - 1) + @test foo(n = 5) == 120 + + # Expr(:thisfunction) with kwargs + let + @eval f2(; n=1) = n <= 1 ? n : n * $(Expr(:thisfunction))(; n=n-1) + result = f2(n=5) + @test result == 120 + end + end + + @testset "Callable structs" begin + # @__FUNCTION__ in callable structs + @gensym A + @eval module $A + struct CallableStruct{T}; val::T; end + (c::CallableStruct)() = @__FUNCTION__ + end + @eval using .$A: CallableStruct + c = CallableStruct(5) + @test c() === c + + # In closures, var"#self#" should refer to the enclosing function, + # NOT the enclosing struct instance + struct CallableStruct2; end + @eval function (obj::CallableStruct2)() + function inner_func() + $(Expr(:thisfunction)) + end + inner_func + end + + let cs = CallableStruct2() + @test cs()() === cs() + @test cs()() !== cs + end + + # Accessing values via self-reference + struct CallableStruct3 + value::Int + end + @eval (obj::CallableStruct3)() = $(Expr(:thisfunction)) + @eval (obj::CallableStruct3)(x) = $(Expr(:thisfunction)).value + x + + let cs = CallableStruct3(42) + @test cs() === cs + @test cs(10) === 52 + end + + # Callable struct with args and kwargs + struct CallableStruct4 + end + @eval function (obj::CallableStruct4)(x, args...; y=2, kws...) + return (; func=(@__FUNCTION__), x, args, y, kws) + end + c = CallableStruct4() + @test c(1).func === c + @test c(2, 3).args == (3,) + @test c(2; y=4).y == 4 + @test c(2; y=4, a=5, b=6, c=7).kws[:c] == 7 + end + + @testset "Special cases" begin + # Generated functions + let @generated foo2() = Expr(:thisfunction) + @test foo2() === foo2 + end + + # Struct constructors + let + @eval struct Cols{T<:Tuple} + cols::T + operator + Cols(args...; operator=union) = (new{typeof(args)}(args, operator); string($(Expr(:thisfunction)))) + end + result = Cols(1, 2, 3) + @test occursin("Cols", result) + end + + # Should not access arg-map for local variables + @gensym f + @eval begin + function $f end + function ($f::typeof($f))() + $f = 1 + $(Expr(:thisfunction)) + end + end + @test @eval($f() === $f) + end + + @testset "Error upon misuse" begin + @gensym B + @test_throws( + "\"@__FUNCTION__\" can only be used inside a function", + @eval(module $B; @__FUNCTION__; end) + ) + + @test_throws( + "\"@__FUNCTION__\" not allowed inside comprehension or generator", + @eval([(@__FUNCTION__) for _ in 1:10]) + ) + end +end From c30199fe17aa24f77530bd64f7762546c15e2941 Mon Sep 17 00:00:00 2001 From: JonasIsensee Date: Fri, 18 Jul 2025 22:28:54 +0200 Subject: [PATCH 545/662] Bugfix: Use Base.aligned_sizeof instead of sizeof in Mmap.mmap (#58998) fix #58982 --- stdlib/Mmap/src/Mmap.jl | 8 ++++---- stdlib/Mmap/test/runtests.jl | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 7d57bf053940d..c527123d51bb8 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -187,16 +187,16 @@ like HDF5 (which can be used with memory-mapping). """ function mmap(io::IO, ::Type{Array{T,N}}=Vector{UInt8}, - dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), + dims::NTuple{N,Integer}=(div(filesize(io)-position(io),Base.aligned_sizeof(T)),), offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T,N} # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) - len = sizeof(T) + len = Base.aligned_sizeof(T) for l in dims len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...) - overflow && throw(ArgumentError("requested size prod($((sizeof(T), dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))")) + overflow && throw(ArgumentError("requested size prod($((len, dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))")) end len >= 0 || throw(ArgumentError("requested size must be ≥ 0, got $len")) len == 0 && return Array{T}(undef, ntuple(x->0,Val(N))) @@ -267,7 +267,7 @@ end mmap(file::AbstractString, ::Type{T}=Vector{UInt8}, - dims::NTuple{N,Integer}=(div(filesize(file),sizeof(eltype(T))),), + dims::NTuple{N,Integer}=(div(filesize(file),Base.aligned_sizeof(eltype(T))),), offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array,N} = open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index f4b6abb147fb7..064217a24e3be 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -44,7 +44,7 @@ s = open(file) @test length(@inferred mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 close(s) -@test_throws ErrorException mmap(file, Vector{Ref}) # must be bit-type +@test_throws ArgumentError mmap(file, Vector{Ref}) # must be bit-type GC.gc(); GC.gc() file = tempname() # new name to reduce chance of issues due slow windows fs @@ -343,6 +343,19 @@ end GC.gc() rm(file) +@testset "test for #58982 - mmap with primitive types" begin + file = tempname() + primitive type PrimType9Bytes 9*8 end + arr = Vector{PrimType9Bytes}(undef, 2) + write(file, arr) + m = mmap(file, Vector{PrimType9Bytes}) + @test length(m) == 2 + @test m[1] == arr[1] + @test m[2] == arr[2] + finalize(m); m = nothing; GC.gc() + rm(file) +end + @testset "Docstrings" begin @test isempty(Docs.undocumented_names(Mmap)) end From 5ebc5b463ead44e98a3108a477084bdd591e3d8e Mon Sep 17 00:00:00 2001 From: Miles Cranmer Date: Sat, 19 Jul 2025 15:16:08 +0200 Subject: [PATCH 546/662] Fix PR reference in NEWS (#59046) --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 38e3eb1fc379c..d46133b6244bc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,7 +6,7 @@ New language features - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) - New `nth` function to access the `n`-th element of a generic iterable. ([#56580]) - - New `@__FUNCTION__` macro to refer to the innermost enclosing function. ([#58909]) + - New `@__FUNCTION__` macro to refer to the innermost enclosing function. ([#58940]) - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. ([JuliaLang/JuliaSyntax.jl#525], [#57143]) From 8172db3798320678888fc3126ac0452c24237754 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 12:41:06 -0400 Subject: [PATCH 547/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?LibCURL=20stdlib=20from=20a65b64f=20to=20038790a=20(#59038)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 | 1 + .../sha512 | 1 + .../LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 | 1 - .../sha512 | 1 - stdlib/LibCURL.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 create mode 100644 deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 delete mode 100644 deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 delete mode 100644 deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 new file mode 100644 index 0000000000000..264c8e5ea6293 --- /dev/null +++ b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 @@ -0,0 +1 @@ +b3a67e92f5a9d5832699623e34abf1b2 diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 new file mode 100644 index 0000000000000..78f04bb4becb8 --- /dev/null +++ b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 @@ -0,0 +1 @@ +d558321aa6099ddb61cc402f85d3ea7f0d5c875aee1371f10e6b50c53d8b578045a132a3fd361b96fabf44658b364496d0377371d1ec232962fc8450eb242217 diff --git a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 b/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 deleted file mode 100644 index f14b87c21f5ed..0000000000000 --- a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -e8c53aa3fb963c80921787d5d565eb2c diff --git a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 b/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 deleted file mode 100644 index ab24e6a9516c3..0000000000000 --- a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -8e442ea834299df9c02acb87226c121395ad8e550025ac5ee1103df09c6ff43817e9e48dd1bcbc92c80331ef3ddff531962430269115179acbec2bab2de5b011 diff --git a/stdlib/LibCURL.version b/stdlib/LibCURL.version index 216ab4e7aca22..9bf12e7a8287b 100644 --- a/stdlib/LibCURL.version +++ b/stdlib/LibCURL.version @@ -1,4 +1,4 @@ LIBCURL_BRANCH = master -LIBCURL_SHA1 = a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0 +LIBCURL_SHA1 = 038790a793203248362cf2bd8d85e42f8c56a72d LIBCURL_GIT_URL := https://github.com/JuliaWeb/LibCURL.jl.git LIBCURL_TAR_URL = https://api.github.com/repos/JuliaWeb/LibCURL.jl/tarball/$1 From 412ea7b271523daf39fd4aac5d161dd979eb887e Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 12:41:32 -0400 Subject: [PATCH 548/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?DelimitedFiles=20stdlib=20from=20db79c84=20to=20a982d5c=20(#590?= =?UTF-8?q?36)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/DelimitedFiles.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 create mode 100644 deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 delete mode 100644 deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 delete mode 100644 deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 new file mode 100644 index 0000000000000..092778b7bb675 --- /dev/null +++ b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 @@ -0,0 +1 @@ +eef0f8463d4a7ba1c697f666e815b0da diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 new file mode 100644 index 0000000000000..abbec567981d6 --- /dev/null +++ b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 @@ -0,0 +1 @@ +e119db9546e698872e5e4cdf027a1c31e694f36c1e775a7d9361ad0db5d6a8e1804732090e68b40765a519ec3f7aa45d40c8c5e22275262303c100367bd681b3 diff --git a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 b/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 deleted file mode 100644 index 9c6e4e44927fe..0000000000000 --- a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ee5afca99801e37fd3a42a9455ae986b diff --git a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 b/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 deleted file mode 100644 index 69a50a7282781..0000000000000 --- a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2adec92de521df1668eb13f2903ffdb01efd6afa5f04ce6fbd1737caa4948f7b629cdda7f75a895853a0cd49dccf8b388860d5c19c29e4d4aad6c7f8fa6b7209 diff --git a/stdlib/DelimitedFiles.version b/stdlib/DelimitedFiles.version index d741690a96838..c3bd5449d11f4 100644 --- a/stdlib/DelimitedFiles.version +++ b/stdlib/DelimitedFiles.version @@ -1,4 +1,4 @@ DELIMITEDFILES_BRANCH = main -DELIMITEDFILES_SHA1 = db79c842f95f55b1f8d8037c0d3363ab21cd3b90 +DELIMITEDFILES_SHA1 = a982d5cf46061593c98b4d80fcc64dd42e6cba74 DELIMITEDFILES_GIT_URL := https://github.com/JuliaData/DelimitedFiles.jl.git DELIMITEDFILES_TAR_URL = https://api.github.com/repos/JuliaData/DelimitedFiles.jl/tarball/$1 From c1c091e45bba36272fac308c7fdfa6c803a22ce5 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 12:44:17 -0400 Subject: [PATCH 549/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?SHA=20stdlib=20from=204451e13=20to=20169a336=20(#59041)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 | 1 + .../SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 | 1 + .../SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 | 1 - .../SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 | 1 - stdlib/SHA.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 create mode 100644 deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 delete mode 100644 deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 delete mode 100644 deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 new file mode 100644 index 0000000000000..ef6edc13c3a2c --- /dev/null +++ b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 @@ -0,0 +1 @@ +ab91b3b95af44071020c564f6cb2b83f diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 new file mode 100644 index 0000000000000..26889e062b48d --- /dev/null +++ b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 @@ -0,0 +1 @@ +ea90fdf05a0c345ff697b91bea45ab5458123cb7d7bad53cebeea2304822545ec422012f116090d2c9491c185a9de61ad44b2eb71237a69e959737a140d8cf71 diff --git a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 b/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 deleted file mode 100644 index cbd2acb2c6f66..0000000000000 --- a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7b511b7dab411685206d0d90cc1fb56e diff --git a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 b/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 deleted file mode 100644 index 5201bbdcc40f9..0000000000000 --- a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -960367406d80e46e8e742bcb0f7f0e4b089b664c2321ca82953eb760b325693ae57f431d891ccf56c3ab9146bc29682d2d1767bc635f4dbe6dd4d80030a42487 diff --git a/stdlib/SHA.version b/stdlib/SHA.version index a5d4372d5798b..507ab1e368f79 100644 --- a/stdlib/SHA.version +++ b/stdlib/SHA.version @@ -1,4 +1,4 @@ SHA_BRANCH = master -SHA_SHA1 = 4451e1362e425bcbc1652ecf55fc0e525b18fb63 +SHA_SHA1 = 169a3369026ee767c454e5b1ca70c62c4db5a933 SHA_GIT_URL := https://github.com/JuliaCrypto/SHA.jl.git SHA_TAR_URL = https://api.github.com/repos/JuliaCrypto/SHA.jl/tarball/$1 From 912c57985dee1bff2fe364aba55d3e2dc27b5670 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 18:02:36 -0400 Subject: [PATCH 550/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=20b85e29428=20to=2038d2b366a=20(#59040)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 | 1 + .../Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 | 1 + .../Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 | 1 - .../Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 create mode 100644 deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 diff --git a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 new file mode 100644 index 0000000000000..bf0f78e3cacc7 --- /dev/null +++ b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 @@ -0,0 +1 @@ +36e6a30b59c45236fbf4a7dcf3c410ee diff --git a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 new file mode 100644 index 0000000000000..3aaa6e7d8231b --- /dev/null +++ b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 @@ -0,0 +1 @@ +018720df5f382678c9c81df189d60d7402c69dc694414420063d20406a1f38c5a31b66fb4177e73dab554629bcea7669cbb65d6b3c2d8035294c57cb3159e2ef diff --git a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 deleted file mode 100644 index d9020239bcd33..0000000000000 --- a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -61a22eacd9e91ddfee859436d79e1541 diff --git a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 b/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 deleted file mode 100644 index 8d9fa14f8b32b..0000000000000 --- a/deps/checksums/Pkg-b85e2942846c6ad3dc1db95fb29ddd6e4f262591.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -63caedfb686a93d9ebd6c511ab833e34de7993b825461632a9b47ec64f25e002aca1ec36fb7845cfb722c6d70d473eac2f92b6b9e0848f15d63848969aacdd43 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 52f07a6c1726e..0cb8d902db7a6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = b85e2942846c6ad3dc1db95fb29ddd6e4f262591 +PKG_SHA1 = 38d2b366a7fb8062f990c9897b1441b27bf63715 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 33e4e4668b412ae795f46934d54f62b8f7511602 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 18:03:55 -0400 Subject: [PATCH 551/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Statistics=20stdlib=20from=2077bd570=20to=2022dee82=20(#59043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/Statistics.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 create mode 100644 deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 delete mode 100644 deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 delete mode 100644 deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 new file mode 100644 index 0000000000000..c5f56d9064e92 --- /dev/null +++ b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 @@ -0,0 +1 @@ +0b60da1286ca8a978cf3c27b8fbc0601 diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 new file mode 100644 index 0000000000000..8cd97202f18ba --- /dev/null +++ b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 @@ -0,0 +1 @@ +2e03fe3b79dfb299caa0ac23e045bf26addeb3d38ef0b5e5430966dc227e771cfd84722d8bb80aaaedc0a988fbd2228bebf56c3e7d80b3e8993c623d8436660c diff --git a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 b/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 deleted file mode 100644 index 600c561d0cf14..0000000000000 --- a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -5235ac479da042d5dc3c572c473b7219 diff --git a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 b/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 deleted file mode 100644 index 2f663a3d7c44d..0000000000000 --- a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0c02ccf1b4988fc701209afb949f27e6f675f37a628385d3f28dc9ea333fed38ce1ca77b001e58fdbe15af833bbe98598cbf478cef21a98b37d54acfe52270b6 diff --git a/stdlib/Statistics.version b/stdlib/Statistics.version index e6f0a62b3cec5..e22fa135f74cd 100644 --- a/stdlib/Statistics.version +++ b/stdlib/Statistics.version @@ -1,4 +1,4 @@ STATISTICS_BRANCH = master -STATISTICS_SHA1 = 77bd5707f143eb624721a7df28ddef470e70ecef +STATISTICS_SHA1 = 22dee82f9824d6045e87aa4b97e1d64fe6f01d8d STATISTICS_GIT_URL := https://github.com/JuliaStats/Statistics.jl.git STATISTICS_TAR_URL = https://api.github.com/repos/JuliaStats/Statistics.jl/tarball/$1 From f70c380076245e499c93c8e4416d558e89083127 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 20 Jul 2025 18:04:42 -0400 Subject: [PATCH 552/662] Expand JULIA_CPU_TARGET docs (#58968) --- doc/src/devdocs/pkgimg.md | 6 ++++-- doc/src/manual/environment-variables.md | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 64f4e640b7c19..0bc28b07b0c29 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -33,8 +33,10 @@ Dynamic libraries on macOS need to link against `-lSystem`. On recent macOS vers To that effect we link with `-undefined dynamic_lookup`. ## [Package images optimized for multiple microarchitectures](@id pkgimgs-multi-versioning) -Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogeneous environment, with a unified cache, -you can set the environment variable `JULIA_CPU_TARGET=generic` to multi-version the object caches. + +Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. This allows creating package caches that can run efficiently on different CPU architectures within the same environment. + +See the [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) environment variable for more information on how to set the CPU target for package images. ## Flags that impact package image creation and selection diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index 636f6711cdbdd..4a3018c481276 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -495,6 +495,10 @@ A `generic` or empty CPU name means the basic required feature set of the target which is at least the architecture the C/C++ runtime is compiled with. Each string is interpreted by LLVM. +!!! note + Package images can only target the same or more specific CPU features than + their base system image. + A few special features are supported: 1. `sysimage` From 4d5122a5a07e26e1f0700232559dbd48c99ff3fe Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sun, 20 Jul 2025 19:08:04 -0400 Subject: [PATCH 553/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?LinearAlgebra=20stdlib=20from=203e4d569=20to=202c3fe9b=20(#5903?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> Co-authored-by: Ian Butterworth --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/LinearAlgebra.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 create mode 100644 deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 delete mode 100644 deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 delete mode 100644 deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 new file mode 100644 index 0000000000000..63847447093fd --- /dev/null +++ b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 @@ -0,0 +1 @@ +6ea57484fcb60da4bf9d7c646fe73b2e diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 new file mode 100644 index 0000000000000..fd0479374f265 --- /dev/null +++ b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 @@ -0,0 +1 @@ +7b75d79804e73a3fc7140fe0636b06a4d0267c96ee6d6b1dbe8eedeefd79a1f8a82501bed4a3038ba309284dc0291e7424463ced1f1b76eb20f5bd57c7c0a7b5 diff --git a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 deleted file mode 100644 index 5d2d7f6a868af..0000000000000 --- a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -00d7f962a817b254a248ce7078290c37 diff --git a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 b/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 deleted file mode 100644 index 4f02c121c62f1..0000000000000 --- a/deps/checksums/LinearAlgebra-3e4d569804bdc8eb891ff1982eb9e90246656d0c.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d7bbf0b7a90cb46ef63e69a0adf2c50ca52c81b679b0d9deb545d98c4400500746ad76be58e3ab8319a6e2d17a5e24fd10e5a0f4ac29e91f3fdd9084229166f1 diff --git a/stdlib/LinearAlgebra.version b/stdlib/LinearAlgebra.version index 5136a508fca27..dc220efcc37ea 100644 --- a/stdlib/LinearAlgebra.version +++ b/stdlib/LinearAlgebra.version @@ -1,4 +1,4 @@ LINEARALGEBRA_BRANCH = master -LINEARALGEBRA_SHA1 = 3e4d569804bdc8eb891ff1982eb9e90246656d0c +LINEARALGEBRA_SHA1 = 2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023 LINEARALGEBRA_GIT_URL := https://github.com/JuliaLang/LinearAlgebra.jl.git LINEARALGEBRA_TAR_URL = https://api.github.com/repos/JuliaLang/LinearAlgebra.jl/tarball/$1 From 0e1aa6c7ebb7e328e630ea206e01eb3f64a83d81 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Mon, 21 Jul 2025 01:04:53 -0400 Subject: [PATCH 554/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?SparseArrays=20stdlib=20from=206d072a8=20to=2030201ab=20(#59042?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 delete mode 100644 deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 new file mode 100644 index 0000000000000..a660fb96bd9d8 --- /dev/null +++ b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 @@ -0,0 +1 @@ +3615e2464cfab2c7d00ae53dedc0510b diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 new file mode 100644 index 0000000000000..be8fbd8d2e1a3 --- /dev/null +++ b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 @@ -0,0 +1 @@ +1998b694bd9451ac92517b70c25fc2683b1482ca1d547500cec8e09eecbb615167aea94c16cae899a32b5cb2fcb842ce081fa90641a2921ea3cfa0666e9ed385 diff --git a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 deleted file mode 100644 index 09c3bafa19e6a..0000000000000 --- a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ec80bedb86483002a78de7859f524621 diff --git a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 b/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 deleted file mode 100644 index 979950d9080f4..0000000000000 --- a/deps/checksums/SparseArrays-6d072a81fca5f4394f88a012f4ce914c70769303.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -563e706de71b7147f44fa28ec1a729e08172bdf27bdf741f0702a451717f561562b2211d74ca2d01218fd9da05268436dc795fe7aeef7d34b46bbd966c23f71e diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index 9230b5c3e6d94..78026afda0489 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = 6d072a81fca5f4394f88a012f4ce914c70769303 +SPARSEARRAYS_SHA1 = 30201abcb41e558a4b5cc23b11dc5676c5655c0b SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From fae0d0ad3e5d9804533435fe81f4eaac819895af Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 22 Jul 2025 08:19:39 -0400 Subject: [PATCH 555/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?JuliaSyntaxHighlighting=20stdlib=20from=20f803fb0=20to=20b666d3?= =?UTF-8?q?c=20(#59037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/JuliaSyntaxHighlighting.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 new file mode 100644 index 0000000000000..4a586ce45dbcc --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 @@ -0,0 +1 @@ +778d62517cab8b4a95920337631f9439 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 new file mode 100644 index 0000000000000..5a88e30a10a3f --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 @@ -0,0 +1 @@ +95db08cd6775920271e347bae3ee4a68ef532ec6dceb834a63bc8f918b785c042c2bed9babf9ca76ff610ee2782130a69b4ecf158e4beae6361115cdef57dc51 diff --git a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 deleted file mode 100644 index 44f1d6505372c..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -b386bcdbabb0b32556d5d84034a6554a diff --git a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 deleted file mode 100644 index 07498d7ad4964..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-f803fb01dae69b116426d163c53b2a00570e1274.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -f0d08ae9965ab0b173bd24394d4e8cd852ca3fc158122aabcb9019ebeeaeaf4fafa40abc16ad103e419b43c71bb1fa4337a9b4ca1f24b456bed5d93fc6921959 diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index 9fd6979c624b2..3a94bcf8c92cd 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = f803fb01dae69b116426d163c53b2a00570e1274 +JULIASYNTAXHIGHLIGHTING_SHA1 = b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/julialang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/julialang/JuliaSyntaxHighlighting.jl/tarball/$1 From 59a7bb3184feee32227aa15d5c39aa59270ad8e5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 22 Jul 2025 16:08:27 -0400 Subject: [PATCH 556/662] stored method interference graph (#58948) Store full method interference relationship graph in interferences field of Method to avoid expensive morespecific calls during dispatch. This provides significant performance improvements: - Replace method comparisons with precomputed interference lookup. - Optimize ml_matches minmax computation using interference lookups. - Optimize sort_mlmatches for large return sets by iterating over interferences instead of all matching methods. - Add method_morespecific_via_interferences in both C and Julia. This representation may exclude some edges that are implied by transitivity since sort_mlmatches will ensure the correct result by following strong edges. Ambiguous edges are guaranteed to be checkable without recursion. Also fix a variety of bugs along the way: - Builtins signature would cause them to try to discard all other methods during `sort_mlmatches`. - Some ambiguities were over-estimated, which now are improved upon. - Setting lim==-1 now gives the same limited list of methods as lim>0, since that is actually faster now than attempting to give the unsorted list. This provides a better fix to #53814 than #57837 and fixes #58766. - Reverts recent METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC attempt (though not the whole commit), since I found a significant problem with any usage of that bit during testing: it only tracks methods that intersect with a target, but new methods do not necessarily intersect with any existing target. This provides a decent performance improvement to `methods` calls, which implies a decent speed up to package loading also (e.g. ModelingToolkit loads in about 4 seconds instead of 5 seconds). --- base/Base.jl | 2 + base/errorshow.jl | 8 +- base/methodshow.jl | 8 +- base/runtime_internals.jl | 10 +- base/staticdata.jl | 215 +++++++--- src/gf.c | 856 ++++++++++++++++++-------------------- src/jltypes.c | 14 +- src/julia.h | 1 + src/julia_internal.h | 3 +- src/method.c | 1 + src/staticdata.c | 2 - src/staticdata_utils.c | 4 + stdlib/Test/src/Test.jl | 1 - test/ambiguous.jl | 40 +- test/core.jl | 2 +- 15 files changed, 638 insertions(+), 529 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index c0737805de99e..9d510b5c5d47c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -267,7 +267,9 @@ include("uuid.jl") include("pkgid.jl") include("toml_parser.jl") include("linking.jl") +module StaticData include("staticdata.jl") +end include("loading.jl") # BinaryPlatforms, used by Artifacts. Needs `Sort`. diff --git a/base/errorshow.jl b/base/errorshow.jl index d876f71520df6..7a25c561aa9e9 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -465,6 +465,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) line_score = Int[] # These functions are special cased to only show if first argument is matched. special = f === convert || f === getindex || f === setindex! + f isa Core.Builtin && return # `methods` isn't very useful for a builtin funcs = Tuple{Any,Vector{Any}}[(f, arg_types_param)] # An incorrect call method produces a MethodError for convert. @@ -472,7 +473,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) # pool MethodErrors for these two functions. if f === convert && !isempty(arg_types_param) at1 = arg_types_param[1] - if isType(at1) && !has_free_typevars(at1) + if isType(at1) && !has_free_typevars(at1) && at1.parameters[1] isa Type push!(funcs, (at1.parameters[1], arg_types_param[2:end])) end end @@ -494,8 +495,8 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) end sig0 = sig0::DataType s1 = sig0.parameters[1] - if sig0 === Tuple || !isa(func, rewrap_unionall(s1, method.sig)) - # function itself doesn't match or is a builtin + if !isa(func, rewrap_unionall(s1, method.sig)) + # function itself doesn't match continue else print(iob, " ") @@ -640,6 +641,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) println(io) # extra newline for spacing to stacktrace end end + nothing end # In case the line numbers in the source code have changed since the code was compiled, diff --git a/base/methodshow.jl b/base/methodshow.jl index 46b915a1dd09e..1470303a01bbc 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -78,7 +78,7 @@ end # NOTE: second argument is deprecated and is no longer used function kwarg_decl(m::Method, kwtype = nothing) - if m.sig !== Tuple # OpaqueClosure or Builtin + if !(m.sig === Tuple || m.sig <: Tuple{Core.Builtin, Vararg}) # OpaqueClosure or Builtin kwtype = typeof(Core.kwcall) sig = rewrap_unionall(Tuple{kwtype, NamedTuple, (unwrap_unionall(m.sig)::DataType).parameters...}, m.sig) kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, get_world_counter()) @@ -219,8 +219,7 @@ function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_width = 1, print_signature_only::Bool = get(io, :print_method_signature_only, false)::Bool) tv, decls, file, line = arg_decl_parts(m) - sig = unwrap_unionall(m.sig) - if sig === Tuple + if m.sig <: Tuple{Core.Builtin, Vararg} # Builtin print(io, m.name, "(...)") file = "none" @@ -425,8 +424,7 @@ end function show(io::IO, ::MIME"text/html", m::Method) tv, decls, file, line = arg_decl_parts(m, true) sig = unwrap_unionall(m.sig) - if sig === Tuple - # Builtin + if sig <: Tuple{Core.Builtin, Vararg} print(io, m.name, "(...) in ", parentmodule(m)) return end diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index bb5c09d80db43..f75824577ebf4 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1515,15 +1515,7 @@ end function matches_to_methods(ms::Array{Any,1}, tn::Core.TypeName, mod) # Lack of specialization => a comprehension triggers too many invalidations via _collect, so collect the methods manually ms = Method[(ms[i]::Core.MethodMatch).method for i in 1:length(ms)] - # Remove shadowed methods with identical type signatures - prev = nothing - filter!(ms) do m - l = prev - repeated = (l isa Method && m.sig == l.sig) - prev = m - return !repeated - end - # Remove methods not part of module (after removing shadowed methods) + # Remove methods not part of module mod === nothing || filter!(ms) do m return parentmodule(m) ∈ mod end diff --git a/base/staticdata.jl b/base/staticdata.jl index ee63f901f9bb8..c6be79c1f6a15 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -1,9 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -module StaticData - using .Core: CodeInstance, MethodInstance -using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi +using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi, morespecific const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1 const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) @@ -158,6 +156,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi elseif maxworld == Base.get_require_world() # if no new worlds were allocated since serializing the base module, then no new validation is worth doing right now either else + matches = [] j = 1 while j ≤ length(callees) local min_valid2::UInt, max_valid2::UInt @@ -168,14 +167,14 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end if edge isa MethodInstance sig = edge.specTypes - min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world, true) + min_valid2, max_valid2 = verify_call(sig, callees, j, 1, world, true, matches) j += 1 elseif edge isa Int sig = callees[j+1] # Handle negative counts (fully_covers=false) nmatches = abs(edge) fully_covers = edge > 0 - min_valid2, max_valid2, matches = verify_call(sig, callees, j+2, nmatches, world, fully_covers) + min_valid2, max_valid2 = verify_call(sig, callees, j+2, nmatches, world, fully_covers, matches) j += 2 + nmatches edge = sig elseif edge isa Core.Binding @@ -192,7 +191,6 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi min_valid2 = 1 max_valid2 = 0 end - matches = nothing else callee = callees[j+1] if callee isa Core.MethodTable # skip the legacy edge (missing backedge) @@ -207,7 +205,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi else meth = callee::Method end - min_valid2, max_valid2, matches = verify_invokesig(edge, meth, world) + min_valid2, max_valid2 = verify_invokesig(edge, meth, world, matches) j += 2 end if minworld < min_valid2 @@ -218,7 +216,7 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi end invalidations = _jl_debug_method_invalidation[] if max_valid2 ≠ typemax(UInt) && invalidations !== nothing - push!(invalidations, edge, "insert_backedges_callee", codeinst, matches) + push!(invalidations, edge, "insert_backedges_callee", codeinst, copy(matches)) end if max_valid2 == 0 && invalidations === nothing break @@ -282,44 +280,172 @@ function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visi return 0, minworld, maxworld end -function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool) +function get_method_from_edge(@nospecialize t) + if t isa Method + return t + else + if t isa CodeInstance + t = get_ci_mi(t)::MethodInstance + else + t = t::MethodInstance + end + return t.def::Method + end +end + +# Check if method2 is in method1's interferences set +# Returns true if method2 is found (meaning !morespecific(method1, method2)) +function method_in_interferences(method2::Method, method1::Method) + interferences = method1.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break + interference_method = interferences[k]::Method + if interference_method === method2 + return true + end + end + return false +end + +# Check if method1 is more specific than method2 via the interference graph +function method_morespecific_via_interferences(method1::Method, method2::Method) + if method1 === method2 + return false + end + ms = method_in_interferences_recursive(method1, method2, IdSet{Method}()) + # slow check: @assert ms === morespecific(method1, method2) || typeintersect(method1.sig, method2.sig) === Union{} || typeintersect(method2.sig, method1.sig) === Union{} + return ms +end + +# Returns true if method1 is in method2's interferences (meaning !morespecific(method2, method1)) +function method_in_interferences_recursive(method1::Method, method2::Method, visited::IdSet{Method}) + if method_in_interferences(method2, method1) + return false + end + if method_in_interferences(method1, method2) + return true + end + + # Recursively check through interference graph + method2 in visited && return false + push!(visited, method2) + interferences = method2.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break + method3 = interferences[k]::Method + if method_in_interferences(method2, method3) + continue # only follow edges to morespecific methods in search of the morespecific target (skip ambiguities) + end + if method_in_interferences_recursive(method1, method3, visited) + return true # found method1 in the interference graph + end + end + + return false +end + +function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool, matches::Vector{Any}) # verify that these edges intersect with the same methods as before mi = nothing - if n == 1 + expected_deleted = false + for j = 1:n + t = expecteds[i+j-1] + meth = get_method_from_edge(t) + if iszero(meth.dispatch_status & METHOD_SIG_LATEST_WHICH) + expected_deleted = true + break + end + end + if expected_deleted + if _jl_debug_method_invalidation[] === nothing && world == get_world_counter() + return UInt(1), UInt(0) + end + elseif n == 1 # first, fast-path a check if the expected method simply dominates its sig anyways # so the result of ml_matches is already simply known - let t = expecteds[i], meth, minworld, maxworld, result - if t isa Method - meth = t - else + let t = expecteds[i], meth, minworld, maxworld + meth = get_method_from_edge(t) + if !(t isa Method) if t isa CodeInstance mi = get_ci_mi(t)::MethodInstance else mi = t::MethodInstance end - meth = mi.def::Method - # Fast path is legal when fully_covers=true OR when METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC is unset - if (fully_covers || iszero(meth.dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC)) && - !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + # Fast path is legal when fully_covers=true + if fully_covers && !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) minworld = meth.primary_world @assert minworld ≤ world maxworld = typemax(UInt) - result = Any[] # result is unused - return minworld, maxworld, result + return minworld, maxworld end end - # Fast path is legal when fully_covers=true OR when METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC is unset - if fully_covers || iszero(meth.dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC) - if !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) - minworld = meth.primary_world - @assert minworld ≤ world - maxworld = typemax(UInt) - result = Any[] # result is unused - return minworld, maxworld, result + # Fast path is legal when fully_covers=true + if fully_covers && !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + return minworld, maxworld + end + end + elseif n > 1 + # Try the interference set fast path: check if all interference sets are covered by expecteds + interference_fast_path_success = fully_covers + # If it didn't fail yet, then check that all interference methods are either expected, or not applicable. + if interference_fast_path_success + local interference_minworld::UInt = 1 + for j = 1:n + meth = get_method_from_edge(expecteds[i+j-1]) + if interference_minworld < meth.primary_world + interference_minworld = meth.primary_world + end + interferences = meth.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break # no more entries + interference_method = interferences[k]::Method + if iszero(interference_method.dispatch_status & METHOD_SIG_LATEST_WHICH) + # detected a deleted interference_method, so need the full lookup to compute minworld + interference_fast_path_success = false + break + end + world < interference_method.primary_world && break # this and later entries are for a future world + local found_in_expecteds = false + for j = 1:n + if interference_method === get_method_from_edge(expecteds[i+j-1]) + found_in_expecteds = true + break + end + end + if !found_in_expecteds + ti = typeintersect(sig, interference_method.sig) + if !(ti === Union{}) + # try looking for a different expected method that fully covers this interference_method anyways over their intersection + for j = 1:n + meth2 = get_method_from_edge(expecteds[i+j-1]) + if method_morespecific_via_interferences(meth2, interference_method) && ti <: meth2.sig + found_in_expecteds = true + break + end + end + if !found_in_expecteds + meth2 = get_method_from_edge(expecteds[i]) + interference_fast_path_success = false + break + end + end + end + end + if !interference_fast_path_success + break end end + if interference_fast_path_success + # All interference sets are covered by expecteds, can return success + @assert interference_minworld ≤ world + maxworld = typemax(UInt) + return interference_minworld, maxworld + end end - end + end # next, compare the current result of ml_matches to the old result lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n minworld = Ref{UInt}(1) @@ -327,6 +453,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n has_ambig = Ref{Int32}(0) result = _methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) if result === nothing + empty!(matches) maxworld[] = 0 else # setdiff!(result, expected) @@ -339,17 +466,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n local found = false for j = 1:n t = expecteds[i+j-1] - if t isa Method - meth = t - else - if t isa CodeInstance - t = get_ci_mi(t)::MethodInstance - else - t = t::MethodInstance - end - meth = t.def::Method - end - if match.method == meth + if match.method == get_method_from_edge(t) found = true break end @@ -368,12 +485,13 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n end if maxworld[] ≠ typemax(UInt) && _jl_debug_method_invalidation[] !== nothing resize!(result, ins) + copy!(matches, result) end end if maxworld[] == typemax(UInt) && mi isa MethodInstance ccall(:jl_promote_mi_to_current, Cvoid, (Any, UInt, UInt), mi, minworld[], world) end - return minworld[], maxworld[], result + return minworld[], maxworld[] end # fast-path dispatch_status bit definitions (false indicates unknown) @@ -381,13 +499,11 @@ end const METHOD_SIG_LATEST_WHICH = 0x1 # true indicates this method would be returned as the only result from `methods` when calling `method.sig` in the current latest world const METHOD_SIG_LATEST_ONLY = 0x2 -# true indicates there exists some other method that is not more specific than this one in the current latest world (which might be more fully covering) -const METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC = 0x8 -function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt) +function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt, matches::Vector{Any}) @assert invokesig isa Type local minworld::UInt, maxworld::UInt - matched = nothing + empty!(matches) if invokesig === expected.sig && !iszero(expected.dispatch_status & METHOD_SIG_LATEST_WHICH) # the invoke match is `expected` for `expected->sig`, unless `expected` is replaced minworld = expected.primary_world @@ -404,14 +520,13 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn if matched === nothing maxworld = 0 else - matched = Any[matched.method] - if matched[] !== expected + matched = matched.method + push!(matches, matched) + if matched !== expected maxworld = 0 end end end end - return minworld, maxworld, matched + return minworld, maxworld end - -end # module StaticData diff --git a/src/gf.c b/src/gf.c index 63350928bbaf8..d454110313881 100644 --- a/src/gf.c +++ b/src/gf.c @@ -302,27 +302,27 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_value_t *type, size_t world) jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED { - jl_method_t *m = jl_new_method_uninit(jl_core_module); + jl_value_t *params[2]; + params[0] = dt->name->wrapper; + params[1] = jl_tparam0(jl_anytuple_type); + jl_datatype_t *tuptyp = (jl_datatype_t*)jl_apply_tuple_type_v(params, 2); + + jl_typemap_entry_t *newentry = NULL; + jl_method_t *m = NULL; + JL_GC_PUSH3(&m, &newentry, &tuptyp); + + m = jl_new_method_uninit(jl_core_module); m->name = sname; m->module = jl_core_module; m->isva = 1; m->nargs = 2; jl_atomic_store_relaxed(&m->primary_world, 1); jl_atomic_store_relaxed(&m->dispatch_status, METHOD_SIG_LATEST_ONLY | METHOD_SIG_LATEST_WHICH); - m->sig = (jl_value_t*)jl_anytuple_type; + m->sig = (jl_value_t*)tuptyp; m->slot_syms = jl_an_empty_string; m->nospecialize = 0; m->nospecialize = ~m->nospecialize; - jl_typemap_entry_t *newentry = NULL; - jl_datatype_t *tuptyp = NULL; - JL_GC_PUSH3(&m, &newentry, &tuptyp); - - jl_value_t *params[2]; - params[0] = dt->name->wrapper; - params[1] = jl_tparam0(jl_anytuple_type); - tuptyp = (jl_datatype_t*)jl_apply_tuple_type_v(params, 2); - jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)tuptyp, jl_emptysvec); jl_atomic_store_relaxed(&m->unspecialized, mi); jl_gc_wb(m, mi); @@ -1881,11 +1881,12 @@ struct matches_env { static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_intersection_env *closure0) { struct matches_env *closure = container_of(closure0, struct matches_env, match); + jl_method_t *oldmethod = oldentry->func.method; assert(oldentry != closure->newentry && "entry already added"); assert(jl_atomic_load_relaxed(&oldentry->min_world) <= jl_atomic_load_relaxed(&closure->newentry->min_world) && "old method cannot be newer than new method"); - assert(jl_atomic_load_relaxed(&oldentry->max_world) != jl_atomic_load_relaxed(&closure->newentry->min_world) && "method cannot be added at the same time as method deleted"); + //assert(jl_atomic_load_relaxed(&oldentry->max_world) != jl_atomic_load_relaxed(&closure->newentry->min_world) && "method cannot be added at the same time as method deleted"); + assert((jl_atomic_load_relaxed(&oldentry->max_world) == ~(size_t)0)); // don't need to consider other similar methods if this oldentry will always fully intersect with them and dominates all of them - jl_method_t *oldmethod = oldentry->func.method; if (closure->match.issubty // e.g. jl_subtype(closure->newentry.sig, oldentry->sig) && jl_subtype(oldmethod->sig, (jl_value_t*)closure->newentry->sig)) { // e.g. jl_type_equal(closure->newentry->sig, oldentry->sig) if (closure->replaced == NULL || jl_atomic_load_relaxed(&closure->replaced->min_world) < jl_atomic_load_relaxed(&oldentry->min_world)) @@ -1893,7 +1894,10 @@ static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_in } if (closure->shadowed == NULL) closure->shadowed = (jl_value_t*)jl_alloc_vec_any(0); - if (closure->match.issubty) { // this should be rarely true (in fact, get_intersect_visitor should be rarely true), but might as well skip the rest of the scan fast anyways since we can + // This should be rarely true (in fact, get_intersect_visitor should be + // rarely true), but might as well skip the rest of the scan fast anyways + // since we can. + if (closure->match.issubty) { int only = jl_atomic_load_relaxed(&oldmethod->dispatch_status) & METHOD_SIG_LATEST_ONLY; if (only) { size_t len = jl_array_nrows(closure->shadowed); @@ -2397,7 +2401,6 @@ struct invalidate_mt_env { jl_typemap_entry_t *newentry; jl_array_t *shadowed; size_t max_world; - int invalidated; }; static int invalidate_mt_cache(jl_typemap_entry_t *oldentry, void *closure0) { @@ -2439,7 +2442,6 @@ static int invalidate_mt_cache(jl_typemap_entry_t *oldentry, void *closure0) JL_GC_POP(); } jl_atomic_store_relaxed(&oldentry->max_world, env->max_world); - env->invalidated = 1; } } return 1; @@ -2584,7 +2586,7 @@ JL_DLLEXPORT void jl_method_table_disable(jl_method_t *method) jl_error("Method changes have been disabled via a call to disable_new_worlds."); int enabled = jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0; if (enabled) { - // Narrow the world age on the method to make it uncallable + // Narrow the world age on the method to make it uncallable size_t world = jl_atomic_load_relaxed(&jl_world_counter); assert(method == methodentry->func.method); jl_atomic_store_relaxed(&method->dispatch_status, 0); @@ -2592,7 +2594,7 @@ JL_DLLEXPORT void jl_method_table_disable(jl_method_t *method) jl_atomic_store_relaxed(&methodentry->max_world, world); jl_method_table_invalidate(method, world); jl_atomic_store_release(&jl_world_counter, world + 1); - } + } JL_UNLOCK(&world_counter_lock); if (!enabled) jl_errorf("Method of %s already disabled", jl_symbol_name(method->name)); @@ -2621,6 +2623,132 @@ jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, return newentry; } +static int has_key(jl_genericmemory_t *keys, jl_value_t *key) +{ + for (size_t l = keys->length, i = 0; i < l; i++) { + jl_value_t *k = jl_genericmemory_ptr_ref(keys, i); + if (k == NULL) + return 0; + if (jl_genericmemory_ptr_ref(keys, i) == key) + return 1; + } + return 0; +} + +// Check if m2 is in m1's interferences set, which means !morespecific(m1, m2) +static int method_in_interferences(jl_method_t *m2, jl_method_t *m1) +{ + return has_key(jl_atomic_load_relaxed(&m1->interferences), (jl_value_t*)m2); +} + +// Find the index of a method in the method match array +static int find_method_in_matches(jl_array_t *t, jl_method_t *method) +{ + size_t len = jl_array_nrows(t); + for (size_t i = 0; i < len; i++) { + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, i); + if (matc->method == method) + return i; + } + return -1; +} + +// Recursively check if any method in interferences covers the given type signature +static int check_interferences_covers(jl_method_t *m, jl_value_t *ti, jl_array_t *t, arraylist_t *visited, arraylist_t *recursion_stack) +{ + // Check if we're already visiting this method (cycle detection and memoization) + for (size_t i = 0; i < recursion_stack->len; i++) + if (recursion_stack->items[i] == (void*)m) + return 0; + + // Add this method to the recursion stack + arraylist_push(recursion_stack, (void*)m); + + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (m2 == NULL) + continue; + int idx = find_method_in_matches(t, m2); + if (idx < 0) + continue; + if (method_in_interferences(m, m2)) + continue; // ambiguous + assert(visited->items[idx] != (void*)0); + if (visited->items[idx] != (void*)1) + continue; // part of the same SCC cycle (handled by ambiguity later) + if (jl_subtype(ti, m2->sig)) + return 1; + // Recursively check m2's interferences since m2 is more specific + if (check_interferences_covers(m2, ti, t, visited, recursion_stack)) + return 1; + } + return 0; +} + +static int check_fully_ambiguous(jl_method_t *m, jl_value_t *ti, jl_array_t *t, int include_ambiguous, int *has_ambiguity) +{ + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (m2 == NULL) + continue; + int idx = find_method_in_matches(t, m2); + if (idx < 0) + continue; + if (!method_in_interferences(m, m2)) + continue; + *has_ambiguity = 1; + if (!include_ambiguous && jl_subtype(ti, m2->sig)) + return 1; + } + return 0; +} + +// Recursively check if target_method is in the interferences of (morespecific than) start_method, but not the reverse +static int method_in_interferences_recursive(jl_method_t *target_method, jl_method_t *start_method, arraylist_t *seen) +{ + // Check direct interferences first + if (method_in_interferences(start_method, target_method)) + return 0; + if (method_in_interferences(target_method, start_method)) + return 1; + + // Check if we're already visiting this method (cycle prevention and memoization) + for (size_t i = 0; i < seen->len; i++) { + if (seen->items[i] == (void*)start_method) + return 0; + } + arraylist_push(seen, (void*)start_method); + + // Recursively check interferences + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&start_method->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *interference_method = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (interference_method == NULL) + continue; + if (method_in_interferences(start_method, interference_method)) + continue; // only follow edges to morespecific methods in search of morespecific target (skip ambiguities) + if (method_in_interferences_recursive(target_method, interference_method, seen)) + return 1; + } + + return 0; +} + +static int method_morespecific_via_interferences(jl_method_t *target_method, jl_method_t *start_method) +{ + if (target_method == start_method) + return 0; + arraylist_t seen; + arraylist_new(&seen, 0); + int result = method_in_interferences_recursive(target_method, start_method, &seen); + arraylist_free(&seen); + //assert(result == jl_method_morespecific(target_method, start_method) || jl_has_empty_intersection(target_method->sig, start_method->sig) || jl_has_empty_intersection(start_method->sig, target_method->sig)); + return result; +} + + void jl_method_table_activate(jl_typemap_entry_t *newentry) { JL_TIMING(ADD_METHOD, ADD_METHOD); @@ -2644,48 +2772,100 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) jl_value_t *loctag = NULL; // debug info for invalidation jl_value_t *isect = NULL; jl_value_t *isect2 = NULL; - jl_value_t *isect3 = NULL; - JL_GC_PUSH6(&oldvalue, &oldmi, &loctag, &isect, &isect2, &isect3); + jl_genericmemory_t *interferences = NULL; + JL_GC_PUSH6(&oldvalue, &oldmi, &loctag, &isect, &isect2, &interferences); jl_typemap_entry_t *replaced = NULL; - // then check what entries we replaced + // Check what entries this intersects with in the prior world. oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, max_world); + jl_method_t *const *d; + size_t j, n; + if (oldvalue == NULL) { + d = NULL; + n = 0; + } + else { + assert(jl_is_array(oldvalue)); + d = (jl_method_t**)jl_array_ptr_data(oldvalue); + n = jl_array_nrows(oldvalue); + oldmi = jl_alloc_vec_any(0); + } + // These get updated from their state stored in the caches files, since content in cache files gets added "all at once". int invalidated = 0; int dispatch_bits = METHOD_SIG_LATEST_WHICH; // Always set LATEST_WHICH // Check precompiled dispatch status bits int precompiled_status = jl_atomic_load_relaxed(&method->dispatch_status); if (!(precompiled_status & METHOD_SIG_PRECOMPILE_MANY)) + // This will store if this method will be currently the only result that would returned from `ml_matches` given `sig`. dispatch_bits |= METHOD_SIG_LATEST_ONLY; // Tentatively set, will be cleared if not applicable - if (precompiled_status & METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC) - dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; - if (replaced) { - oldvalue = (jl_value_t*)replaced; - jl_method_t *m = replaced->func.method; - invalidated = 1; - method_overwrite(newentry, m); - // This is an optimized version of below, given we know the type-intersection is exact - jl_method_table_invalidate(m, max_world); - int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); - // Clear METHOD_SIG_LATEST_ONLY and METHOD_SIG_LATEST_WHICH bits, only keeping NOTMORESPECIFIC - jl_atomic_store_relaxed(&m->dispatch_status, m_dispatch & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC); - // Edge case: don't set dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC unconditionally since `m` is not an visible method for invalidations - dispatch_bits |= (m_dispatch & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC); - if (!(m_dispatch & METHOD_SIG_LATEST_ONLY)) - dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; - } - else { - jl_method_t *const *d; - size_t j, n; - if (oldvalue == NULL) { + // Holds the set of all intersecting methods not more specific than this one. + // Note: this set may be incomplete (may exclude methods whose intersection + // is covered by another method that is morespecific than both, causing them + // to have no relevant type intersection for sorting). + interferences = (jl_genericmemory_t*)jl_atomic_load_relaxed(&method->interferences); + if (oldvalue) { + assert(n > 0); + if (replaced) { + oldvalue = (jl_value_t*)replaced; + jl_method_t *m = replaced->func.method; + invalidated = 1; + method_overwrite(newentry, m); + // This is an optimized version of below, given we know the type-intersection is exact + jl_method_table_invalidate(m, max_world); + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + // Clear METHOD_SIG_LATEST_ONLY and METHOD_SIG_LATEST_WHICH bits + jl_atomic_store_relaxed(&m->dispatch_status, 0); + if (!(m_dispatch & METHOD_SIG_LATEST_ONLY)) + dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; + // Take over the interference list from the replaced method + jl_genericmemory_t *m_interferences = jl_atomic_load_relaxed(&m->interferences); + if (interferences->length == 0) { + interferences = jl_genericmemory_copy(m_interferences); + } + else { + for (size_t i = 0; i < m_interferences->length; i++) { + jl_value_t *k = jl_genericmemory_ptr_ref(m_interferences, i); + if (k && !has_key(interferences, (jl_value_t*)k)) { + ssize_t idx; + interferences = jl_idset_put_key(interferences, (jl_value_t*)k, &idx); + } + } + } + ssize_t idx; + m_interferences = jl_idset_put_key(m_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m->interferences, m_interferences); + jl_gc_wb(m, m_interferences); + for (j = 0; j < n; j++) { + jl_method_t *m2 = d[j]; + if (m2 && method_in_interferences(m, m2)) { + jl_genericmemory_t *m2_interferences = jl_atomic_load_relaxed(&m2->interferences); + ssize_t idx; + m2_interferences = jl_idset_put_key(m2_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m2->interferences, m2_interferences); + jl_gc_wb(m2, m2_interferences); + } + } + loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot + _Atomic(jl_method_instance_t*) *data; + size_t l; + if (jl_is_svec(loctag)) { + data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(loctag); + l = jl_svec_len(loctag); + } + else { + data = (_Atomic(jl_method_instance_t*)*) &loctag; + l = 1; + } + for (size_t i = 0; i < l; i++) { + jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); + if ((jl_value_t*)mi == jl_nothing) + continue; + jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); + } d = NULL; n = 0; } else { - assert(jl_is_array(oldvalue)); - d = (jl_method_t**)jl_array_ptr_data(oldvalue); - n = jl_array_nrows(oldvalue); - - oldmi = jl_alloc_vec_any(0); char *morespec = (char*)alloca(n); // Compute all morespec values upfront for (j = 0; j < n; j++) @@ -2699,16 +2879,27 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) if (morespec[j] || ambig) { // !morespecific(new, old) dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; - m_dispatch |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; + // Add the old method to this interference set + ssize_t idx; + if (!has_key(interferences, (jl_value_t*)m)) + interferences = jl_idset_put_key(interferences, (jl_value_t*)m, &idx); } if (!morespec[j]) { // !morespecific(old, new) - dispatch_bits |= METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC; m_dispatch &= ~METHOD_SIG_LATEST_ONLY; + // Add the new method to its interference set + jl_genericmemory_t *m_interferences = jl_atomic_load_relaxed(&m->interferences); + ssize_t idx; + m_interferences = jl_idset_put_key(m_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m->interferences, m_interferences); + jl_gc_wb(m, m_interferences); } + // Add methods that intersect but are not more specific to interference list jl_atomic_store_relaxed(&m->dispatch_status, m_dispatch); if (morespec[j]) continue; + + // Now examine if this caused any invalidations. loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot _Atomic(jl_method_instance_t*) *data; size_t l; @@ -2724,12 +2915,11 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); if ((jl_value_t*)mi == jl_nothing) continue; - isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes); - if (jl_type_intersection2(type, isect3, &isect, &isect2)) { + if (jl_type_intersection2(type, mi->specTypes, &isect, &isect2)) { // Replacing a method--see if this really was the selected method previously - // over the intersection (not ambiguous) and the new method will be selected now (morespec_is). + // over the intersection (not ambiguous) and the new method will be selected now (morespec). // TODO: this only checks pair-wise for ambiguities, but the ambiguities could arise from the interaction of multiple methods - // and thus might miss a case where we introduce an ambiguity between two existing methods + // and thus might miss a case where we introduce an ambiguity between`.u two existing methods // We could instead work to sort this into 3 groups `morespecific .. ambiguous .. lesspecific`, with `type` in ambiguous, // such that everything in `morespecific` dominates everything in `ambiguous`, and everything in `ambiguous` dominates everything in `lessspecific` // And then compute where each isect falls, and whether it changed group--necessitating invalidation--or not. @@ -2738,9 +2928,10 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); // but ignore invoke-type edges int invalidatedmi = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); - if (replaced_dispatch) + if (replaced_dispatch) { jl_atomic_store_relaxed(&mi->dispatch_status, 0); - jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); + jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); + } if (_jl_debug_method_invalidation && invalidatedmi) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); loctag = jl_cstr_to_string("jl_method_table_insert"); @@ -2748,49 +2939,64 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) } invalidated |= invalidatedmi; } + // TODO: do we have any interesting cases left where isect3 is useful + //jl_value_t *isect3 = NULL; + //jl_value_t *isect4 = NULL; + //jl_value_t *isect5 = NULL; + //JL_GC_PUSH3(&isec3, &isect4, &isect5); + //isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes); + //jl_type_intersection2(type, isect3, &isect4, &isect5); + //if (!jl_types_equal(isect, isect4) && (!isect2 || !jl_types_equal(isect2, isect4)) && + // (!isect5 || (!jl_types_equal(isect, isect5) && (!isect2 || !jl_types_equal(isect2, isect5))))) { + // jl_(type); + // jl_(mi->specTypes); + // jl_(m->sig); + //} + //JL_GC_POP(); + isect = NULL; + isect2 = NULL; } } } + } - jl_methcache_t *mc = jl_method_table->cache; - JL_LOCK(&mc->writelock); - struct _typename_invalidate_backedge typename_env = {type, &isect, &isect2, d, n, max_world, invalidated}; - if (!jl_foreach_top_typename_for(_typename_invalidate_backedges, type, 1, &typename_env)) { - // if the new method cannot be split into exact backedges, scan the whole table for anything that might be affected - jl_genericmemory_t *allbackedges = jl_method_table->backedges; - for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { - jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); - jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i+1); - if (tn && tn != jl_nothing && backedges) - _typename_invalidate_backedges((jl_typename_t*)tn, 0, &typename_env); - } - } - invalidated |= typename_env.invalidated; - if (oldmi && jl_array_nrows(oldmi)) { - // search mc->cache and leafcache and drop anything that might overlap with the new method - // this is very cheap, so we don't mind being fairly conservative at over-approximating this - struct invalidate_mt_env mt_cache_env; - mt_cache_env.max_world = max_world; - mt_cache_env.shadowed = oldmi; - mt_cache_env.newentry = newentry; - mt_cache_env.invalidated = 0; - - jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), invalidate_mt_cache, (void*)&mt_cache_env); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); - size_t i, l = leafcache->length; - for (i = 1; i < l; i += 2) { - jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); - if (entry) { - while (entry != jl_nothing) { - invalidate_mt_cache((jl_typemap_entry_t*)entry, (void*)&mt_cache_env); - entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); - } + jl_methcache_t *mc = jl_method_table->cache; + JL_LOCK(&mc->writelock); + struct _typename_invalidate_backedge typename_env = {type, &isect, &isect2, d, n, max_world, invalidated}; + if (!jl_foreach_top_typename_for(_typename_invalidate_backedges, type, 1, &typename_env)) { + // if the new method cannot be split into exact backedges, scan the whole table for anything that might be affected + jl_genericmemory_t *allbackedges = jl_method_table->backedges; + for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i+1); + if (tn && tn != jl_nothing && backedges) + _typename_invalidate_backedges((jl_typename_t*)tn, 0, &typename_env); + } + } + invalidated |= typename_env.invalidated; + if (oldmi && jl_array_nrows(oldmi)) { + // drop leafcache and search mc->cache and drop anything that might overlap with the new method + // this is very cheap, so we don't mind being very conservative at over-approximating this + struct invalidate_mt_env mt_cache_env; + mt_cache_env.max_world = max_world; + mt_cache_env.shadowed = oldmi; + mt_cache_env.newentry = newentry; + + jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), invalidate_mt_cache, (void*)&mt_cache_env); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); + size_t i, l = leafcache->length; + for (i = 1; i < l; i += 2) { + jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); + if (entry) { + while (entry != jl_nothing) { + jl_atomic_store_relaxed(&((jl_typemap_entry_t*)entry)->max_world, max_world); + entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); } } - invalidated |= mt_cache_env.invalidated; } - JL_UNLOCK(&mc->writelock); + jl_atomic_store_relaxed(&mc->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); } + JL_UNLOCK(&mc->writelock); if (invalidated && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)method); loctag = jl_cstr_to_string("jl_method_table_insert"); @@ -2798,6 +3004,8 @@ void jl_method_table_activate(jl_typemap_entry_t *newentry) } jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); jl_atomic_store_relaxed(&method->dispatch_status, dispatch_bits); // TODO: this should be sequenced fully after the world counter store + jl_atomic_store_release(&method->interferences, interferences); + jl_gc_wb(method, interferences); JL_GC_POP(); } @@ -4242,16 +4450,9 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio return 1; } -static int ml_mtable_visitor(jl_methtable_t *mt, void *closure0) -{ - struct typemap_intersection_env* env = (struct typemap_intersection_env*)closure0; - return jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), 0, env); -} - // Visit the candidate methods, starting from t[idx], to determine a possible valid sort ordering, // where every morespecific method appears before any method which it has a common -// intersection with but is not partly ambiguous with (ambiguity is transitive, particularly -// if lim==-1, although morespecific is not transitive). +// intersection with but is not partly ambiguous with (ambiguity is not transitive, since morespecific is not transitive). // Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable // Inputs: // * `t`: the array of vertexes (method matches) @@ -4259,260 +4460,118 @@ static int ml_mtable_visitor(jl_methtable_t *mt, void *closure0) // * `visited`: the state of the algorithm for each vertex in `t`: either 1 if we visited it already or 1+depth if we are visiting it now // * `stack`: the state of the algorithm for the current vertex (up to length equal to `t`): the list of all vertexes currently in the depth-first path or in the current SCC // * `result`: the output of the algorithm, a sorted list of vertexes (up to length `lim`) -// * `allambig`: a list of all vertexes with an ambiguity (up to length equal to `t`), discovered while running the rest of the algorithm +// * `recursion_stack`: an array for temporary use // * `lim`: either -1 for unlimited matches, or the maximum length for `result` before returning failure (return -1). -// If specified as -1, this will return extra matches that would have been elided from the list because they were already covered by an earlier match. -// This gives a sort of maximal set of matching methods (up to the first minmax method). -// If specified as -1, the sorting will also include all "weak" edges (every ambiguous pair) which will create much larger ambiguity cycles, -// resulting in a less accurate sort order and much less accurate `*has_ambiguity` result. // * `include_ambiguous`: whether to filter out fully ambiguous matches from `result` // * `*has_ambiguity`: whether the algorithm does not need to compute if there is an unresolved ambiguity // * `*found_minmax`: whether there is a minmax method already found, so future fully_covers matches should be ignored // Outputs: -// * `*has_ambiguity`: whether the caller should check if there remains an unresolved ambiguity (in `allambig`) +// * `*has_ambiguity`: whether there are any ambiguities that mean the sort order is not exact // Returns: // * -1: too many matches for lim, other outputs are undefined // * 0: the child(ren) have been added to the output // * 1+: the children are part of this SCC (up to this depth) // TODO: convert this function into an iterative call, rather than recursive -static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, arraylist_t *stack, arraylist_t *result, arraylist_t *allambig, int lim, int include_ambiguous, int *has_ambiguity, int *found_minmax) +static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, arraylist_t *stack, arraylist_t *result, arraylist_t *recursion_stack, int lim, int include_ambiguous, int *has_ambiguity, int *found_minmax) { size_t cycle = (size_t)visited->items[idx]; if (cycle != 0) return cycle - 1; // depth remaining - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, idx); - jl_method_t *m = matc->method; - jl_value_t *ti = (jl_value_t*)matc->spec_types; - int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - // first check if this new method is actually already fully covered by an - // existing match and we can just ignore this entry quickly - size_t result_len = 0; - if (subt) { - if (*found_minmax == 2) - visited->items[idx] = (void*)1; - } - else if (lim != -1) { - for (; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; - } - visited->items[idx] = (void*)1; - break; - } - } - } - if ((size_t)visited->items[idx] == 1) - return 0; arraylist_push(stack, (void*)idx); size_t depth = stack->len; - visited->items[idx] = (void*)(1 + depth); - cycle = depth; - int addambig = 0; - int mayexclude = 0; - // First visit all "strong" edges where the child is definitely better. - // This likely won't hit any cycles, but might (because morespecific is not transitive). - // Along the way, record if we hit any ambiguities-we may need to track those later. - for (size_t childidx = 0; childidx < jl_array_nrows(t); childidx++) { - if (childidx == idx) - continue; - int child_cycle = (size_t)visited->items[childidx]; - if (child_cycle == 1) - continue; // already handled - if (child_cycle != 0 && child_cycle - 1 >= cycle) - continue; // already part of this cycle - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - // TODO: we could change this to jl_has_empty_intersection(ti, (jl_value_t*)matc2->spec_types); - // since we only care about sorting of the intersections the user asked us about - if (!subt2 && jl_has_empty_intersection(m2->sig, m->sig)) - continue; - int msp = jl_method_morespecific(m, m2); - int msp2 = !msp && jl_method_morespecific(m2, m); - if (!msp) { - if (subt || !include_ambiguous || (lim != -1 && msp2)) { - if (subt2 || ((lim != -1 || (!include_ambiguous && !msp2)) && jl_subtype((jl_value_t*)ti, m2->sig))) { - // this may be filtered out as fully intersected, if applicable later - mayexclude = 1; - } - } - if (!msp2) { - addambig = 1; // record there is a least one previously-undetected ambiguity that may need to be investigated later (between m and m2) - } - } - if (lim == -1 ? msp : !msp2) // include only strong or also weak edges, depending on whether the result size is limited - continue; - // m2 is (lim!=-1 ? better : not-worse), so attempt to visit it first - // if limited, then we want to visit only better edges, because that results in finding k best matches quickest - // if not limited, then we want to visit all edges, since that results in finding the largest SCC cycles, which requires doing the fewest intersections - child_cycle = sort_mlmatches(t, childidx, visited, stack, result, allambig, lim, include_ambiguous, has_ambiguity, found_minmax); - if (child_cycle == -1) - return -1; - if (child_cycle && child_cycle < cycle) { + { // scope block + visited->items[idx] = (void*)(1 + depth); + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, idx); + jl_method_t *m = matc->method; + jl_value_t *ti = (jl_value_t*)matc->spec_types; + int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); + cycle = depth; + // Iterate over the interferences set to get the morespecific methods + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (m2 == NULL) + continue; + int childidx = find_method_in_matches(t, m2); + if (childidx < 0 || (size_t)childidx == idx) + continue; + int child_cycle = (size_t)visited->items[childidx]; + if (child_cycle == 1) + continue; // already handled + if (child_cycle != 0 && child_cycle - 1 >= cycle) + continue; // already part of this cycle + if (method_in_interferences(m, m2)) + continue; + // m2 is morespecific, so attempt to visit it first + child_cycle = sort_mlmatches(t, childidx, visited, stack, result, recursion_stack, lim, include_ambiguous, has_ambiguity, found_minmax); + if (child_cycle == -1) + return -1; // record the cycle will resolve at depth "cycle" - cycle = child_cycle; - } - if (stack->len == depth) { - // if this child resolved without hitting a cycle, then there is - // some probability that this method is already fully covered now - // (same check as before), and we can delete this vertex now without - // anyone noticing (too much) - if (subt) { - if (*found_minmax == 2) - visited->items[idx] = (void*)1; - } - else if (lim != -1) { - for (; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; - } - visited->items[idx] = (void*)1; - break; - } - } - } - if ((size_t)visited->items[idx] == 1) { - // n.b. cycle might be < depth, if we had a cycle with a child - // idx, but since we are on the top of the stack, nobody - // observed that and so we are content to ignore this - size_t childidx = (size_t)arraylist_pop(stack); - assert(childidx == idx); (void)childidx; - assert(!subt || *found_minmax == 2); - return 0; - } + if (child_cycle && child_cycle < cycle) + cycle = child_cycle; } - } - if (matc->fully_covers == NOT_FULLY_COVERS && addambig) - arraylist_push(allambig, (void*)idx); - if (cycle != depth) - return cycle; - result_len = result->len; - if (stack->len == depth) { - // Found one "best" method to add right now. But we might exclude it if - // we determined earlier that we had that option. - if (mayexclude) { - if (!subt || *found_minmax == 2) + // There is some probability that this method is already fully covered + // now, and we can delete this vertex now without anyone noticing. + if (subt && *found_minmax) { + if (*found_minmax == 2) visited->items[idx] = (void*)1; } + else if (check_interferences_covers(m, ti, t, visited, recursion_stack)) { + visited->items[idx] = (void*)1; + } + else if (check_fully_ambiguous(m, ti, t, include_ambiguous, has_ambiguity)) { + visited->items[idx] = (void*)1; + } + + // If there were no cycles hit either, then we can potentially delete all of its edges too. + if ((size_t)visited->items[idx] == 1 && stack->len == depth) { + // n.b. cycle might be < depth, if we had a cycle with a child + // idx, but since we are on the top of the stack, nobody + // observed that and so we are content to ignore this + size_t childidx = (size_t)arraylist_pop(stack); + assert(childidx == idx); (void)childidx; + return 0; + } + if (cycle != depth) + return cycle; } - else { - // We have a set of ambiguous methods. Record that. - // This is greatly over-approximated for lim==-1 - *has_ambiguity = 1; - // If we followed weak edges above, then this also fully closed the ambiguity cycle - if (lim == -1) - addambig = 0; - // If we're only returning possible matches, now filter out this method - // if its intersection is fully ambiguous in this SCC group. - // This is a repeat of the "first check", now that we have completed the cycle analysis + // If this is in an SCC group, do some additional checks before returning or setting has_ambiguity + if (depth != stack->len) { + int scc_count = 0; for (size_t i = depth - 1; i < stack->len; i++) { size_t childidx = (size_t)stack->items[i]; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_value_t *ti = (jl_value_t*)matc->spec_types; - int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - if ((size_t)visited->items[childidx] == 1) { - assert(subt); - continue; - } - assert(visited->items[childidx] == (void*)(2 + i)); - // if we only followed strong edges before above - // check also if this set has an unresolved ambiguity missing from it - if (lim != -1 && !addambig) { - for (size_t j = 0; j < allambig->len; j++) { - if ((size_t)allambig->items[j] == childidx) { - addambig = 1; - break; - } - } - } - // always remove fully_covers matches after the first minmax ambiguity group is handled - if (subt) { - if (*found_minmax) - visited->items[childidx] = (void*)1; + if (visited->items[childidx] == (void*)1) continue; - } - else if (lim != -1) { - // when limited, don't include this match if it was covered by an earlier one - for (size_t result_len = 0; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; - } - visited->items[childidx] = (void*)1; - break; - } - } - } - } - if (!include_ambiguous && lim == -1) { - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - if ((size_t)visited->items[childidx] == 1) - continue; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_method_t *m = matc->method; - jl_value_t *ti = (jl_value_t*)matc->spec_types; - for (size_t j = depth - 1; j < stack->len; j++) { - if (i == j) - continue; - size_t idx2 = (size_t)stack->items[j]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - // if their intersection contributes to the ambiguity cycle - // and the contribution of m is fully ambiguous with the portion of the cycle from m2 - if (subt2 || jl_subtype((jl_value_t*)ti, m2->sig)) { - // but they aren't themselves simply ordered (here - // we don't consider that a third method might be - // disrupting that ordering and just consider them - // pairwise to keep this simple). - if (!jl_method_morespecific(m, m2) && !jl_method_morespecific(m2, m)) { - visited->items[childidx] = (void*)-1; - break; - } - } - } - } + scc_count++; } + if (scc_count > 1) + *has_ambiguity = 1; } // copy this cycle into the results for (size_t i = depth - 1; i < stack->len; i++) { size_t childidx = (size_t)stack->items[i]; + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers != NOT_FULLY_COVERS; + if (subt && *found_minmax) + visited->items[childidx] = (void*)1; if ((size_t)visited->items[childidx] == 1) continue; - if ((size_t)visited->items[childidx] != -1) { - assert(visited->items[childidx] == (void*)(2 + i)); - visited->items[childidx] = (void*)-1; - if (lim == -1 || result->len < lim) - arraylist_push(result, (void*)childidx); - else - return -1; - } + assert(visited->items[childidx] == (void*)(2 + i)); + visited->items[childidx] = (void*)1; + if (lim == -1 || result->len < lim) + arraylist_push(result, (void*)childidx); + else + return -1; } // now finally cleanup the stack while (stack->len >= depth) { size_t childidx = (size_t)arraylist_pop(stack); // always remove fully_covers matches after the first minmax ambiguity group is handled - //jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - if (matc->fully_covers != NOT_FULLY_COVERS && !addambig) + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers == FULLY_COVERS; + if (subt && *found_minmax == 1) *found_minmax = 2; - if (visited->items[childidx] != (void*)-1) - continue; - visited->items[childidx] = (void*)1; + assert(visited->items[childidx] == (void*)1); } return 0; } @@ -4624,7 +4683,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, } } // then scan everything - if (!ml_mtable_visitor(mt, &env.match) && env.t == jl_an_empty_vec_any) { + if (!jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), 0, &env.match) && env.t == jl_an_empty_vec_any) { JL_GC_POP(); // if we return early without returning methods, set only the min/max valid collected from matching *min_valid = env.match.min_valid; @@ -4638,43 +4697,31 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, env.match.ti = NULL; env.matc = NULL; env.match.env = NULL; search.env = NULL; size_t i, j, len = jl_array_nrows(env.t); jl_method_match_t *minmax = NULL; - int minmax_ambig = 0; - int all_subtypes = 1; + int any_subtypes = 0; if (len > 1) { // first try to pre-process the results to find the most specific - // result that fully covers the input, since we can do this in linear - // time, and the rest is O(n^2) + // result that fully covers the input, since we can do this in O(n^2) + // time, and the rest is O(n^3) // - first find a candidate for the best of these method results for (i = 0; i < len; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); if (matc->fully_covers == FULLY_COVERS) { + any_subtypes = 1; jl_method_t *m = matc->method; - if (minmax != NULL) { - jl_method_t *minmaxm = minmax->method; - if (jl_method_morespecific(minmaxm, m)) + for (j = 0; j < len; j++) { + if (i == j) continue; + jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(env.t, j); + if (matc2->fully_covers == FULLY_COVERS) { + jl_method_t *m2 = matc2->method; + if (!method_morespecific_via_interferences(m, m2)) + break; + } } - minmax = matc; - } - else { - all_subtypes = 0; - } - } - // - then see if it dominated all of the other choices - if (minmax != NULL) { - for (i = 0; i < len; i++) { - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); - if (matc == minmax) + if (j == len) { + // Found the minmax method + minmax = matc; break; - if (matc->fully_covers == FULLY_COVERS) { - jl_method_t *m = matc->method; - jl_method_t *minmaxm = minmax->method; - if (!jl_method_morespecific(minmaxm, m)) { - minmax_ambig = 1; - minmax = NULL; - has_ambiguity = 1; - break; - } } } } @@ -4687,24 +4734,31 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, // cost much extra and is less likely to help us hit a fast path // (we will look for this later, when we compute ambig_groupid, for // correctness) - if (!all_subtypes && minmax != NULL) { - jl_method_t *minmaxm = minmax->method; - all_subtypes = 1; + int all_subtypes = any_subtypes; + if (any_subtypes) { + jl_method_t *minmaxm = NULL; + if (minmax != NULL) + minmaxm = minmax->method; for (i = 0; i < len; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); if (matc->fully_covers != FULLY_COVERS) { jl_method_t *m = matc->method; - if (jl_method_morespecific(minmaxm, m)) - matc->fully_covers = SENTINEL; // put a sentinel value here for sorting - else - all_subtypes = 0; + if (minmaxm) { + if (method_morespecific_via_interferences(minmaxm, m)) { + matc->fully_covers = SENTINEL; // put a sentinel value here for sorting + continue; + } + if (method_in_interferences(minmaxm, m)) // !morespecific(m, minmaxm) + has_ambiguity = 1; + } + all_subtypes = 0; } } } // - now we might have a fast-return here, if we see that // we've already processed all of the possible outputs if (all_subtypes) { - if (minmax_ambig) { + if (minmax == NULL) { if (!include_ambiguous) { len = 0; env.t = jl_an_empty_vec_any; @@ -4715,7 +4769,6 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, } } else { - assert(minmax != NULL); jl_array_ptr_set(env.t, 0, minmax); jl_array_del_end((jl_array_t*)env.t, len - 1); len = 1; @@ -4728,20 +4781,23 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, } } if (len > 1) { - arraylist_t stack, visited, result, allambig; + arraylist_t stack, visited, result, recursion_stack; arraylist_new(&result, lim != -1 && lim < len ? lim : len); arraylist_new(&stack, 0); arraylist_new(&visited, len); - arraylist_new(&allambig, len); + arraylist_new(&recursion_stack, len); arraylist_grow(&visited, len); memset(visited.items, 0, len * sizeof(size_t)); // if we had a minmax method (any subtypes), now may now be able to // quickly cleanup some of methods int found_minmax = 0; - if (minmax != NULL) + if (has_ambiguity) + found_minmax = 1; + else if (minmax != NULL) found_minmax = 2; - else if (minmax_ambig && !include_ambiguous) + else if (any_subtypes && !include_ambiguous) found_minmax = 1; + has_ambiguity = 0; if (ambig == NULL) // if we don't care about the result, set it now so we won't bother attempting to compute it accurately later has_ambiguity = 1; for (i = 0; i < len; i++) { @@ -4752,9 +4808,9 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, // by visiting it and it might be a bit costly continue; } - int child_cycle = sort_mlmatches((jl_array_t*)env.t, i, &visited, &stack, &result, &allambig, lim == -1 || minmax == NULL ? lim : lim - 1, include_ambiguous, &has_ambiguity, &found_minmax); + int child_cycle = sort_mlmatches((jl_array_t*)env.t, i, &visited, &stack, &result, &recursion_stack, lim == -1 || minmax == NULL ? lim : lim - 1, include_ambiguous, &has_ambiguity, &found_minmax); if (child_cycle == -1) { - arraylist_free(&allambig); + arraylist_free(&recursion_stack); arraylist_free(&visited); arraylist_free(&stack); arraylist_free(&result); @@ -4765,89 +4821,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, assert(stack.len == 0); assert(visited.items[i] == (void*)1); } - // now compute whether there were ambiguities left in this cycle - if (has_ambiguity == 0 && allambig.len > 0) { - if (lim == -1) { - // lim is over-approximated, so has_ambiguities is too - has_ambiguity = 1; - } - else { - // go back and find the additional ambiguous methods and temporary add them to the stack - // (potentially duplicating them from lower on the stack to here) - jl_value_t *ti = NULL; - jl_value_t *isect2 = NULL; - JL_GC_PUSH2(&ti, &isect2); - for (size_t i = 0; i < allambig.len; i++) { - size_t idx = (size_t)allambig.items[i]; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx); - jl_method_t *m = matc->method; - int subt = matc->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - for (size_t idx2 = 0; idx2 < jl_array_nrows(env.t); idx2++) { - if (idx2 == idx) - continue; - // laborious test, checking for existence and coverage of another method (m3) - // outside of the ambiguity group that dominates any ambiguous methods, - // and means we can ignore this for has_ambiguity - // (has_ambiguity is overestimated for lim==-1, since we don't compute skipped matches either) - // n.b. even if we skipped them earlier, they still might - // contribute to the ambiguities (due to lock of transitivity of - // morespecific over subtyping) - // TODO: we could improve this result by checking if the removal of some - // edge earlier means that this subgraph is now well-ordered and then be - // allowed to ignore these vertexes entirely here - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx2); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - if (subt) { - ti = (jl_value_t*)matc2->spec_types; - isect2 = NULL; - } - else if (subt2) { - ti = (jl_value_t*)matc->spec_types; - isect2 = NULL; - } - else { - jl_type_intersection2((jl_value_t*)matc->spec_types, (jl_value_t*)matc2->spec_types, &ti, &isect2); - } - // if their intersection contributes to the ambiguity cycle - if (ti == jl_bottom_type) - continue; - // and they aren't themselves simply ordered - if (jl_method_morespecific(m, m2) || jl_method_morespecific(m2, m)) - continue; - // now look for a third method m3 that dominated these and that fully covered this intersection already - size_t k; - for (k = 0; k < result.len; k++) { - size_t idx3 = (size_t)result.items[k]; - if (idx3 == idx || idx3 == idx2) { - has_ambiguity = 1; - break; - } - jl_method_match_t *matc3 = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx3); - jl_method_t *m3 = matc3->method; - if ((jl_subtype(ti, m3->sig) || (isect2 && jl_subtype(isect2, m3->sig))) - && jl_method_morespecific(m3, m) && jl_method_morespecific(m3, m2)) { - //if (jl_subtype(matc->spec_types, ti) || jl_subtype(matc->spec_types, matc3->m3->sig)) - // // check if it covered not only this intersection, but all intersections with matc - // // if so, we do not need to check all of them separately - // j = len; - break; - } - } - if (k == result.len) - has_ambiguity = 1; - isect2 = NULL; - ti = NULL; - if (has_ambiguity) - break; - } - if (has_ambiguity) - break; - } - JL_GC_POP(); - } - } - arraylist_free(&allambig); + arraylist_free(&recursion_stack); arraylist_free(&visited); arraylist_free(&stack); for (j = 0; j < result.len; j++) { diff --git a/src/jltypes.c b/src/jltypes.c index dac68506cb0d4..b3434b2cbb95b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3537,12 +3537,13 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_type = jl_new_datatype(jl_symbol("Method"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(32, + jl_perm_symsvec(33, "name", "module", "file", "line", "dispatch_status", // atomic + "interferences", // atomic "primary_world", // atomic "sig", "specializations", // !const @@ -3570,12 +3571,13 @@ void jl_init_types(void) JL_GC_DISABLED "constprop", "max_varargs", "purity"), - jl_svec(32, + jl_svec(33, jl_symbol_type, jl_module_type, jl_symbol_type, jl_int32_type, jl_uint8_type, + jl_memory_any_type, jl_ulong_type, jl_type_type, jl_any_type, // union(jl_simplevector_type, jl_method_instance_type), @@ -3605,9 +3607,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_uint16_type), jl_emptysvec, 0, 1, 10); - //const static uint32_t method_constfields[1] = { 0b0 }; // (1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<6)|(1<<9)|(1<<10)|(1<<17)|(1<<21)|(1<<22)|(1<<23)|(1<<24)|(1<<25)|(1<<26)|(1<<27)|(1<<28)|(1<<29)|(1<<30); + //const static uint32_t method_constfields[] = { 0b0, 0b0 }; // (1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<6)|(1<<9)|(1<<10)|(1<<17)|(1<<21)|(1<<22)|(1<<23)|(1<<24)|(1<<25)|(1<<26)|(1<<27)|(1<<28)|(1<<29)|(1<<30); //jl_method_type->name->constfields = method_constfields; - const static uint32_t method_atomicfields[1] = { 0x10000030 }; // (1<<4)|(1<<5)||(1<<28) + const static uint32_t method_atomicfields[] = { 0x20000070, 0x0 }; // (1<<4)|(1<<5)|(1<<6)|(1<<29) jl_method_type->name->atomicfields = method_atomicfields; jl_method_instance_type = @@ -3634,7 +3636,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, 0, 1, 3); // These fields should be constant, but Serialization wants to mutate them in initialization - //const static uint32_t method_instance_constfields[1] = { 0b0000111 }; // fields 1, 2, 3 + //const static uint32_t method_instance_constfields[1] = { 0b00000111 }; // fields 1, 2, 3 const static uint32_t method_instance_atomicfields[1] = { 0b11010000 }; // fields 5, 7, 8 //Fields 4 and 5 must be protected by method->write_lock, and thus all operations on jl_method_instance_t are threadsafe. //jl_method_instance_type->name->constfields = method_instance_constfields; @@ -3854,7 +3856,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_methcache_type->types, 2, jl_long_type); // voidpointer jl_svecset(jl_methcache_type->types, 3, jl_long_type); // uint32_t plus alignment jl_svecset(jl_methtable_type->types, 3, jl_module_type); - jl_svecset(jl_method_type->types, 13, jl_method_instance_type); + jl_svecset(jl_method_type->types, 14, jl_method_instance_type); //jl_svecset(jl_debuginfo_type->types, 0, jl_method_instance_type); // union(jl_method_instance_type, jl_method_type, jl_symbol_type) jl_svecset(jl_method_instance_type->types, 4, jl_code_instance_type); jl_svecset(jl_code_instance_type->types, 19, jl_voidpointer_type); diff --git a/src/julia.h b/src/julia.h index 19c8408f22b4a..e07fabcb10349 100644 --- a/src/julia.h +++ b/src/julia.h @@ -324,6 +324,7 @@ typedef struct _jl_method_t { jl_sym_t *file; int32_t line; _Atomic(uint8_t) dispatch_status; // bits defined in staticdata.jl + _Atomic(jl_genericmemory_t*) interferences; // set of intersecting methods not more specific _Atomic(size_t) primary_world; // method's type signature. redundant with TypeMapEntry->specTypes diff --git a/src/julia_internal.h b/src/julia_internal.h index 7df8e6bca499b..1312f3d5cb497 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -696,8 +696,6 @@ typedef union { #define METHOD_SIG_LATEST_WHICH 0b0001 #define METHOD_SIG_LATEST_ONLY 0b0010 #define METHOD_SIG_PRECOMPILE_MANY 0b0100 -#define METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC 0b1000 // indicates there exists some other method that is not more specific than this one -#define METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC 0b10000 // precompiled version of METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC JL_DLLEXPORT jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner); JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src); @@ -1765,6 +1763,7 @@ JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *a); JL_DLLEXPORT int jl_stored_inline(jl_value_t *el_type); JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a); JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); +JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy(jl_genericmemory_t *mem); JL_DLLEXPORT uintptr_t jl_object_id_(uintptr_t tv, jl_value_t *v) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT; diff --git a/src/method.c b/src/method.c index a04035086df90..8220178964333 100644 --- a/src/method.c +++ b/src/method.c @@ -1008,6 +1008,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->nargs = 0; jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); jl_atomic_store_relaxed(&m->dispatch_status, 0); + jl_atomic_store_relaxed(&m->interferences, (jl_genericmemory_t*)jl_an_empty_memory_any); m->is_for_opaque_closure = 0; m->nospecializeinfer = 0; jl_atomic_store_relaxed(&m->did_scan_source, 0); diff --git a/src/staticdata.c b/src/staticdata.c index ba6b40e2e5ea3..be1188572f3b0 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1854,8 +1854,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED int new_dispatch_status = 0; if (!(dispatch_status & METHOD_SIG_LATEST_ONLY)) new_dispatch_status |= METHOD_SIG_PRECOMPILE_MANY; - if (dispatch_status & METHOD_SIG_LATEST_HAS_NOTMORESPECIFIC) - new_dispatch_status |= METHOD_SIG_PRECOMPILE_HAS_NOTMORESPECIFIC; jl_atomic_store_relaxed(&newm->dispatch_status, new_dispatch_status); arraylist_push(&s->fixup_objs, (void*)reloc_offset); } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 0b8cfc1cf4ebd..e676eabbddad3 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -726,7 +726,11 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size } for (i = 0; i < l; i++) { jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_array_ptr_ref(external, i); + //uint64_t t0 = uv_hrtime(); jl_method_table_activate(entry); + //jl_printf(JL_STDERR, "%f ", (double)(uv_hrtime() - t0) / 1e6); + //jl_static_show(JL_STDERR, entry->func.value); + //jl_printf(JL_STDERR, "\n"); } } } diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index cafc3040ef511..b9129707def46 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -2226,7 +2226,6 @@ function detect_ambiguities(mods::Module...; end function examine(mt::Core.MethodTable) for m in Base.MethodList(mt) - m.sig == Tuple && continue # ignore Builtins is_in_mods(parentmodule(m), recursive, mods) || continue world = Base.get_world_counter() ambig = Ref{Int32}(0) diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 0f29817e74dd5..df3ba89e3eb2a 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -19,7 +19,7 @@ include("testenv.jl") @test length(methods(ambig, (Int, Int))) == 1 @test length(methods(ambig, (UInt8, Int))) == 0 -@test length(Base.methods_including_ambiguous(ambig, (UInt8, Int))) == 3 +@test length(Base.methods_including_ambiguous(ambig, (UInt8, Int))) == 2 @test ambig("hi", "there") == 1 @test ambig(3.1, 3.2) == 5 @@ -42,7 +42,6 @@ let err = try errstr = String(take!(io)) @test occursin(" ambig(x, y::Integer)\n @ $curmod_str", errstr) @test occursin(" ambig(x::Integer, y)\n @ $curmod_str", errstr) - @test occursin(" ambig(x::Number, y)\n @ $curmod_str", errstr) @test occursin("Possible fix, define\n ambig(::Integer, ::Integer)", errstr) end @@ -160,7 +159,7 @@ ambig(::Signed, ::Int) = 3 ambig(::Int, ::Signed) = 4 end ambs = detect_ambiguities(Ambig48312) -@test length(ambs) == 4 +@test length(ambs) == 1 # only ambiguous over (Int, Int), which is 3 or 4 module UnboundAmbig55868 module B @@ -287,7 +286,7 @@ end @test isempty(methods(Ambig8.f, (Int,))) @test isempty(methods(Ambig8.g, (Int,))) for f in (Ambig8.f, Ambig8.g) - @test length(methods(f, (Integer,))) == 2 # 1 is also acceptable + @test length(methods(f, (Integer,))) == 2 # 3 is also acceptable @test length(methods(f, (Signed,))) == 1 # 2 is also acceptable @test length(Base.methods_including_ambiguous(f, (Signed,))) == 2 @test f(0x00) == 1 @@ -413,7 +412,7 @@ let has_ambig = Ref(Int32(0)) ms = Base._methods_by_ftype(Tuple{typeof(fnoambig), Any, Any}, nothing, 4, Base.get_world_counter(), false, Ref(typemin(UInt)), Ref(typemax(UInt)), has_ambig) @test ms isa Vector @test length(ms) == 4 - @test has_ambig[] == 0 + @test has_ambig[] == 1 # 0 is better, but expensive and probably unnecessary to compute end # issue #11407 @@ -459,15 +458,38 @@ struct U55231{P} end struct V55231{P} end U55231(::V55231) = nothing (::Type{T})(::V55231) where {T<:U55231} = nothing -@test length(methods(U55231)) == 2 +@test length(methods(U55231)) == 1 U55231(a, b) = nothing -@test length(methods(U55231)) == 3 +@test length(methods(U55231)) == 2 struct S55231{P} end struct T55231{P} end (::Type{T})(::T55231) where {T<:S55231} = nothing S55231(::T55231) = nothing -@test length(methods(S55231)) == 2 +@test length(methods(S55231)) == 1 S55231(a, b) = nothing -@test length(methods(S55231)) == 3 +@test length(methods(S55231)) == 2 + +ambig10() = 1 +ambig10(a::Vararg{Any}) = 2 +ambig10(a::Vararg{Union{Int32,Int64}}) = 6 +ambig10(a::Vararg{Matrix}) = 4 +ambig10(a::Vararg{Number}) = 7 +ambig10(a::Vararg{N}) where {N<:Number} = 5 +let ambig = Ref{Int32}(0) + ms = Base._methods_by_ftype(Tuple{typeof(ambig10), Vararg}, nothing, -1, Base.get_world_counter(), false, Ref{UInt}(typemin(UInt)), Ref{UInt}(typemax(UInt)), ambig) + @test ms isa Vector + @test length(ms) == 6 + @test_broken ambig[] == 0 +end +let ambig = Ref{Int32}(0) + ms = Base._methods_by_ftype(Tuple{typeof(ambig10), Vararg{Number}}, nothing, -1, Base.get_world_counter(), false, Ref{UInt}(typemin(UInt)), Ref{UInt}(typemax(UInt)), ambig) + @test ms isa Vector + @test length(ms) == 4 + @test_broken ambig[] == 0 + @test ms[1].method === which(ambig10, ()) + @test ms[2].method === which(ambig10, (Vararg{Union{Int32, Int64}},)) + @test ms[3].method === which(ambig10, Tuple{Vararg{N}} where N<:Number,) + @test ms[4].method === which(ambig10, (Vararg{Number},)) +end nothing diff --git a/test/core.jl b/test/core.jl index 4af61233ac458..3d79802e33a87 100644 --- a/test/core.jl +++ b/test/core.jl @@ -36,7 +36,7 @@ end for (T, c) in ( (Core.CodeInfo, []), (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), - (Core.Method, [:primary_world, :did_scan_source, :dispatch_status]), + (Core.Method, [:primary_world, :did_scan_source, :dispatch_status, :interferences]), (Core.MethodInstance, [:cache, :flags, :dispatch_status]), (Core.MethodTable, [:defs]), (Core.MethodCache, [:leafcache, :cache, :var""]), From 5f0f2ebfb071b58d21681d0c21240fa0fb9ae6f9 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 22 Jul 2025 20:08:33 -0400 Subject: [PATCH 557/662] build/llvm: Remove bash-specific curly expansion (#59058) Fixes #59050 --- deps/llvm.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 7c4acff9e537b..21b8479fe4ae7 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -313,7 +313,9 @@ endif LLVM_INSTALL = \ cd $1 && mkdir -p $2$$(build_depsbindir)/lit && \ - cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/{*.py,*.toml,lit/} $2$$(build_depsbindir)/lit/ && \ + cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.py $2$$(build_depsbindir)/lit/ && \ + cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.toml $2$$(build_depsbindir)/lit/ && \ + cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/lit $2$$(build_depsbindir)/lit/ && \ $$(CMAKE) -DCMAKE_INSTALL_PREFIX="$2$$(build_prefix)" -P cmake_install.cmake ifeq ($(OS), WINNT) LLVM_INSTALL += && cp $2$$(build_shlibdir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_depsbindir) From 9578e4bbe1cd77a96d92bbaab4bcc8c2aeedb6c4 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 23 Jul 2025 06:35:42 -0400 Subject: [PATCH 558/662] build: More msys2 fixes (#59028) --- contrib/mac/app/Makefile | 2 +- deps/libgit2.mk | 3 ++- deps/libssh2.mk | 5 +---- deps/libuv.mk | 2 +- deps/llvm.mk | 2 +- deps/openssl.mk | 9 +++++++-- deps/tools/common.mk | 20 ++++++++++++++++---- deps/zstd.mk | 8 ++++++-- src/Makefile | 11 ++++++----- src/flisp/Makefile | 2 +- 10 files changed, 42 insertions(+), 22 deletions(-) diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index a8fd6e16b3f44..70436a857c265 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -47,7 +47,7 @@ dmg/$(APP_NAME): startup.applescript julia.icns plutil -insert CFBundleVersion -string "$(JULIA_VERSION_OPT_COMMIT)" $@/Contents/Info.plist plutil -insert NSHumanReadableCopyright -string "$(APP_COPYRIGHT)" $@/Contents/Info.plist -mkdir -p $@/Contents/Resources/julia - make -C $(JULIAHOME) binary-dist + $(MAKE) -C $(JULIAHOME) binary-dist $(TAR) -xzf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 find $@/Contents/Resources/julia -type f -exec chmod -w {} \; # Even though the tarball may already be signed, we re-sign here to make it easier to add diff --git a/deps/libgit2.mk b/deps/libgit2.mk index a6affb3897110..fad5db5fd4f79 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -4,6 +4,7 @@ ifneq ($(USE_BINARYBUILDER_LIBGIT2),1) LIBGIT2_GIT_URL := https://github.com/libgit2/libgit2.git LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1 $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,,$(SRCCACHE))) +$(SRCCACHE)/$(LIBGIT2_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) ifeq ($(USE_SYSTEM_LIBSSH2), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/libssh2 @@ -21,7 +22,7 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/zlib endif -LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=OFF -DUSE_SSH=ON -DREGEX_BACKEND=pcre2 -DBUILD_CLI=OFF +LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=OFF -DUSE_SSH=ON -DREGEX_BACKEND=pcre2 -DBUILD_CLI=OFF -DBUILD_TESTS=OFF ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifeq ($(USE_SYSTEM_LIBSSH2), 0) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 2108ec4d2ef7f..3439ace10da16 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -17,9 +17,6 @@ endif ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF -ifeq ($(BUILD_OS),WINNT) -LIBSSH2_OPTS += -G"MSYS Makefiles" -endif else LIBSSH2_OPTS += -DCRYPTO_BACKEND=OpenSSL -DENABLE_ZLIB_COMPRESSION=OFF endif @@ -37,7 +34,7 @@ LIBSSH2_SRC_PATH := $(SRCCACHE)/$(LIBSSH2_SRC_DIR) $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: $(LIBSSH2_SRC_PATH)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) -G"Unix Makefiles" $(dir $<) $(LIBSSH2_OPTS) + $(CMAKE) $(CMAKE_GENERATOR_COMMAND) $(dir $<) $(LIBSSH2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured diff --git a/deps/libuv.mk b/deps/libuv.mk index eacabac55e34f..993aa4fc144da 100644 --- a/deps/libuv.mk +++ b/deps/libuv.mk @@ -4,7 +4,7 @@ LIBUV_GIT_URL:=https://github.com/JuliaLang/libuv.git LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1 $(eval $(call git-external,libuv,LIBUV,configure,,$(SRCCACHE))) -UV_CFLAGS := -O2 +UV_CFLAGS := -O2 -DBUILDING_UV_SHARED=1 UV_FLAGS := LDFLAGS="$(LDFLAGS) $(CLDFLAGS) -v" UV_FLAGS += CFLAGS="$(CFLAGS) $(UV_CFLAGS) $(SANITIZE_OPTS)" diff --git a/deps/llvm.mk b/deps/llvm.mk index 21b8479fe4ae7..77b8f4ae978c1 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -14,7 +14,7 @@ $(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) # symlinks. We don't particularly care either way - we just need to symlinks # to succeed. We could guard this by a uname check, but it's harmless elsewhere, # so let's not incur the additional overhead. -$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native +$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) LLVM_BUILDDIR := $(BUILDDIR)/$(LLVM_SRC_DIR) LLVM_BUILDDIR_withtype := $(LLVM_BUILDDIR)/build_$(LLVM_BUILDTYPE) diff --git a/deps/openssl.mk b/deps/openssl.mk index 6f96717b2fb74..734ddb3274e78 100644 --- a/deps/openssl.mk +++ b/deps/openssl.mk @@ -59,7 +59,7 @@ $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-configured: $(SRCCACHE)/openssl-$(OPENS mkdir -p $(dir $@) cd $(dir $@) && \ CC="$(CC) $(SANITIZE_OPTS)" CXX="$(CXX) $(SANITIZE_OPTS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN) $(SANITIZE_LDFLAGS)" \ - $(dir $<)/Configure shared --prefix=$(abspath $(build_prefix)) $(OPENSSL_TARGET) + $(dir $<)/Configure shared --prefix=$(abspath $(build_prefix)) --libdir=$(abspath $(build_libdir)) $(OPENSSL_TARGET) echo 1 > $@ $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-compiled: $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-configured @@ -72,9 +72,14 @@ ifeq ($(OS),$(BUILD_OS)) endif echo 1 > $@ +# Override bindir and only install runtime libraries, otherwise they'll go into build_depsbindir. +OPENSSL_INSTALL = \ + mkdir -p $2$$(build_shlibdir) && \ + $$(MAKE) -C $1 install_runtime_libs $$(MAKE_COMMON) bindir=$$(build_shlibdir) $3 DESTDIR="$2" + $(eval $(call staged-install, \ openssl,openssl-$(OPENSSL_VER), \ - MAKE_INSTALL,,, \ + OPENSSL_INSTALL,,, \ $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libcrypto-*.dll $(build_bindir)/libcrypto.dll && \ $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libssl-*.dll $(build_bindir)/libssl.dll && \ $$(INSTALL_NAME_CMD)libcrypto.$$(SHLIB_EXT) $$(build_shlibdir)/libcrypto.$$(SHLIB_EXT) && \ diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 5e4b9ce0b9b62..d689da52daf2f 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -75,7 +75,6 @@ endif CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)" endif -# For now this is LLVM specific, but I expect it won't be in the future ifeq ($(CMAKE_GENERATOR),Ninja) CMAKE_GENERATOR_COMMAND := -G Ninja else ifeq ($(CMAKE_GENERATOR),make) @@ -84,14 +83,27 @@ else $(error Unknown CMake generator '$(CMAKE_GENERATOR)'. Options are 'Ninja' and 'make') endif -# Detect MSYS2 with cygwin CMake rather than MinGW cmake - the former fails to -# properly drive MinGW tools ifneq (,$(findstring MINGW,$(RAW_BUILD_OS))) ifneq (,$(shell ldd $(shell which cmake) | grep msys-2.0.dll)) +# Detect MSYS2 with cygwin CMake rather than MinGW cmake - the former fails to +# properly drive MinGW tools override CMAKE := echo "ERROR: CMake is Cygwin CMake, not MinGW CMake. Build will fail. Use 'pacman -S mingw-w64-{i686,x86_64}-cmake'."; exit 1; $(CMAKE) endif +# In our setup, CMAKE_INSTALL_PREFIX is a relative path inside usr-staging. +# We do not want this converted to a windows path, because our make system +# assumes it to be relative to msys `/`. +override CMAKE := MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX" $(CMAKE) endif +# Some dependencies' tarballs contains symlinks to non-existent targets. This breaks the +# the default msys strategy `deepcopy` symlink strategy. To workaround this, +# switch to `native` which tries native windows symlinks (possible if the +# machine is in developer mode) - or if not, falls back to cygwin-style +# symlinks. We don't particularly care either way - we just need to symlinks +# to succeed. We could guard this by a uname check, but it's harmless elsewhere, +# so let's not incur the additional overhead. +MSYS_NONEXISTENT_SYMLINK_TARGET_FIX := export MSYS=winsymlinks:native + # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libdir=$(build_libdir) shlibdir=$(build_shlibdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= @@ -168,7 +180,7 @@ upper = $(shell echo $1 | tr a-z A-Z) # this rule ensures that make install is more nearly atomic # so it's harder to get half-installed (or half-reinstalled) dependencies # # and enables sharing deps compiles, uninstall, and fast reinstall -MAKE_INSTALL = +$$(MAKE) -C $1 install $$(MAKE_COMMON) $3 DESTDIR="$2" +MAKE_INSTALL = MSYS2_ARG_CONV_EXCL="prefix=" $$(MAKE) -C $1 install $$(MAKE_COMMON) $3 DESTDIR="$2" define SHLIBFILE_INSTALL mkdir -p $2/$$(build_shlibdir) diff --git a/deps/zstd.mk b/deps/zstd.mk index 73cdde2b6b27c..4fc64a8442588 100644 --- a/deps/zstd.mk +++ b/deps/zstd.mk @@ -3,10 +3,14 @@ ifneq ($(USE_BINARYBUILDER_ZSTD), 1) ZSTD_GIT_URL := https://github.com/facebook/zstd.git ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 $(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) -# See note in llvm.mk for source tarballs with symlinks to non-existent targets -$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: export MSYS=winsymlinks:native +$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) +ifeq ($(OS), WINNT) +# Zstd detects "Windows" not WINNT, ordinarily from the inherited $(OS), but it expects the +# override to be done using TARGET_SYSTEM. +ZSTD_BUILD_OPTS += TARGET_SYSTEM="Windows" +endif $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted echo 1 > $@ diff --git a/src/Makefile b/src/Makefile index a8926a46c9b00..37e58f0620760 100644 --- a/src/Makefile +++ b/src/Makefile @@ -77,7 +77,7 @@ else GC_CODEGEN_SRCS += llvm-late-gc-lowering-stock endif CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop \ - llvm-pass-helpers llvm-ptls llvm-propagate-addrspaces null_sysimage \ + llvm-pass-helpers llvm-ptls llvm-propagate-addrspaces \ llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \ llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures llvm-expand-atomic-modify \ pipeline llvm_api \ @@ -198,6 +198,7 @@ endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) +# NB: CG needs uv_mutex_* symbols, but we expect to export them from libjulia-internal CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) ifeq (${USE_THIRD_PARTY_GC},mmtk) @@ -494,10 +495,10 @@ libjulia-codegen-debug: $(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_MINO libjulia-codegen-debug libjulia-codegen-release: $(PUBLIC_HEADER_TARGETS) # set the exports for the source files based on where they are getting linked -$(OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -$(DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -$(CODEGEN_OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -$(CODEGEN_DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN +$(OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -DBUILDING_UV_SHARED +$(DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -DBUILDING_UV_SHARED +$(CODEGEN_OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -DUSING_UV_SHARED +$(CODEGEN_DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -DUSING_UV_SHARED clean: -rm -fr $(build_shlibdir)/libjulia-internal* $(build_shlibdir)/libjulia-codegen* $(build_shlibdir)/libccalltest* $(build_shlibdir)/libllvmcalltest* diff --git a/src/flisp/Makefile b/src/flisp/Makefile index eca1de86e588a..2e902a31dbeed 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -117,7 +117,7 @@ $(BUILDDIR)/host/Makefile: @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ $(BUILDDIR)/host/$(EXENAME): $(BUILDDIR)/host/Makefile | ${BUILDDIR}/host/flisp.boot - make -C $(BUILDDIR)/host $(EXENAME) + $(MAKE) -C $(BUILDDIR)/host $(EXENAME) $(BUILDDIR)/host/flisp.boot: $(SRCDIR)/flisp.boot | $(BUILDDIR)/host/Makefile From 4718f43b5afe34ae7aa12d60755a817f0e4f6808 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 23 Jul 2025 06:44:34 -0400 Subject: [PATCH 559/662] remove a testset from MMAP that might cause CI to now fail on Windows (#59062) --- stdlib/Mmap/test/runtests.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index 064217a24e3be..4ab04bf733190 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -343,18 +343,18 @@ end GC.gc() rm(file) -@testset "test for #58982 - mmap with primitive types" begin - file = tempname() - primitive type PrimType9Bytes 9*8 end - arr = Vector{PrimType9Bytes}(undef, 2) - write(file, arr) - m = mmap(file, Vector{PrimType9Bytes}) - @test length(m) == 2 - @test m[1] == arr[1] - @test m[2] == arr[2] - finalize(m); m = nothing; GC.gc() - rm(file) -end +# test for #58982 - mmap with primitive types +file = tempname() +primitive type PrimType9Bytes 9*8 end +arr = Vector{PrimType9Bytes}(undef, 2) +write(file, arr) +m = mmap(file, Vector{PrimType9Bytes}) +@test length(m) == 2 +@test m[1] == arr[1] +@test m[2] == arr[2] +finalize(m); m = nothing; GC.gc() +rm(file) + @testset "Docstrings" begin @test isempty(Docs.undocumented_names(Mmap)) From 96b5b2c0a238107fda6c571eba72d18f998c01fb Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 23 Jul 2025 14:38:18 -0400 Subject: [PATCH 560/662] Use a dedicated parameter attribute to identify the gstack arg. (#59059) Otherwise, on systems without SwitfCC support (i.e. RISC-V) `getPGCstack` may return null, disabling the final GC pass. --- src/codegen.cpp | 8 ++++++-- src/llvm-pass-helpers.cpp | 9 +++++---- src/llvm-ptls.cpp | 3 ++- test/llvmpasses/alloc-opt-pass.ll | 10 +++++----- test/llvmpasses/late-lower-gc-sret.ll | 20 ++++++++++---------- test/llvmpasses/late-lower-gc.ll | 2 +- 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 3c37a80e5e088..6ff0a2000fc75 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6788,9 +6788,12 @@ static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, con rhs.promotion_point = nullptr; rhs.promotion_ssa = -1; if (gcstack_arg) { - w->setCallingConv(CallingConv::Swift); AttrBuilder param(ctx.builder.getContext()); - param.addAttribute(Attribute::SwiftSelf); + if (ctx.emission_context.use_swiftcc) { + w->setCallingConv(CallingConv::Swift); + param.addAttribute(Attribute::SwiftSelf); + } + param.addAttribute("gcstack"); param.addAttribute(Attribute::NonNull); Argument *gcstackarg = &*AI++; gcstackarg->addAttrs(param); @@ -7966,6 +7969,7 @@ static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module AttrBuilder param(M->getContext()); if (params.use_swiftcc) param.addAttribute(Attribute::SwiftSelf); + param.addAttribute("gcstack"); param.addAttribute(Attribute::NonNull); attrs.push_back(AttributeSet::get(M->getContext(), param)); fsig.push_back(PointerType::get(M->getContext(), 0)); diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index dbafc12aeff94..a6a16a7f4956c 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -88,10 +88,11 @@ llvm::Value *JuliaPassContext::getPGCstack(llvm::Function &F) const } } } - if (F.getCallingConv() == CallingConv::Swift) { - for (auto &arg : F.args()) { - if (arg.hasSwiftSelfAttr()) - return &arg; + for (auto &arg : F.args()) { + // Check for the "gcstack" attribute + AttributeSet attrs = F.getAttributes().getParamAttrs(arg.getArgNo()); + if (attrs.hasAttribute("gcstack")) { + return &arg; } } return nullptr; diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index ca1868b85d4e0..a7bc79afd3eb4 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -344,7 +344,8 @@ bool LowerPTLS::run(bool *CFGModified) auto f = call->getCaller(); Value *pgcstack = NULL; for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end(); ++arg) { - if (arg->hasSwiftSelfAttr()) { + AttributeSet attrs = f->getAttributes().getParamAttrs(arg->getArgNo()); + if (attrs.hasAttribute("gcstack")) { pgcstack = &*arg; break; } diff --git a/test/llvmpasses/alloc-opt-pass.ll b/test/llvmpasses/alloc-opt-pass.ll index 83f2118412cc1..c6c279ae36fc6 100644 --- a/test/llvmpasses/alloc-opt-pass.ll +++ b/test/llvmpasses/alloc-opt-pass.ll @@ -197,10 +197,10 @@ union_move9: ; No predecessors! @1 = private unnamed_addr constant i64 0, align 8 ; CHECK-LABEL: @cmpxchg -; CHECK: alloca +; CHECK: alloca ; CHECK: alloca ; CHECK: %20 = cmpxchg ptr %2, -define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself %0) #0 { +define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself "gcstack" %0) #0 { %2 = alloca i64, align 16 %3 = call ptr @julia.get_pgcstack() %4 = getelementptr inbounds i8, ptr %3, i32 -152 @@ -229,7 +229,7 @@ define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself %0) #0 { 19: ; preds = %19, %1 %20 = phi i64 [ %17, %1 ], [ %23, %19 ] - %21 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself %3, i64 signext %20, i64 signext 1) + %21 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself "gcstack" %3, i64 signext %20, i64 signext 1) %22 = cmpxchg ptr addrspace(11) %16, i64 %20, i64 %21 seq_cst monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 %23 = extractvalue { i64, i1 } %22, 0 %24 = extractvalue { i64, i1 } %22, 1 @@ -241,7 +241,7 @@ define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself %0) #0 { ; CHECK: alloca ; CHECK: alloca ; CHECK: atomicrmw xchg ptr %2, -define swiftcc i64 @"atomicrmw"(ptr nonnull swiftself %0) #0 { +define swiftcc i64 @"atomicrmw"(ptr nonnull swiftself "gcstack" %0) #0 { %2 = alloca i64, align 16 %3 = call ptr @julia.get_pgcstack() %4 = getelementptr inbounds i8, ptr %3, i32 -152 @@ -263,7 +263,7 @@ define swiftcc i64 @"atomicrmw"(ptr nonnull swiftself %0) #0 { call void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) align 8 %15, ptr align 8 @1, i64 8, i1 false), !tbaa !20, !alias.scope !23, !noalias !24 %16 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) %17 = load atomic i64, ptr addrspace(11) %16 monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 - %18 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself %3, i64 signext %17, i64 signext 1) + %18 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself "gcstack" %3, i64 signext %17, i64 signext 1) %19 = atomicrmw xchg ptr addrspace(11) %16, i64 %18 seq_cst, align 8, !tbaa !25, !alias.scope !23, !noalias !24 ; preds = %19 ret i64 %19 } diff --git a/test/llvmpasses/late-lower-gc-sret.ll b/test/llvmpasses/late-lower-gc-sret.ll index d0ad94fcf8990..b8593f691bb6f 100644 --- a/test/llvmpasses/late-lower-gc-sret.ll +++ b/test/llvmpasses/late-lower-gc-sret.ll @@ -6,7 +6,7 @@ declare ptr @julia.get_pgcstack() declare swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]), ptr nonnull swiftself, ptr addrspace(10) nonnull) -define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftself %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { +define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { ; CHECK-LABEL: @sret_select ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 6) ; CHECK: call ptr @julia.get_gc_frame_slot(ptr %gcframe, i32 3) @@ -17,12 +17,12 @@ define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftse %3 = alloca [3 x i64], align 8 %4 = alloca [3 x i64], align 8 %5 = select i1 %unpredictable, ptr %3, ptr %4 - call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %5, ptr nonnull swiftself %0, ptr addrspace(10) nonnull %1) + call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %5, ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) nonnull %1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret ptr addrspace(10) %1 } -define hidden swiftcc nonnull ptr addrspace(10) @sret_phi(ptr nonnull swiftself %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { +define hidden swiftcc nonnull ptr addrspace(10) @sret_phi(ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { top: ; CHECK-LABEL: @sret_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 6) @@ -43,14 +43,14 @@ false: ; preds = %top ret: ; preds = %false, %true %4 = phi ptr [ %2, %true ], [ %3, %false ] - call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %4, ptr nonnull swiftself %0, ptr addrspace(10) nonnull %1) + call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %4, ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) nonnull %1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret ptr addrspace(10) %1 } declare swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }), ptr noalias nocapture noundef, ptr nonnull swiftself) -define hidden swiftcc void @sret_gc_root_phi(ptr nonnull swiftself %0, i1 %unpredictable) { +define hidden swiftcc void @sret_gc_root_phi(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable) { top: ; CHECK-LABEL: @sret_gc_root_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 2) @@ -75,13 +75,13 @@ false: ; preds = %top ret: ; preds = %false, %true %4 = phi ptr [ %2, %true ], [ %3, %false ] - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %4, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %4, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } -define hidden swiftcc void @sret_gc_root_phi_select(ptr nonnull swiftself %0, i1 %unpredictable, i1 %unpredictable2) { +define hidden swiftcc void @sret_gc_root_phi_select(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable, i1 %unpredictable2) { top: ; CHECK-LABEL: @sret_gc_root_phi_select ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 3) @@ -110,12 +110,12 @@ false: ; preds = %top ret: ; preds = %false, %true %5 = phi ptr [ %2, %true ], [ %3, %false ] %6 = select i1 %unpredictable2, ptr %4, ptr %5 - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } -define hidden swiftcc void @sret_gc_root_select_phi(ptr nonnull swiftself %0, i1 %unpredictable, i1 %unpredictable2) { +define hidden swiftcc void @sret_gc_root_select_phi(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable, i1 %unpredictable2) { top: ; CHECK-LABEL: @sret_gc_root_select_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 3) @@ -145,7 +145,7 @@ false: ; preds = %top ret: ; preds = %false, %true %6 = phi ptr [ %2, %true ], [ %5, %false ] - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index c3d6ea10c1511..346e19e537819 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -199,7 +199,7 @@ define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { ; CHECK: %r = call i32 @callee_root(ptr addrspace(10) %l0, ptr addrspace(10) %l1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) -define swiftcc ptr addrspace(10) @insert_element(ptr swiftself %0) { +define swiftcc ptr addrspace(10) @insert_element(ptr swiftself "gcstack" %0) { ; CHECK-LABEL: @insert_element %2 = alloca [10 x i64], i32 1, align 8 ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 10) From 9aa2e6817a8ef317ec8f5a87eec0414e8e72cc08 Mon Sep 17 00:00:00 2001 From: Thomas Christensen Date: Thu, 24 Jul 2025 01:55:42 +0200 Subject: [PATCH 561/662] skip unnecessary alias-check in `collect(::AbstractArray)` from `copyto!` (#55748) As discussed on Slack with @MasonProtter & @jakobnissen, `collect` currently does a usually cheap - but sometimes expensive - aliasing check (via `unalias`->`mightalias`->`dataid` -> `objectid`) before copying contents over; this check is unnecessary, however, since the source array is newly created and cannot possibly alias the input. This PR fixes that by swapping from `copyto!` to `copyto_unaliased!` in the `_collect_indices` implementations where the swap is straightforward (e.g., it is not so straightforward for the fallback `_collect_indices(indsA, A)`, so I skipped it there). This improves the following example substantially: ```jl struct GarbageVector{N} <: AbstractVector{Int} v :: Vector{Int} garbage :: NTuple{N, Int} end GarbageVector{N}(v::Vector{Int}) where N = GarbageVector{N}(v, ntuple(identity, Val(N))) Base.getindex(gv::GarbageVector, i::Int) = gv.v[i] Base.size(gv::GarbageVector) = size(gv.v) using BenchmarkTools v = rand(Int, 10) gv = GarbageVector{100}(v) @btime collect($v); # 30 ns (v1.10.4) -> 30 ns (PR) @btime collect($gv); # 179 ns (v1.10.4) -> 30 ns (PR) ``` Relatedly, it seems the fact that `mightalias` is comparing immutable contents as well - and hence slowing down the `unalias` check for the above `GarbageVector` via a slow `objectid` on tuples - is suboptimal. I don't know how to fix that though, so I'd like to leave that outside this PR. (Probably related to https://github.com/JuliaLang/julia/pull/26237) Co-authored-by: Matt Bauman --- base/array.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index cacef7c4acd5d..48f197d429740 100644 --- a/base/array.jl +++ b/base/array.jl @@ -748,9 +748,16 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) return a end -_collect_indices(::Tuple{}, A) = copyto!(Array{eltype(A),0}(undef), A) -_collect_indices(indsA::Tuple{Vararg{OneTo}}, A) = - copyto!(Array{eltype(A)}(undef, length.(indsA)), A) +function _collect_indices(::Tuple{}, A) + dest = Array{eltype(A),0}(undef) + isempty(A) && return dest + return copyto_unaliased!(IndexStyle(dest), dest, IndexStyle(A), A) +end +function _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) + dest = Array{eltype(A)}(undef, length.(indsA)) + isempty(A) && return dest + return copyto_unaliased!(IndexStyle(dest), dest, IndexStyle(A), A) +end function _collect_indices(indsA, A) B = Array{eltype(A)}(undef, length.(indsA)) copyto!(B, CartesianIndices(axes(B)), A, CartesianIndices(indsA)) From c7460488e4013b3260715e71cc011ca135fcbb93 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 23 Jul 2025 22:23:34 -0400 Subject: [PATCH 562/662] Fix and update Revise manifest (#59077) --- deps/jlutilities/revise/Manifest.toml | 29 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/deps/jlutilities/revise/Manifest.toml b/deps/jlutilities/revise/Manifest.toml index 27ea36fc31de4..143f20b08e9c1 100644 --- a/deps/jlutilities/revise/Manifest.toml +++ b/deps/jlutilities/revise/Manifest.toml @@ -18,6 +18,16 @@ git-tree-sha1 = "062c5e1a5bf6ada13db96a4ae4749a4c2234f521" uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" version = "1.3.9" +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" @@ -44,12 +54,12 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" version = "1.9.1+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -58,10 +68,10 @@ uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" [[deps.LoweredCodeUtils]] -deps = ["JuliaInterpreter"] -git-tree-sha1 = "4ef1c538614e3ec30cb6383b9eb0326a5c3a9763" +deps = ["Compiler", "JuliaInterpreter"] +git-tree-sha1 = "b882a7dd7ef37643066ae8f9380beea8fdd89cae" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "3.3.0" +version = "3.4.2" [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] @@ -75,13 +85,18 @@ version = "1.3.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.0+0" +version = "3.5.1+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.8.1" +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.45.0+0" + [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -105,7 +120,7 @@ version = "1.3.1" [[deps.Revise]] deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] -git-tree-sha1 = "1d03585ab1bb9a6604094d0e521034df95f92cf2" +git-tree-sha1 = "82dc140c7f52e4daeeec3675a411d48167a85a87" repo-rev = "master" repo-url = "https://github.com/timholy/Revise.jl.git" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" From a39797a4fb5460cb96649dbd5dccadb903323761 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Thu, 24 Jul 2025 01:42:06 -0400 Subject: [PATCH 563/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=2038d2b366a=20to=20542ca0caf=20(#59083)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 | 1 - .../Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 | 1 - .../Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 | 1 + .../Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 create mode 100644 deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 diff --git a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 deleted file mode 100644 index bf0f78e3cacc7..0000000000000 --- a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -36e6a30b59c45236fbf4a7dcf3c410ee diff --git a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 b/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 deleted file mode 100644 index 3aaa6e7d8231b..0000000000000 --- a/deps/checksums/Pkg-38d2b366a7fb8062f990c9897b1441b27bf63715.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -018720df5f382678c9c81df189d60d7402c69dc694414420063d20406a1f38c5a31b66fb4177e73dab554629bcea7669cbb65d6b3c2d8035294c57cb3159e2ef diff --git a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 new file mode 100644 index 0000000000000..38776e9555f56 --- /dev/null +++ b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 @@ -0,0 +1 @@ +94cd000dbe7a5741819b50867201b593 diff --git a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 new file mode 100644 index 0000000000000..6aa9b0203b5ca --- /dev/null +++ b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 @@ -0,0 +1 @@ +e9d1dbcf468af53a7f0fab4c4fbae7222a0111e8bf20bea55307d8f5793959385ba11fbd400a0134047763d689e5f8ead7ff45a6002d7ea2a5850463a6bf1abd diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 0cb8d902db7a6..7cef3404cee52 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 38d2b366a7fb8062f990c9897b1441b27bf63715 +PKG_SHA1 = 542ca0cafc506188995a02c0c9d01bf66616a8f7 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From cc81a0b65466b7529b8c495694a623efa42c34b8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 24 Jul 2025 09:03:00 -0400 Subject: [PATCH 564/662] Do not needlessly disable CPU features. (#59080) On QEMU's RISC-V cpu, LLVM's `getHostCPUFeatures` reports: ``` +zksed,+zkne,+zksh,+zfh,+zfhmin,+zacas,+v,+f,+c,+zvknha,+a,+zfa,+ztso,+zicond,+zihintntl,+zvbb,+zvksh,+zvkg,+zbkb,+zvkned,+zvbc,+zbb,+zvfhmin,+zbkc,+d,+i,+zknh,+zicboz,+zbs,+zvksed,+zbc,+zba,+zvknhb,+zknd,+zvkt,+zbkx,+zkt,+zvfh,+zvkb,+m ``` We change that to: ``` +zksed,+zkne,+zksh,+zfh,+zfhmin,+zacas,+v,+f,+c,+zvknha,+a,+zfa,+ztso,+zicond,+zihintntl,+zvbb,+zvksh,+zvkg,+zbkb,+zvkned,+zvbc,+zbb,+zvfhmin,+zbkc,+d,+i,+zknh,+zicboz,+zbs,+zvksed,+zbc,+zba,+zvknhb,+zknd,+zvkt,+zbkx,+zkt,+zvfh,+zvkb,+m,-zcmop,-zca,-zcd,-zcb,-zve64d,-zve64x,-zve64f,-zawrs,-zve32x,-zimop,-zihintpause,-zcf,-zve32f ``` i.e. we add `-zcmop,-zca,-zcd,-zcb,-zve64d,-zve64x,-zve64f,-zawrs,-zve32x,-zimop,-zihintpause,-zcf,-zve32f`, disabling stuff `zve*` after first enabling `v` (which includes `zvl*b`). That's not valid: ``` LLVM ERROR: 'zvl*b' requires 'v' or 'zve*' extension to also be specified ``` ... so disable this post-processing of LLVM feature sets and trust what it spits out. AFAICT this only matters for the fallback path of `processor.cpp`, so shouldn't impact most users. --- src/processor.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/processor.cpp b/src/processor.cpp index 52c8f51741405..443b86531b3a1 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -992,19 +992,6 @@ static std::string jl_get_cpu_features_llvm(void) attr.append(ele.getKey().str()); } } - // Explicitly disabled features need to be added at the end so that - // they are not re-enabled by other features that implies them by default. - for (auto &ele: HostFeatures) { - if (!ele.getValue()) { - if (!attr.empty()) { - attr.append(",-"); - } - else { - attr.append("-"); - } - attr.append(ele.getKey().str()); - } - } return attr; } From 095e75304306484cb189249c3d21cf376afb9f6f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 24 Jul 2025 11:54:35 -0400 Subject: [PATCH 565/662] build: Also pass -fno-strict-aliasing for C++ (#59066) As diagnosed by Andrew Pinski (https://github.com/JuliaLang/julia/issues/58466#issuecomment-3105141193), we are not respecting strict aliasing currently. We turn this off for C, but the flag appears to be missing for C++. Looks like it's been that way ever since that flag was first added to our build system (#484). We should probably consider running TypeSanitizer over our code base to see if we can make our code correct under strict aliasing as compilers are increasingly taking advantage of it. Fixes #58466 --- Make.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 8701fcdfcfd7c..4a2e75c40bf3f 100644 --- a/Make.inc +++ b/Make.inc @@ -562,7 +562,7 @@ JCPPFLAGS_COMMON := -fasynchronous-unwind-tables JCPPFLAGS_CLANG := $(JCPPFLAGS_COMMON) -mllvm -enable-tail-merge=0 JCPPFLAGS_GCC := $(JCPPFLAGS_COMMON) -fno-tree-tail-merge -JCXXFLAGS_COMMON := -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security +JCXXFLAGS_COMMON := -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security -fno-strict-aliasing JCXXFLAGS_CLANG := $(JCXXFLAGS_COMMON) -pedantic JCXXFLAGS_GCC := $(JCXXFLAGS_COMMON) -fno-gnu-unique From 18976d8d61da368c7365e1ff725aceb438ce563f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4cker?= Date: Thu, 24 Jul 2025 21:15:36 +0200 Subject: [PATCH 566/662] Fix typo in `include`'s docstring (#59055) --- base/sysimg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index 7e205ca955409..fd71544c205cc 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -35,7 +35,7 @@ Use [`Base.include`](@ref) to evaluate a file into another module. at top-level and inserts an implicit `@Core.latestworld` to make any include'd definitions visible to subsequent code. Note however that this recognition is *syntactic*. I.e. assigning `const myinclude = include` may require - and explicit `@Core.latestworld` call after `myinclude`. + an explicit `@Core.latestworld` call after `myinclude`. !!! compat "Julia 1.5" Julia 1.5 is required for passing the `mapexpr` argument. From 8b86693d737f645fac7dd3cc56952365082c0bcf Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 24 Jul 2025 15:59:45 -0400 Subject: [PATCH 567/662] results.json: Fix repo paths so links to github work (#59090) --- test/buildkitetestjson.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index ac23fa8a932b2..f0771e9c005d0 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -99,10 +99,9 @@ function generalize_file_paths(path::AbstractString) string(bindir_dir, pathsep) => "" ) @static if Sys.iswindows() - return replace(path, "\\" => "/") - else - return path + path = replace(path, "\\" => "/") end + return replace(path, "share/julia/" => "") end end From 1cab9a7eda8b9a35b2e4dbff5492212a9575e265 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 24 Jul 2025 17:37:03 -0400 Subject: [PATCH 568/662] Update RISC-V building docs. (#59088) We have pre-built binaries for RISC-V now. --- doc/src/devdocs/build/riscv.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/doc/src/devdocs/build/riscv.md b/doc/src/devdocs/build/riscv.md index 7c0e7ab29d9f8..51939c25e41a1 100644 --- a/doc/src/devdocs/build/riscv.md +++ b/doc/src/devdocs/build/riscv.md @@ -11,17 +11,9 @@ including the output from `cat /proc/cpuinfo`. ## Compiling Julia -For now, Julia will need to be compiled entirely from source, i.e., including -all of its dependencies. This can be accomplished with the following -`Make.user`: - -```make -USE_BINARYBUILDER := 0 -``` - -Additionally, it is required to indicate what architecture, and optionally which -CPU to build for. This can be done by setting the `MARCH` and `MCPU` variables -in `Make.user` +To compilie Julia for RISC-V, you need to manually indicate what architecture, and +optionally which CPU to build for. This can be done by setting the `MARCH` and `MCPU` +variables in `Make.user` The `MARCH` variable needs to be set to a RISC-V ISA string, which can be found by looking at the documentation of your device, or by inspecting `/proc/cpuinfo`. Only From 99eb121068a8952de810fbd9d2aa45e12f8b445e Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 24 Jul 2025 21:22:24 -0400 Subject: [PATCH 569/662] Test: improve type stabilities (#59082) Also simplifies code a bit, by removing unnecessary branches. --- stdlib/Test/src/Test.jl | 59 ++++++++++++++++++------------------ stdlib/Test/test/runtests.jl | 2 +- test/runtests.jl | 4 +-- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index b9129707def46..09594e5be0025 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -217,14 +217,19 @@ struct Error <: Result context::Union{Nothing, String} source::LineNumberNode - function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode, context::Union{Nothing, String}=nothing) - if test_type === :test_error - bt = scrub_exc_stack(bt, nothing, extract_file(source)) - end - if test_type === :test_error || test_type === :nontest_error - bt_str = try # try the latest world for this, since we might have eval'd new code for show + function Error(test_type::Symbol, orig_expr, value, excs::Union{Base.ExceptionStack,Nothing}, + source::LineNumberNode, context::Union{Nothing, String}=nothing) + @nospecialize orig_expr value + bt_str = "" + if !isnothing(excs) + if test_type === :test_error + excs = scrub_exc_stack(excs, nothing, extract_file(source)) + end + if test_type === :test_error || test_type === :nontest_error + bt_str = try + # try the latest world for this, since we might have eval'd new code for show # Apply REPL backtrace scrubbing to hide REPL internals, similar to how REPL.jl handles it - Base.invokelatest(sprint, Base.show_exception_stack, Base.scrub_repl_backtrace(bt); context=stdout) + Base.invokelatest(sprint, Base.show_exception_stack, Base.scrub_repl_backtrace(excs); context=stdout) catch ex "#=ERROR showing exception stack=# " * try @@ -233,8 +238,7 @@ struct Error <: Result "of type " * string(typeof(ex)) end end - else - bt_str = "" + end end value = try # try the latest world for this, since we might have eval'd new code for show Base.invokelatest(sprint, show, value, context = :limit => true) @@ -346,7 +350,7 @@ end struct Threw <: ExecutionResult exception - backtrace::Union{Nothing,Vector{Any}} + current_exceptions::Base.ExceptionStack source::LineNumberNode end @@ -746,7 +750,7 @@ end # An internal function, called by the code generated by the @test # macro to actually perform the evaluation and manage the result. -function do_test(result::ExecutionResult, orig_expr) +function do_test(result::ExecutionResult, @nospecialize orig_expr) # get_testset() returns the most recently added test set # We then call record() with this test set and the test result if isa(result, Returned) @@ -768,13 +772,13 @@ function do_test(result::ExecutionResult, orig_expr) # The predicate couldn't be evaluated without throwing an # exception, so that is an Error and not a Fail @assert isa(result, Threw) - testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source, nothing) + testres = Error(:test_error, orig_expr, result.exception, result.current_exceptions, result.source, nothing) end isa(testres, Pass) || trigger_test_failure_break(result) record(get_testset(), testres) end -function do_broken_test(result::ExecutionResult, orig_expr) +function do_broken_test(result::ExecutionResult, @nospecialize orig_expr) testres = Broken(:test, orig_expr) # Assume the test is broken and only change if the result is true if isa(result, Returned) @@ -860,7 +864,7 @@ end # An internal function, called by the code generated by @test_throws # to evaluate and catch the thrown exception - if it exists -function do_test_throws(result::ExecutionResult, orig_expr, extype) +function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), extype) if isa(result, Threw) # Check that the right type of exception was thrown success = false @@ -908,20 +912,17 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) if success testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only) else - if result.backtrace !== nothing - bt = scrub_exc_stack(result.backtrace, nothing, extract_file(result.source)) - bt_str = try # try the latest world for this, since we might have eval'd new code for show - Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) - catch ex - "#=ERROR showing exception stack=# " * - try - sprint(Base.showerror, ex, catch_backtrace(); context=stdout) - catch - "of type " * string(typeof(ex)) - end - end - else - bt_str = nothing + excs = result.current_exceptions + bt = scrub_exc_stack(excs, nothing, extract_file(result.source)) + bt_str = try # try the latest world for this, since we might have eval'd new code for show + Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) + catch ex + "#=ERROR showing exception stack=# " * + try + sprint(Base.showerror, ex, catch_backtrace(); context=stdout) + catch + "of type " * string(typeof(ex)) + end end testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, message_only, bt_str) end @@ -1073,7 +1074,7 @@ end A simple fallback test set that throws immediately on a failure. """ struct FallbackTestSet <: AbstractTestSet end -fallback_testset = FallbackTestSet() +const fallback_testset = FallbackTestSet() struct FallbackTestSetException <: Exception msg::String diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index a5d2ca831d673..0d6486a26e33e 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -390,7 +390,7 @@ let retval_tests = @testset NoThrowTestSet begin ts = Test.DefaultTestSet("Mock for testing retval of record(::DefaultTestSet, ::T <: Result) methods") pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock")) @test Test.record(ts, pass_mock) isa Test.Pass - error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock"), nothing) + error_mock = Test.Error(:test, 1, 2, nothing, LineNumberNode(0, "An Error Mock"), nothing) @test Test.record(ts, error_mock; print_result=false) isa Test.Error fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false) @test Test.record(ts, fail_mock; print_result=false) isa Test.Fail diff --git a/test/runtests.jl b/test/runtests.jl index 00653a55e4a28..0695139844ba3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -414,7 +414,7 @@ cd(@__DIR__) do # deserialization errors or something similar. Record this testset as Errored. fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1), nothing)) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(Any[(resp, [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() @@ -423,7 +423,7 @@ cd(@__DIR__) do for test in all_tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1), nothing)) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack(Any[("skipped", [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() From 16a2bf0a3b106b03dda23b8c9478aab90ffda5e1 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 25 Jul 2025 09:40:41 -0400 Subject: [PATCH 570/662] LibCURL_jll: New version 8.15.0 (#59057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that CURL 8.15.0 does not support using Secure Transport on MacOS any more. This PR thus switches CURL to using OpenSSL on MacOS. --------- Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com> --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/checksums/curl | 76 +++++++++---------- deps/curl.mk | 5 +- deps/curl.version | 2 +- stdlib/Downloads.version | 2 +- stdlib/LibCURL_jll/Project.toml | 2 +- stdlib/LibCURL_jll/src/LibCURL_jll.jl | 6 +- test/stdlib_dependencies.jl | 1 + 11 files changed, 47 insertions(+), 51 deletions(-) create mode 100644 deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 create mode 100644 deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 delete mode 100644 deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 delete mode 100644 deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 new file mode 100644 index 0000000000000..ed84729b00910 --- /dev/null +++ b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 @@ -0,0 +1 @@ +ac2209576b09a9c7a4da02a058e9ec87 diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 new file mode 100644 index 0000000000000..aa81c3a943f3f --- /dev/null +++ b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 @@ -0,0 +1 @@ +0636b4e7f17d8747408949b77470f5f7f1b4c4b618dc905e20d84d0a5ca4713834e8a4d6a10bcccf7a28982edefe03eeff11bb0aca7d8cb49cb230b7e1ddc06f diff --git a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 b/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 deleted file mode 100644 index 09301a8f7a65e..0000000000000 --- a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ad1074d6518982b6e1a5ecb6537aece1 diff --git a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 b/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 deleted file mode 100644 index 5ef1b996bb785..0000000000000 --- a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7fc7ec01a354d0c4afd6d8c7ad5423afae2d128683f1e2006b0a37b6f34e6b49ad2ed6286c7fc0b5d74392fdbad00e12cc8871c55b28def078d7f962313840a7 diff --git a/deps/checksums/curl b/deps/checksums/curl index 04938f1bfd37e..3e947ff9d8070 100644 --- a/deps/checksums/curl +++ b/deps/checksums/curl @@ -1,38 +1,38 @@ -LibCURL.v8.14.1+1.aarch64-apple-darwin.tar.gz/md5/77465587a033e919ce347c9df3909b23 -LibCURL.v8.14.1+1.aarch64-apple-darwin.tar.gz/sha512/04a6bb04f6fc190423349787a1b345923f52b140099a3c62fdd077b3145a22a20ea06808c488712d686bc6d51fe86caea3b13cd9acde36822fec7249905ff16d -LibCURL.v8.14.1+1.aarch64-linux-gnu.tar.gz/md5/a5b78f4d06821de8c57abba8fbfc1c23 -LibCURL.v8.14.1+1.aarch64-linux-gnu.tar.gz/sha512/a09f94bd8556d63b7850b8ac29f8272c7437cc32c8440200d7044a4eb89bba8752d74ad26d3da0433ab07565dfefb1edb058ef13a1949546994321ea88e9e515 -LibCURL.v8.14.1+1.aarch64-linux-musl.tar.gz/md5/09e1b11301296cddb9039e0247ba0cde -LibCURL.v8.14.1+1.aarch64-linux-musl.tar.gz/sha512/380159eec2b638a190d284a02fdb6c60e925f4f0268ea4e2ae7272fffe19bfff5de93d9a71f3830147ece69417bbd2f7faca229cb1a4b4df4f1828e3f8a0ff53 -LibCURL.v8.14.1+1.aarch64-unknown-freebsd.tar.gz/md5/f872ba79808c0b24dfee579bf2fc7229 -LibCURL.v8.14.1+1.aarch64-unknown-freebsd.tar.gz/sha512/aa6ac50c0c3eaed768f228b2a4d320d44afe1bd34ee81273f976ca4027ccbafe996c2e349eab87718e962da0a18afa8b3c5d297e913d46ba09a8d68bba7dabf1 -LibCURL.v8.14.1+1.armv6l-linux-gnueabihf.tar.gz/md5/4e28def7a2d3bffc8d2adddbe670dd04 -LibCURL.v8.14.1+1.armv6l-linux-gnueabihf.tar.gz/sha512/444df0655e0d85cd35e20013e79787af17138dc16a292bf4557ebbd049c8f855449b9aa78eb45b0a00255e8320a4ad2b44a08c6161b7155467c9da95e8bb0a7d -LibCURL.v8.14.1+1.armv6l-linux-musleabihf.tar.gz/md5/4c037de41ff577fe217792e5d7c7f9a0 -LibCURL.v8.14.1+1.armv6l-linux-musleabihf.tar.gz/sha512/65122b75188cc7dfb8dd97c27398f614d2e3149c73e95bd9c60cdad2eb1c95a3c3dd7558a02a5d24b0e129c8c3a315efe45dd49f5f219a525dbeea0b4ab9b93f -LibCURL.v8.14.1+1.armv7l-linux-gnueabihf.tar.gz/md5/f7dedafd8eb09c7bf9cbac4c0adae37a -LibCURL.v8.14.1+1.armv7l-linux-gnueabihf.tar.gz/sha512/7c8187ce79de364488b7d8606638f7cfc1094f9a1d5a26dc71dc5a1a6135834641fe459b08f71999556d5ff2dfd1820ff3f1371c2b5fa94b8ea5985744044527 -LibCURL.v8.14.1+1.armv7l-linux-musleabihf.tar.gz/md5/6ede02ecc42e088e60d5a6c8a0181fd9 -LibCURL.v8.14.1+1.armv7l-linux-musleabihf.tar.gz/sha512/e3ea55dd8035506b5c2cc11ea2dbf30533f3fdef017decdca35fafe19a09f9b945f821f722fa60651fb083204a8d0aab47bb2024544b4d81b5b647b7bf260e01 -LibCURL.v8.14.1+1.i686-linux-gnu.tar.gz/md5/b823b709fd3a26a2861962b8f538718b -LibCURL.v8.14.1+1.i686-linux-gnu.tar.gz/sha512/4d94062910ed94cdf013ba3ce546231e733a780078f106a42937e26400de65634991df440e27865368b4f1f395743629cfd88dd62dd27b90654f4766c0a5f468 -LibCURL.v8.14.1+1.i686-linux-musl.tar.gz/md5/a80641db6cc9731342c08e85835481cd -LibCURL.v8.14.1+1.i686-linux-musl.tar.gz/sha512/9983e83469b38f17645b4cfdf9c0005684fb35a29bf97b1be8c2a34bdeb1115cf029353811eef7a4c96c2c7da3e0c4e1ba8fa7c589b8452675232544f591cf7d -LibCURL.v8.14.1+1.i686-w64-mingw32.tar.gz/md5/f1703e82d19e3a9c9fe5068532a2f98e -LibCURL.v8.14.1+1.i686-w64-mingw32.tar.gz/sha512/03a98243ea31f04a05ad94b0ce378830ee486d313bc33e652f44fd11fa812d3bff31556484c66a1fe467815affb5f191e7e8212996ac0a7b89b46b7743db58e1 -LibCURL.v8.14.1+1.powerpc64le-linux-gnu.tar.gz/md5/324982f4b780aba414384c369580ec85 -LibCURL.v8.14.1+1.powerpc64le-linux-gnu.tar.gz/sha512/b6c0d05ce457e8d1c97098891982921c4282a1a9f2984ed82735fc4251cf12d230826d7e96bcbbaf073ae6cfa9357adfd51341abb937db3ffca57156eac2a0eb -LibCURL.v8.14.1+1.riscv64-linux-gnu.tar.gz/md5/363f5480fd9d59c232ae30e799e96090 -LibCURL.v8.14.1+1.riscv64-linux-gnu.tar.gz/sha512/3a821895af3d6ed2363a39057f691bec2f139c196fe430c8d4f6b6404ecdc4d5d563e02c451dea8304fe403b7c1b3b2fe5fb8a7376db594037e31f29bb07cfb6 -LibCURL.v8.14.1+1.x86_64-apple-darwin.tar.gz/md5/e369db8d77ea5234de6f2efad75c856a -LibCURL.v8.14.1+1.x86_64-apple-darwin.tar.gz/sha512/5de40ee8f893d36a91d547c70ecbb396e4a3bef63eaaa7dab37a9374ce20651a3b51b2a415d7803183a76f5828bb39e3df428e6f4bd913e735226bd7732e0f23 -LibCURL.v8.14.1+1.x86_64-linux-gnu.tar.gz/md5/7af620f052ca833f495f8a770685877d -LibCURL.v8.14.1+1.x86_64-linux-gnu.tar.gz/sha512/443dccf17f1946e12b53a50b1a65494b1101a7d61853f9357df49891c36e49eae6f309a49f860534bd359d1e258c43fdd3e1d1e2e81a00084ddd6a6a47ae11f0 -LibCURL.v8.14.1+1.x86_64-linux-musl.tar.gz/md5/84bd751a151c3eb27d64a49f3d6a86ca -LibCURL.v8.14.1+1.x86_64-linux-musl.tar.gz/sha512/2c2b52bb5eaf980acd1ae0db7b8b2c0b7f88dbd9f4e2dde7e2f06052492479fafe7ac17e54e255dccf7f9ef5b4ef095ad54a4ae614a670d2efad713cac83bbb5 -LibCURL.v8.14.1+1.x86_64-unknown-freebsd.tar.gz/md5/0575d2d4b334bc5bae99c04b6534cff2 -LibCURL.v8.14.1+1.x86_64-unknown-freebsd.tar.gz/sha512/3aed29b4a762d300247a9a5a6b90393c8d8d41c895a334060a296983c8d8fdb47a2c4f59f8f3467aacb88f2034b0969b5bfabace97a619e26f5194d9f8baba4d -LibCURL.v8.14.1+1.x86_64-w64-mingw32.tar.gz/md5/03391a09da53b96531f95f764ebf5a49 -LibCURL.v8.14.1+1.x86_64-w64-mingw32.tar.gz/sha512/231f76cdf61cda36e4aa94bb8d4a5cb2607bde11cad172e516dd8c8689f36628eb01629c36bfc89be4c578866723d7e42257027ebb5be8dc16edc3d242deb86d -curl-8.14.1.tar.bz2/md5/bf683fcf55bed4c5bf66ca57ab99a96b -curl-8.14.1.tar.bz2/sha512/95fd0fffb354e60bc2a7251ce53122dc41d5b0aae00863654cae65f2e9698f02ff67d60b367f1a9dd556bfa77e3bac1e5e985efe00aaecfb99e1ea68148f3344 +LibCURL.v8.15.0+0.aarch64-apple-darwin.tar.gz/md5/47db9bf4549442b466ae465b48544f46 +LibCURL.v8.15.0+0.aarch64-apple-darwin.tar.gz/sha512/a4819a6a94958cfeb0b035b87cfe6cb40ac3c0f015b7b52b5314ec3f8a129a9aa96ccae04fe3774318bc1a9316299c3fbe1a47b2e2e5ce37a38d9a8e01d6f5e5 +LibCURL.v8.15.0+0.aarch64-linux-gnu.tar.gz/md5/7e04291557b99a53f8b8fca1235862b3 +LibCURL.v8.15.0+0.aarch64-linux-gnu.tar.gz/sha512/399b9a538dc54647b5ca1ad182e5c1c4da8ba0f3217fe8d01737bf0a689a67ed0be185ebbd98d60b2739900df664feae6cedb90643c2e53dc7c12ec5a5a8b57d +LibCURL.v8.15.0+0.aarch64-linux-musl.tar.gz/md5/63f3e0171c8f56592a3c810136154ccd +LibCURL.v8.15.0+0.aarch64-linux-musl.tar.gz/sha512/03b67d73bd41597c6080155e5a0eb58d8cb559d1325c2954aa92c54905dd6f425ba46dd16c5411b16ac683c041ed685893954c4ab431d794bd3f929c11de4ae7 +LibCURL.v8.15.0+0.aarch64-unknown-freebsd.tar.gz/md5/31f249d293bbea3a0669197e6559d467 +LibCURL.v8.15.0+0.aarch64-unknown-freebsd.tar.gz/sha512/bb047b609e69926d3837dfe39d68057c55c7380427a3807ccc71e631530d75f64e9edd0a4ef5ab30b44e3d763d2798e9f8c4727fb565674de228cf95ebccd922 +LibCURL.v8.15.0+0.armv6l-linux-gnueabihf.tar.gz/md5/462094d229be838090eb6d3353bb3907 +LibCURL.v8.15.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/3bac7d0b0a065e5793d466823eb2a62d8256d41291c9bf6926bafc434eaec51a72ca289b2803b0b6fc19c4d0e6776205e55b235701496e84d56abdd4da7be8a2 +LibCURL.v8.15.0+0.armv6l-linux-musleabihf.tar.gz/md5/3f95a180a1fe427b642f05814029f9d0 +LibCURL.v8.15.0+0.armv6l-linux-musleabihf.tar.gz/sha512/8a4483486dd355d61f0c6f60d0a1b38d9942dc126b36368adcb9499daf2a134087fb54ffbd2930adf3fe1cc39497f2f7e1f6304f48f7540cf05ebeb792d0ec1f +LibCURL.v8.15.0+0.armv7l-linux-gnueabihf.tar.gz/md5/d033e4eba1455ef17b5d86d9911d2d32 +LibCURL.v8.15.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/91e31ec413ee15574ee7e4258eebd6963d6679ea8b41c43e0f02f18521d52f5a9f258c46f8df27163019dd89a2598b7c3bcd3d2aa2bd3fd3a545de9060ef7dd1 +LibCURL.v8.15.0+0.armv7l-linux-musleabihf.tar.gz/md5/3ba22fe71ba61b92e297f64593d69387 +LibCURL.v8.15.0+0.armv7l-linux-musleabihf.tar.gz/sha512/a87bc28cac150d76c281b1baa6f79da8140bf849595dcca684fcd64299fc801c3c2dcd733b264c5a809fbba95dbe8dfac0c30ecd7c5d0bb77fee7d3da91d6816 +LibCURL.v8.15.0+0.i686-linux-gnu.tar.gz/md5/67612302faac6b9822271d01c1959d78 +LibCURL.v8.15.0+0.i686-linux-gnu.tar.gz/sha512/0f47c1f8307054eca308650f13dfee62a6eaae70f5385f460339a821cb73f97eb56ac5f685cb9e14d51e109a510508e6e0e7b6f31319e57413396473ac19a1c6 +LibCURL.v8.15.0+0.i686-linux-musl.tar.gz/md5/b14f7012d2aa9ed353387c3b357255c6 +LibCURL.v8.15.0+0.i686-linux-musl.tar.gz/sha512/177b4581c4717854420073731398dc8bb2a3370a572fd6b61e545a8e852dcb9ce76f4c26bf28e1250cece1dcbe34e53d0edd2c421c6251c117e2f3255b8945a6 +LibCURL.v8.15.0+0.i686-w64-mingw32.tar.gz/md5/2afb5832ab12120e755505916cad0e95 +LibCURL.v8.15.0+0.i686-w64-mingw32.tar.gz/sha512/4e78b5d0a8ec3e1bc7131c66fc5f041090f5fb64fe6b85224fea1e3ebcbdbf56604492a059ae311281052da7ff67c066910c6c74fb7d19b3e46b4d3cdb7163b7 +LibCURL.v8.15.0+0.powerpc64le-linux-gnu.tar.gz/md5/9d5bd94798a63a14ba945949b054195f +LibCURL.v8.15.0+0.powerpc64le-linux-gnu.tar.gz/sha512/eb4c6a555b1b6dc39a890e819803d4d5015e9ad8a841e21b484fa7442330ea0b3c2da07fcc1afb09faf115fc4818eaf2f328f92fc07400a523aca45828b71e4b +LibCURL.v8.15.0+0.riscv64-linux-gnu.tar.gz/md5/68b576c1f2086490a3d7997ce8627a32 +LibCURL.v8.15.0+0.riscv64-linux-gnu.tar.gz/sha512/e7dc57b985a6f6289d7a56a1cae49522cc623831167ec317bcdb66b578f1d967befb5d849d2c24dc49e183524a6f3b72b6c688e3500c5cb8a3f00ce748728fc6 +LibCURL.v8.15.0+0.x86_64-apple-darwin.tar.gz/md5/75320e06cd2370343684279f744b694a +LibCURL.v8.15.0+0.x86_64-apple-darwin.tar.gz/sha512/198f520a860337aaf367db970083db8c7a3742b3f37f5594e4dc86669b319cc4dc4bdfc9e875c3e4ca9f93aae24705b7c47711bac66daa629ed0d386d352b053 +LibCURL.v8.15.0+0.x86_64-linux-gnu.tar.gz/md5/3f159aada40e7f4489b179a570d36c9a +LibCURL.v8.15.0+0.x86_64-linux-gnu.tar.gz/sha512/42e646400aca6d0b233ab8feb3b32c3a9e89ca4b27fb0c82e0c52b4b008e9d9aad6640988f32397d705cc52c36ffa4379f6ba4c015ee24b6e2bd750235c8f10c +LibCURL.v8.15.0+0.x86_64-linux-musl.tar.gz/md5/d4e35eee8af7d9e640f9f2616abf1ebf +LibCURL.v8.15.0+0.x86_64-linux-musl.tar.gz/sha512/16a9ee5afe025db931e7209fbc711494ff7795a9463cf81a307e7ea74cf53b64a1c5413d388e328a372e4480255b4b1324448e17f51db141adcb2716aa00d698 +LibCURL.v8.15.0+0.x86_64-unknown-freebsd.tar.gz/md5/94e871b6200ab1768112aa3a7195f31e +LibCURL.v8.15.0+0.x86_64-unknown-freebsd.tar.gz/sha512/84474b36d21ab1d90331850b914908add5afd239f9111a48dec9b6e7ff770e9a730c7cad6d322f0cdd3567822fb157e1470716e181dd29b594a4e471b8fd6c54 +LibCURL.v8.15.0+0.x86_64-w64-mingw32.tar.gz/md5/99c54cbc43fc719281b48bb3137e9364 +LibCURL.v8.15.0+0.x86_64-w64-mingw32.tar.gz/sha512/d817951ec0ba4513e50c2bf2159ccefb6b38fa74478b84b8f006b6a47897c610fe6c9f1c9db0533595c243f8b84d40456aa9fd1aa3afb22ea272dced5b6e237c +curl-8.15.0.tar.bz2/md5/8b475c3eec74c5e78a9fb45a902dae76 +curl-8.15.0.tar.bz2/sha512/797fc9af599de88ceb36c8bc284d3f1a2a1ca0703c7bdd377d67ce6da4317ca673ba4a946c61f3bd5a66febee37b5aa88826f26fbbb398f5e20630769a0de033 diff --git a/deps/curl.mk b/deps/curl.mk index fa106ac2259fc..5de8c1f766072 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -54,12 +54,9 @@ CURL_CONFIGURE_FLAGS += \ # We use different TLS libraries on different platforms. # On Windows, we use schannel -# On MacOS, we use SecureTransport -# On Linux, we use OpenSSL +# On other platforms, we use OpenSSL ifeq ($(OS), WINNT) CURL_TLS_CONFIGURE_FLAGS := --with-schannel -else ifeq ($(OS), Darwin) -CURL_TLS_CONFIGURE_FLAGS := --with-secure-transport else CURL_TLS_CONFIGURE_FLAGS := --with-openssl endif diff --git a/deps/curl.version b/deps/curl.version index e7a22071c7566..ecd8089d9fd1f 100644 --- a/deps/curl.version +++ b/deps/curl.version @@ -3,4 +3,4 @@ CURL_JLL_NAME := LibCURL ## source build -CURL_VER := 8.14.1 +CURL_VER := 8.15.0 diff --git a/stdlib/Downloads.version b/stdlib/Downloads.version index 16675209860f4..4829d63b31668 100644 --- a/stdlib/Downloads.version +++ b/stdlib/Downloads.version @@ -1,4 +1,4 @@ DOWNLOADS_BRANCH = master -DOWNLOADS_SHA1 = 1199e9039dcce6072601c3f29cea8da4a0acbff6 +DOWNLOADS_SHA1 = 06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3 DOWNLOADS_GIT_URL := https://github.com/JuliaLang/Downloads.jl.git DOWNLOADS_TAR_URL = https://api.github.com/repos/JuliaLang/Downloads.jl/tarball/$1 diff --git a/stdlib/LibCURL_jll/Project.toml b/stdlib/LibCURL_jll/Project.toml index 0debbc077d594..3a2025ea10834 100644 --- a/stdlib/LibCURL_jll/Project.toml +++ b/stdlib/LibCURL_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibCURL_jll" uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.14.1+1" +version = "8.15.0+0" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index 43ae2dec24aab..fb091caee4d50 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -4,7 +4,7 @@ baremodule LibCURL_jll using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll -if !(Sys.iswindows() || Sys.isapple()) +if !Sys.iswindows() using OpenSSL_jll end if Sys.iswindows() && Sys.WORD_SIZE == 32 @@ -37,10 +37,8 @@ const libcurl = LazyLibrary( else LazyLibrary[libz, libnghttp2, libssh2] end - elseif Sys.islinux() || Sys.isfreebsd() + else LazyLibrary[libz, libnghttp2, libssh2, libssl, libcrypto] - elseif Sys.isapple() - LazyLibrary[libz, libnghttp2, libssh2] end ) diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl index 0af7d02f6e6a3..04e5a65a61b23 100644 --- a/test/stdlib_dependencies.jl +++ b/test/stdlib_dependencies.jl @@ -178,6 +178,7 @@ try "bcrypt", "winhttp", "secur32", + "iphlpapi", ] return any(syslib -> lowercase(lib) == syslib, system_libs) end From 0d9e0ef0feeab2c620b4f482e63ddc56e21229ea Mon Sep 17 00:00:00 2001 From: Zentrik Date: Fri, 25 Jul 2025 18:23:38 +0100 Subject: [PATCH 571/662] Switch RISC-V to large model on LLVM 20 (#57865) Co-authored-by: Tim Besard --- src/jitlayers.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c5588be794201..460ba016c95be 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1385,11 +1385,13 @@ namespace { #endif if (TheTriple.isAArch64()) codemodel = CodeModel::Small; +#if JL_LLVM_VERSION < 200000 else if (TheTriple.isRISCV()) { - // RISC-V will support large code model in LLVM 21 + // RISC-V only supports large code model from LLVM 20 // https://github.com/llvm/llvm-project/pull/70308 codemodel = CodeModel::Medium; } +#endif // Generate simpler code for JIT Reloc::Model relocmodel = Reloc::Static; if (TheTriple.isRISCV()) { From f82635a4c6139a011dee28291c1caba51f0c0f8b Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Sat, 26 Jul 2025 11:29:03 -0400 Subject: [PATCH 572/662] Support complex numbers in eps (#21858) This came up in https://github.com/JuliaMath/IterativeSolvers.jl/pull/113#issuecomment-301273365 . JuliaDiffEq and IterativeSolvers.jl have to make sure that the real-type is pulled out in order for `eps` to work: ```julia eps(real(typeof(b))) ``` This detail can make many algorithms with tolerances that are written generically that would otherwise work with complex numbers error. This PR proposes to do just that trick, so that way `eps(1.0 + 1.0im)` returns machine epsilon for a Float64 (and generally works for `AbstractFloat` of course). --------- Co-authored-by: Steven G. Johnson --- base/complex.jl | 10 ++++++++++ base/float.jl | 7 +++++++ test/complex.jl | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/base/complex.jl b/base/complex.jl index 37fa974cc0c7d..be42e7db1dde4 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -1141,3 +1141,13 @@ function complex(A::AbstractArray{T}) where T end convert(AbstractArray{typeof(complex(zero(T)))}, A) end + +## Promotion to complex ## + +_default_type(T::Type{Complex}) = Complex{Int} + +## Machine epsilon for complex ## + +eps(z::Complex{<:AbstractFloat}) = hypot(eps(real(z)), eps(imag(z))) + +eps(::Type{Complex{T}}) where {T<:AbstractFloat} = sqrt(2*one(T))*eps(T) diff --git a/base/float.jl b/base/float.jl index ba380102f8751..d5bfd044cf3b0 100644 --- a/base/float.jl +++ b/base/float.jl @@ -1064,6 +1064,13 @@ julia> 1.0 + eps() julia> 1.0 + eps()/2 1.0 ``` + +More generally, for any floating-point numeric type, `eps` corresponds to an +upper bound on the distance to the nearest floating-point complex value: if ``\text{fl}(x)`` is the closest +floating-point value to a number ``x`` (e.g. an arbitrary real number), then ``\text{fl}(x)`` +satisfies ``|x - \text{fl}(x)| ≤ \text{eps}(x)/2``, not including overflow cases. +This allows the definition of `eps` to be extended to complex numbers, +for which ``\text{fl}(a + ib) = \text{fl}(a) + i \text{fl}(b)``. """ eps(::Type{<:AbstractFloat}) diff --git a/test/complex.jl b/test/complex.jl index e24e54febc815..10b4bc8aa03d6 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -932,6 +932,13 @@ end end end +@testset "eps" begin + @test eps(1.0+1.0im) === 3.1401849173675503e-16 + @test eps(Complex128) === eps(1.0+1.0im) + @test eps(Complex64) === 1.6858739f-7 + @test eps(Float32(1.0)+Float32(1.0)im) === eps(Complex{Float32}) +end + @testset "cis" begin @test cis(0.0+1.0im) ≈ 0.367879441171442321595523770161460867445811131031767834507836+0.0im @test cis(1.0+0.0im) ≈ 0.54030230586813971740093660744297660373231042061+0.84147098480789650665250232163029899962256306079im From 44dd95a5901ac65d7c65f76e266bba334f8b620c Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Sat, 26 Jul 2025 11:37:36 -0400 Subject: [PATCH 573/662] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20?= =?UTF-8?q?Pkg=20stdlib=20from=20542ca0caf=20to=20d94f8a1d9=20(#59093)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: IanButterworth <1694067+IanButterworth@users.noreply.github.com> --- .../Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 | 1 - .../Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 | 1 - .../Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 | 1 + .../Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 create mode 100644 deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 diff --git a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 deleted file mode 100644 index 38776e9555f56..0000000000000 --- a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -94cd000dbe7a5741819b50867201b593 diff --git a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 b/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 deleted file mode 100644 index 6aa9b0203b5ca..0000000000000 --- a/deps/checksums/Pkg-542ca0cafc506188995a02c0c9d01bf66616a8f7.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e9d1dbcf468af53a7f0fab4c4fbae7222a0111e8bf20bea55307d8f5793959385ba11fbd400a0134047763d689e5f8ead7ff45a6002d7ea2a5850463a6bf1abd diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 new file mode 100644 index 0000000000000..f1a89122167f9 --- /dev/null +++ b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 @@ -0,0 +1 @@ +474392cbf2fe976bca300944fc62f5ab diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 new file mode 100644 index 0000000000000..e3a190082f455 --- /dev/null +++ b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 @@ -0,0 +1 @@ +99c1b57efd80eade5f89f350d244bedc80717809c108056bf28b2f6213dba040bbed2ef4869f97a624efa1ca8474504c087481860758bb366025093a8bec34b5 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 7cef3404cee52..ae88e816ffbc6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 542ca0cafc506188995a02c0c9d01bf66616a8f7 +PKG_SHA1 = d94f8a1d946f90b00b836afc1dedf034604c6627 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From c5ba4e28f2c9c49be01ed7409efedef8627fe96d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 26 Jul 2025 12:38:00 -0400 Subject: [PATCH 574/662] add array element mutex offset in print and gc (#58997) The layout, printing, and gc logic need to correctly offset and align the inset fields to account for the per-element mutex of an atomic array with large elements. Fix #58993 --- base/runtime_internals.jl | 6 +++- src/cgutils.cpp | 13 +++++---- src/codegen.cpp | 19 ++++++------- src/datatype.c | 59 ++++++++++++++++++++++++++------------- src/genericmemory.c | 15 ++++------ src/julia.h | 6 +++- src/rtutils.c | 5 ++++ test/atomics.jl | 24 ++++++++++------ 8 files changed, 92 insertions(+), 55 deletions(-) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index f75824577ebf4..ba224acf897d4 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -565,6 +565,10 @@ struct DataTypeLayout # fielddesc_type : 2; # arrayelem_isboxed : 1; # arrayelem_isunion : 1; + # arrayelem_isatomic : 1; + # arrayelem_islocked : 1; + # isbitsegal : 1; + # padding : 8; end """ @@ -637,7 +641,7 @@ function datatype_isbitsegal(dt::DataType) @_foldable_meta dt.layout == C_NULL && throw(UndefRefError()) flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return (flags & (1<<5)) != 0 + return (flags & (1<<7)) != 0 end """ diff --git a/src/cgutils.cpp b/src/cgutils.cpp index ef36eeb728689..97a58f8aa419c 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -3299,7 +3299,7 @@ static Value *emit_genericmemoryelsize(jl_codectx_t &ctx, Value *v, jl_value_t * if (jl_is_genericmemoryref_type(sty)) sty = (jl_datatype_t*)jl_field_type_concrete(sty, 1); size_t sz = sty->layout->size; - if (sty->layout->flags.arrayelem_isunion) + if (sty->layout->flags.arrayelem_isunion && add_isunion) sz++; auto elsize = ConstantInt::get(ctx.types().T_size, sz); return elsize; @@ -4716,8 +4716,8 @@ static jl_cgval_t emit_memoryref_direct(jl_codectx_t &ctx, const jl_cgval_t &mem } else { data = emit_genericmemoryptr(ctx, boxmem, layout, 0); - Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(typ)); - data = ctx.builder.CreateInBoundsGEP(elty, data, idx0); + idx0 = ctx.builder.CreateMul(idx0, emit_genericmemoryelsize(ctx, boxmem, mem.typ, false), "", true, true); + data = ctx.builder.CreatePtrAdd(data, idx0); } return _emit_memoryref(ctx, boxmem, data, layout, typ); @@ -4820,9 +4820,10 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg setName(ctx.emission_context, ovflw, "memoryref_ovflw"); } #endif - Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(ref.typ)); - newdata = ctx.builder.CreateGEP(elty, data, offset); - setName(ctx.emission_context, newdata, "memoryref_data_offset"); + boffset = ctx.builder.CreateMul(offset, elsz); + setName(ctx.emission_context, boffset, "memoryref_byteoffset"); + newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), data, boffset); + setName(ctx.emission_context, newdata, "memoryref_data_byteoffset"); (void)boffset; // LLVM is very bad at handling GEP with types different from the load if (bc) { BasicBlock *failBB, *endBB; diff --git a/src/codegen.cpp b/src/codegen.cpp index 6ff0a2000fc75..2d9c94baee9b4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3862,8 +3862,8 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; bool isboxed = layout->flags.arrayelem_isboxed; bool isunion = layout->flags.arrayelem_isunion; - bool isatomic = kind == (jl_value_t*)jl_atomic_sym; - bool needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + bool isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; + bool needlock = layout->flags.arrayelem_islocked; size_t elsz = layout->size; size_t al = layout->alignment; if (al > JL_HEAP_ALIGNMENT) @@ -4231,7 +4231,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, size_t al = layout->alignment; if (al > JL_HEAP_ALIGNMENT) al = JL_HEAP_ALIGNMENT; - bool needlock = isatomic && !isboxed && elsz > MAX_ATOMIC_SIZE; + bool needlock = layout->flags.arrayelem_islocked; AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) : get_llvm_atomic_order(order); @@ -4317,7 +4317,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, *ret = jl_cgval_t(); // unreachable return true; } - bool isatomic = kind == (jl_value_t*)jl_atomic_sym; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; if (!isatomic && order != jl_memory_order_notatomic && order != jl_memory_order_unspecified) { emit_atomic_error(ctx, "memoryref_isassigned: non-atomic memory cannot be accessed atomically"); *ret = jl_cgval_t(); // unreachable @@ -4333,13 +4334,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } jl_value_t *boundscheck = argv[3].constant; emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, fname); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; bool isboxed = layout->flags.arrayelem_isboxed; if (isboxed || layout->first_ptr >= 0) { - bool needlock = isatomic && !isboxed && layout->size > MAX_ATOMIC_SIZE; + bool needlock = layout->flags.arrayelem_islocked; AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) : get_llvm_atomic_order(order); @@ -4359,13 +4359,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ctx.builder.SetInsertPoint(passBB); } Value *elem = emit_memoryref_ptr(ctx, ref, layout); - if (needlock) { + if (!isboxed) + elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); + else if (needlock) // n.b. no actual lock acquire needed, as the check itself only needs to load a single pointer and check for null // elem += sizeof(lock); elem = emit_ptrgep(ctx, elem, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); - } - if (!isboxed) - elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); // emit this using the same type as BUILTIN(memoryrefget) // so that LLVM may be able to load-load forward them and fold the result auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; diff --git a/src/datatype.c b/src/datatype.c index 2df2bdb2aaa71..450bf89748474 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -239,8 +239,10 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, flddesc->flags.haspadding = haspadding; flddesc->flags.isbitsegal = isbitsegal; flddesc->flags.fielddesc_type = fielddesc_type; - flddesc->flags.arrayelem_isboxed = arrayelem == 1; - flddesc->flags.arrayelem_isunion = arrayelem == 2; + flddesc->flags.arrayelem_isboxed = (arrayelem & 1) != 0; + flddesc->flags.arrayelem_isunion = (arrayelem & 2) != 0; + flddesc->flags.arrayelem_isatomic = (arrayelem & 4) != 0; + flddesc->flags.arrayelem_islocked = (arrayelem & 8) != 0; flddesc->flags.padding = 0; flddesc->npointers = npointers; flddesc->first_ptr = first_ptr; @@ -537,6 +539,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) uint32_t *pointers = &first_ptr; int needlock = 0; + const jl_datatype_layout_t *el_layout = NULL; if (isunboxed) { elsz = LLT_ALIGN(elsz, al); if (kind == (jl_value_t*)jl_atomic_sym) { @@ -551,12 +554,12 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) else { assert(jl_is_datatype(eltype)); zi = ((jl_datatype_t*)eltype)->zeroinit; - const jl_datatype_layout_t *layout = ((jl_datatype_t*)eltype)->layout; - if (layout->first_ptr >= 0) { - first_ptr = layout->first_ptr; - npointers = layout->npointers; - if (layout->flags.fielddesc_type == 2) { - pointers = (uint32_t*)jl_dt_layout_ptrs(layout); + el_layout = ((jl_datatype_t*)eltype)->layout; + if (el_layout->first_ptr >= 0) { + first_ptr = el_layout->first_ptr; + npointers = el_layout->npointers; + if (el_layout->flags.fielddesc_type == 2 && !needlock) { + pointers = (uint32_t*)jl_dt_layout_ptrs(el_layout); } else { pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); @@ -568,10 +571,22 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) } if (needlock) { assert(al <= JL_SMALL_BYTE_ALIGNMENT); - size_t offset = LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); - elsz += offset; + size_t lock_offset = LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); + elsz += lock_offset; + if (al < sizeof(void*)) { + al = sizeof(void*); + elsz = LLT_ALIGN(elsz, al); + } haspadding = 1; zi = 1; + // Adjust pointer offsets to account for the lock at the beginning + if (first_ptr != -1) { + uint32_t lock_offset_words = lock_offset / sizeof(void*); + first_ptr += lock_offset_words; + for (int j = 0; j < npointers; j++) { + pointers[j] += lock_offset_words; + } + } } } else { @@ -580,13 +595,17 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) zi = 1; } - int arrayelem; + // arrayelem is a bitfield: 1=isboxed, 2=isunion, 4=isatomic, 8=islocked + int arrayelem = 0; if (!isunboxed) - arrayelem = 1; - else if (isunion) - arrayelem = 2; - else - arrayelem = 0; + arrayelem |= 1; // arrayelem_isboxed + if (isunion) + arrayelem |= 2; // arrayelem_isunion + if (kind == (jl_value_t*)jl_atomic_sym) { + arrayelem |= 4; // arrayelem_isatomic + if (needlock) + arrayelem |= 8; // arrayelem_islocked + } assert(!st->layout); st->layout = jl_get_layout(elsz, nfields, npointers, al, haspadding, isbitsegal, arrayelem, NULL, pointers); st->zeroinit = zi; @@ -647,17 +666,17 @@ void jl_compute_field_offsets(jl_datatype_t *st) // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob - static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, -1, 1, { .isbitsegal=1 }}; st->layout = &opaque_byte_layout; return; } else if (st == jl_simplevector_type || st == jl_module_type) { - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), { .isbitsegal=1 }}; st->layout = &opaque_ptr_layout; return; } else { - static const jl_datatype_layout_t singleton_layout = {0, 0, 0, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t singleton_layout = {0, 0, 0, -1, 1, { .isbitsegal=1 }}; st->layout = &singleton_layout; } } @@ -1001,6 +1020,8 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, layout->flags.padding = 0; layout->flags.arrayelem_isboxed = 0; layout->flags.arrayelem_isunion = 0; + layout->flags.arrayelem_isatomic = 0; + layout->flags.arrayelem_islocked = 0; jl_fielddescdyn_t * desc = (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout)); desc->markfunc = markfunc; diff --git a/src/genericmemory.c b/src/genericmemory.c index a81a4a199956c..35ef2b545a32d 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -262,8 +262,8 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *mem, size_t i) { - int isatomic = (jl_tparam0(jl_typetagof(mem)) == (jl_value_t*)jl_atomic_sym); const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout; + int isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; jl_genericmemoryref_t m; m.mem = mem; m.ptr_or_offset = (layout->flags.arrayelem_isunion || layout->size == 0) ? (void*)i : (void*)((char*)mem->ptr + layout->size * i); @@ -342,8 +342,8 @@ static jl_value_t *jl_ptrmemrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, i JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m, int isatomic) { - assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + assert(isatomic == (layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked)); if (layout->flags.arrayelem_isboxed) return jl_ptrmemrefget(m, isatomic); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); @@ -365,7 +365,7 @@ JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m, int isatomic) assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); jl_value_t *r; size_t fsz = jl_datatype_size(eltype); - int needlock = isatomic && fsz > MAX_ATOMIC_SIZE; + int needlock = layout->flags.arrayelem_islocked; if (isatomic && !needlock) { r = jl_atomic_new_bits(eltype, data); } @@ -393,9 +393,6 @@ static int _jl_memoryref_isassigned(jl_genericmemoryref_t m, int isatomic) if (layout->flags.arrayelem_isboxed) { } else if (layout->first_ptr >= 0) { - int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; - if (needlock) - elem = elem + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT) / sizeof(jl_value_t*); elem = &elem[layout->first_ptr]; } else { @@ -411,7 +408,8 @@ JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m, int is JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, int isatomic) { - assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + assert(isatomic == (layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked)); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { JL_GC_PUSH1(&rhs); @@ -419,7 +417,6 @@ JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, j jl_type_error("memoryrefset!", eltype, rhs); JL_GC_POP(); } - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; if (layout->flags.arrayelem_isboxed) { assert((char*)m.ptr_or_offset - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); if (isatomic) @@ -449,7 +446,7 @@ JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, j } if (layout->size != 0) { assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); - int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + int needlock = layout->flags.arrayelem_islocked; size_t fsz = jl_datatype_size((jl_datatype_t*)jl_typeof(rhs)); // need to shrink-wrap the final copy if (isatomic && !needlock) { jl_atomic_store_bits(data, rhs, fsz); diff --git a/src/julia.h b/src/julia.h index e07fabcb10349..a5564670479db 100644 --- a/src/julia.h +++ b/src/julia.h @@ -575,10 +575,12 @@ typedef struct { // metadata bit only for GenericMemory eltype layout uint16_t arrayelem_isboxed : 1; uint16_t arrayelem_isunion : 1; + uint16_t arrayelem_isatomic : 1; + uint16_t arrayelem_islocked : 1; // If set, this type's egality can be determined entirely by comparing // the non-padding bits of this datatype. uint16_t isbitsegal : 1; - uint16_t padding : 10; + uint16_t padding : 8; } flags; // union { // jl_fielddesc8_t field8[nfields]; @@ -1668,6 +1670,8 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_addrspacecore(v) jl_typetagis(v,jl_addrspacecore_type) #define jl_is_abioverride(v) jl_typetagis(v,jl_abioverride_type) #define jl_genericmemory_isbitsunion(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_isunion) +#define jl_genericmemory_isatomic(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_isatomic) +#define jl_genericmemory_islocked(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_islocked) #define jl_is_array_any(v) jl_typetagis(v,jl_array_any_type) JL_DLLEXPORT int jl_subtype(jl_value_t *a, jl_value_t *b); diff --git a/src/rtutils.c b/src/rtutils.c index e3672ab5d887e..a64f2e2e757c0 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -1294,6 +1294,11 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt } else { char *ptr = ((char*)m->ptr) + j * layout->size; + if (layout->flags.arrayelem_islocked) { + // Skip the lock at the beginning for locked arrays + size_t lock_size = sizeof(jl_mutex_t); + ptr += lock_size; + } n += jl_static_show_x_(out, (jl_value_t*)ptr, (jl_datatype_t*)(typetagdata ? jl_nth_union_component(el_type, typetagdata[j]) : el_type), depth, ctx); diff --git a/test/atomics.jl b/test/atomics.jl index 2d4a713b1d30d..3572824741459 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -111,6 +111,7 @@ Base.show(io::IO, x::Int24) = print(io, "Int24(", Core.Intrinsics.zext_int(Int, ## Fields @noinline function _test_field_operators(r) + GC.gc(false) r = r[] TT = fieldtype(typeof(r), :x) T = typeof(getfield(r, :x)) @@ -147,6 +148,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @noinline function _test_field_orderings(r, x, y) @nospecialize x y + GC.gc(false) r = r[] TT = fieldtype(typeof(r), :x) @@ -328,8 +330,9 @@ test_field_orderings(ARefxy{Any}(true, false), true, false) test_field_orderings(ARefxy{Union{Nothing,Missing}}(nothing, missing), nothing, missing) test_field_orderings(ARefxy{Union{Nothing,Int}}(nothing, 123_1), nothing, 123_1) test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_field_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_field_orderings(Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) test_field_orderings(Complex{Rational{Integer}}(10, 30), Complex{Rational{Integer}}(20, 40)) +test_field_orderings(Pair{NTuple{3,Float64},NTuple{3,Real}}((10.5,11.5,12.5), (30.5,40.5,50.5)), Pair{NTuple{3,Float64},NTuple{3,Real}}((110.5,111.5,112.5), (130.5,140.5,150.5))) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) @@ -705,7 +708,7 @@ test_global_orderings(Any, true, false) test_global_orderings(Union{Nothing,Missing}, nothing, missing) test_global_orderings(Union{Nothing,Int}, nothing, 123_1) test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_global_orderings(Complex{Real}, Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_global_orderings(Complex{Real}, Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) test_global_orderings(Float64, 10.0, 20.0) test_global_orderings(Float64, NaN, Inf) @@ -1024,15 +1027,17 @@ test_memory_operators(Float64) end @noinline function test_memory_orderings(T::Type, x, y) @nospecialize - xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) - memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x - yr = GenericMemoryRef(Memory{T}(undef, 1)) + xr = GenericMemoryRef(AtomicMemory{T}(undef, 2), 2) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[2] = x + yr = GenericMemoryRef(Memory{T}(undef, 2), 2) yr[] = y + GC.gc(false) _test_memory_orderings(Ref(xr), Ref(yr), x, y) - xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) - memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x - yr = GenericMemoryRef(Memory{T}(undef, 1)) + xr = GenericMemoryRef(AtomicMemory{T}(undef, 2), 2) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[2] = x + yr = GenericMemoryRef(Memory{T}(undef, 2), 2) yr[] = y + GC.gc(false) _test_memory_orderings(Ref{Any}(xr), Ref{Any}(yr), x, y) nothing end @@ -1047,7 +1052,8 @@ test_memory_orderings(Any, true, false) test_memory_orderings(Union{Nothing,Missing}, nothing, missing) test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_memory_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_memory_orderings(Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) +test_memory_orderings(Pair{NTuple{3,Float64},NTuple{3,Real}}((10.5,11.5,12.5), (30.5,40.5,50.5)), Pair{NTuple{3,Float64},NTuple{3,Real}}((110.5,111.5,112.5), (130.5,140.5,150.5))) test_memory_orderings(10.0, 20.0) test_memory_orderings(NaN, Inf) From d5fdf04cb2c903c6fffa3db5da199ba1fce83065 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Sat, 26 Jul 2025 13:12:54 -0400 Subject: [PATCH 575/662] Fix typo in tests introduced by #21858 (#59102) That [2017 PR](https://github.com/JuliaLang/julia/pull/21858) used very old types and had a semantic merge conflict. --- test/complex.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/complex.jl b/test/complex.jl index 10b4bc8aa03d6..ac867c6f0e5fb 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -934,8 +934,8 @@ end @testset "eps" begin @test eps(1.0+1.0im) === 3.1401849173675503e-16 - @test eps(Complex128) === eps(1.0+1.0im) - @test eps(Complex64) === 1.6858739f-7 + @test eps(Complex{Float64}) === eps(1.0+1.0im) + @test eps(Complex{Float32}) === 1.6858739f-7 @test eps(Float32(1.0)+Float32(1.0)im) === eps(Complex{Float32}) end From 2afae111bb621b139c3291783ec1e0b529f50ac0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 26 Jul 2025 14:10:29 -0400 Subject: [PATCH 576/662] Fix msys symlink override rule (#59101) The `export VAR=VAL` is syntax, so it can't be expanded. Fixes #59096 --- deps/libgit2.mk | 2 +- deps/llvm.mk | 2 +- deps/tools/common.mk | 2 +- deps/zstd.mk | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index fad5db5fd4f79..85bc0629f6f28 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -4,7 +4,7 @@ ifneq ($(USE_BINARYBUILDER_LIBGIT2),1) LIBGIT2_GIT_URL := https://github.com/libgit2/libgit2.git LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1 $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,,$(SRCCACHE))) -$(SRCCACHE)/$(LIBGIT2_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) +$(SRCCACHE)/$(LIBGIT2_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) ifeq ($(USE_SYSTEM_LIBSSH2), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/libssh2 diff --git a/deps/llvm.mk b/deps/llvm.mk index 77b8f4ae978c1..e3303aba55afd 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -14,7 +14,7 @@ $(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) # symlinks. We don't particularly care either way - we just need to symlinks # to succeed. We could guard this by a uname check, but it's harmless elsewhere, # so let's not incur the additional overhead. -$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) +$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) LLVM_BUILDDIR := $(BUILDDIR)/$(LLVM_SRC_DIR) LLVM_BUILDDIR_withtype := $(LLVM_BUILDDIR)/build_$(LLVM_BUILDTYPE) diff --git a/deps/tools/common.mk b/deps/tools/common.mk index d689da52daf2f..b7ebc39169221 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -102,7 +102,7 @@ endif # symlinks. We don't particularly care either way - we just need to symlinks # to succeed. We could guard this by a uname check, but it's harmless elsewhere, # so let's not incur the additional overhead. -MSYS_NONEXISTENT_SYMLINK_TARGET_FIX := export MSYS=winsymlinks:native +MSYS_NONEXISTENT_SYMLINK_TARGET_FIX := winsymlinks:native # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure diff --git a/deps/zstd.mk b/deps/zstd.mk index 4fc64a8442588..ecce416ab3f38 100644 --- a/deps/zstd.mk +++ b/deps/zstd.mk @@ -3,7 +3,7 @@ ifneq ($(USE_BINARYBUILDER_ZSTD), 1) ZSTD_GIT_URL := https://github.com/facebook/zstd.git ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 $(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) -$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: $(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) +$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) ifeq ($(OS), WINNT) From 2b71259d025557ca3d86925eb9e73bd5ddebb694 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 26 Jul 2025 21:13:19 -0400 Subject: [PATCH 577/662] inference: Make test indepdent of the `Complex` method table (#59105) --- Compiler/test/inference.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 58f2231319af9..58cd3db49c371 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -3549,8 +3549,14 @@ f31974(n::Int) = f31974(1:n) # call cycles. @test code_typed(f31974, Tuple{Int}) !== nothing -f_overly_abstract_complex() = Complex(Ref{Number}(1)[]) -@test Base.return_types(f_overly_abstract_complex, Tuple{}) == [Complex] +# Issue #33472 +struct WrapperWithUnionall33472{T<:Real} + x::T +end + +f_overly_abstract33472() = WrapperWithUnionall33472(Base.inferencebarrier(1)::Number) +# Check that this doesn't infer as `WrapperWithUnionall33472{T<:Number}`. +@test Base.return_types(f_overly_abstract33472, Tuple{}) == [WrapperWithUnionall33472] # Issue 26724 const IntRange = AbstractUnitRange{<:Integer} From a2457e6ed7b425f01daa157c71ca72f0bf4be573 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 26 Jul 2025 21:13:42 -0400 Subject: [PATCH 578/662] Add uptime to CI test info (#59107) --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index 0695139844ba3..392c1c9828587 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -135,6 +135,7 @@ cd(@__DIR__) do Sys.CPU_THREADS = $(Sys.CPU_THREADS) Sys.total_memory() = $(Base.format_bytes(Sys.total_memory())) Sys.free_memory() = $(Base.format_bytes(Sys.free_memory())) + Sys.uptime() = $(Sys.uptime()) ($(round(Sys.uptime() / (60 * 60), digits=1)) hours) """) #pretty print the information about gc and mem usage From 55e2bd78708e97df704fd806c4051321c31b218f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20H=C3=A9not?= <38465572+OlivierHnt@users.noreply.github.com> Date: Sun, 27 Jul 2025 13:27:14 +0200 Subject: [PATCH 579/662] Fix rounding when converting Rational to BigFloat (#59063) --- base/mpfr.jl | 8 +++++++- test/mpfr.jl | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/base/mpfr.jl b/base/mpfr.jl index ee1e4eac39145..cb51d21aa5bc5 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -392,12 +392,18 @@ BigFloat(x::Union{Float16,Float32}, r::MPFRRoundingMode=rounding_raw(BigFloat); BigFloat(Float64(x), r; precision=precision) function BigFloat(x::Rational, r::MPFRRoundingMode=rounding_raw(BigFloat); precision::Integer=_precision_with_base_2(BigFloat)) + r_den = _opposite_round(r) setprecision(BigFloat, precision) do setrounding_raw(BigFloat, r) do - BigFloat(numerator(x))::BigFloat / BigFloat(denominator(x))::BigFloat + BigFloat(numerator(x))::BigFloat / BigFloat(denominator(x), r_den)::BigFloat end end end +function _opposite_round(r::MPFRRoundingMode) + r == MPFRRoundUp && return MPFRRoundDown + r == MPFRRoundDown && return MPFRRoundUp + return r +end function tryparse(::Type{BigFloat}, s::AbstractString; base::Integer=0, precision::Integer=_precision_with_base_2(BigFloat), rounding::MPFRRoundingMode=rounding_raw(BigFloat)) !isempty(s) && isspace(s[end]) && return tryparse(BigFloat, rstrip(s), base = base) diff --git a/test/mpfr.jl b/test/mpfr.jl index 8a537d1d4acd8..48477fc4dbcb7 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -35,6 +35,9 @@ import Base.MPFR @test typeof(BigFloat(1//1)) == BigFloat @test typeof(BigFloat(one(Rational{BigInt}))) == BigFloat + rat = 1 // (big(2)^300 + 1) + @test BigFloat(rat, RoundDown) < rat < BigFloat(rat, RoundUp) + @test BigFloat(-rat, RoundUp) < -rat < BigFloat(-rat, RoundDown) # BigFloat constructor respects global precision when not specified let prec = precision(BigFloat) < 16 ? 256 : precision(BigFloat) ÷ 2 From cf9b7a86c5777df0bb6f9aa08d4ba73ed17791d1 Mon Sep 17 00:00:00 2001 From: Andy Dienes <51664769+adienes@users.noreply.github.com> Date: Sun, 27 Jul 2025 09:54:14 -0400 Subject: [PATCH 580/662] make ReinterpretArray more Offset-safe (#58898) --- base/reinterpretarray.jl | 15 +++++++++++++++ test/reinterpretarray.jl | 18 ++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index f1cd9c9e82918..dc993515ed0cd 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -311,6 +311,7 @@ SimdLoop.simd_inner_length(::SCartesianIndices2{K}, ::Any) where K = K SCartesianIndex2{K}(I1+1, Ilast) end +_maybe_reshape(::IndexSCartesian2, A::AbstractArray, I...) = _maybe_reshape(IndexCartesian(), A, I...) _maybe_reshape(::IndexSCartesian2, A::ReshapedReinterpretArray, I...) = A # fallbacks @@ -329,11 +330,25 @@ function _getindex(::IndexSCartesian2, A::AbstractArray{T,N}, ind::SCartesianInd J = _ind2sub(tail(axes(A)), ind.j) getindex(A, ind.i, J...) end + +function _getindex(::IndexSCartesian2{2}, A::AbstractArray{T,2}, ind::SCartesianIndex2) where {T} + @_propagate_inbounds_meta + J = first(axes(A, 2)) + ind.j - 1 + getindex(A, ind.i, J) +end + function _setindex!(::IndexSCartesian2, A::AbstractArray{T,N}, v, ind::SCartesianIndex2) where {T,N} @_propagate_inbounds_meta J = _ind2sub(tail(axes(A)), ind.j) setindex!(A, v, ind.i, J...) end + +function _setindex!(::IndexSCartesian2{2}, A::AbstractArray{T,2}, v, ind::SCartesianIndex2) where {T} + @_propagate_inbounds_meta + J = first(axes(A, 2)) + ind.j - 1 + setindex!(A, v, ind.i, J) +end + eachindex(style::IndexSCartesian2, A::AbstractArray) = eachindex(style, parent(A)) ## AbstractArray interface diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index 873c3bc32c993..f6269792ef0a6 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -10,12 +10,18 @@ tslow(a::AbstractArray) = TSlow(a) wrapper(a::AbstractArray) = WrapperArray(a) fcviews(a::AbstractArray) = view(a, ntuple(Returns(:),ndims(a)-1)..., axes(a)[end]) fcviews(a::AbstractArray{<:Any, 0}) = view(a) +offset_nominal(a::AbstractArray) = OffsetArray(a) +offset_maybe(a::AbstractArray) = (eltype(a) <: Real) ? a : OffsetArray(a, (1-ndims(A)):2:(ndims(A)-1)...) tslow(t::Tuple) = map(tslow, t) wrapper(t::Tuple) = map(wrapper, t) fcviews(t::Tuple) = map(fcviews, t) +offset_nominal(t::Tuple) = map(offset_nominal, t) +offset_maybe(t::Tuple) = map(offset_maybe, t) test_many_wrappers(testf, A, wrappers) = foreach(w -> testf(w(A)), wrappers) -test_many_wrappers(testf, A) = test_many_wrappers(testf, A, (identity, tslow, wrapper, fcviews)) +test_many_wrappers(testf, A) = test_many_wrappers( + testf, A, (identity, tslow, wrapper, fcviews, offset_nominal, offset_maybe) +) A = Int64[1, 2, 3, 4] Ars = Int64[1 3; 2 4] @@ -37,10 +43,6 @@ test_many_wrappers(B, (identity, tslow)) do _B @test @inferred(size(reinterpret(reshape, Int128, _B))) == (3,) end -test_many_wrappers(C) do Cr - @test reinterpret(reshape, Tuple{Int8, Int}, Cr) == fill((1,1)) -end - @test_throws ArgumentError("cannot reinterpret `Int64` as `Vector{Int64}`, type `Vector{Int64}` is not a bits type") reinterpret(Vector{Int64}, A) @test_throws ArgumentError("cannot reinterpret `Vector{Int32}` as `Int32`, type `Vector{Int32}` is not a bits type") reinterpret(Int32, Av) @test_throws ArgumentError("cannot reinterpret a zero-dimensional `Int64` array to `Int32` which is of a different size") reinterpret(Int32, reshape([Int64(0)])) @@ -160,8 +162,10 @@ test_many_wrappers(A3) do A3_ @test A3[2,1,2] == 400 end -test_many_wrappers(C) do Cr +test_many_wrappers(C) do Cr_ + Cr = deepcopy(Cr_) r = reinterpret(reshape, Tuple{Int, Int}, Cr) + @test r == fill((1,1)) r[] = (2,2) @test r[] === (2,2) r[1] = (3,3) @@ -378,6 +382,8 @@ let a = rand(ComplexF32, 5) r = reinterpret(reshape, Float32, a) ref = Array(r) + @test all(r .== OffsetArray(r)[:, :, :]) + @test r[1, :, 1] == ref[1, :] @test r[1, :, 1, 1, 1] == ref[1, :] @test r[1, :, UInt8(1)] == ref[1, :] From 55b1f3c41baeb6b0ccfc0ad89d7c941e6c8bc229 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Sun, 27 Jul 2025 17:27:32 -0400 Subject: [PATCH 581/662] remove extraneous function included in #21858 (#59109) Removes an apparently extraneous function accidentally included in #21858, as noted in https://github.com/JuliaLang/julia/pull/21858/files#r2233250284. --- base/complex.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index be42e7db1dde4..6651581a96240 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -1142,10 +1142,6 @@ function complex(A::AbstractArray{T}) where T convert(AbstractArray{typeof(complex(zero(T)))}, A) end -## Promotion to complex ## - -_default_type(T::Type{Complex}) = Complex{Int} - ## Machine epsilon for complex ## eps(z::Complex{<:AbstractFloat}) = hypot(eps(real(z)), eps(imag(z))) From 853aba587d8045ed81a4fef3f7051af6aadad506 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 28 Jul 2025 12:22:09 -0700 Subject: [PATCH 582/662] [REPL] Handle empty completion, keywords better (#59045) When the context is empty, (like ""), return only names local to the module (fixes #58931). If the cursor is on something that "looks like" an identifier, like a boolean or one of the keywords, treat it as if it was one for completion purposes. Typing a keyword and hitting tab no longer returns the completions for the empty input (fixes #58309, #58832). --- stdlib/REPL/src/REPLCompletions.jl | 11 ++++++----- stdlib/REPL/test/replcompletions.jl | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 7de1246566b48..e05d0435d81d8 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -186,7 +186,7 @@ function complete_symbol!(suggestions::Vector{Completion}, complete_modules_only::Bool=false, shift::Bool=false) local mod, t, val - complete_internal_only = false + complete_internal_only = isempty(name) if prefix !== nothing res = repl_eval_ex(prefix, context_module) res === nothing && return Completion[] @@ -1095,18 +1095,19 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif # Symbol completion # TODO: Should completions replace the identifier at the cursor? + looks_like_ident = Base.isidentifier(@view string[intersect(char_range(cur), 1:pos)]) if cur.parent !== nothing && kind(cur.parent) == K"var" # Replace the entire var"foo", but search using only "foo". r = intersect(char_range(cur.parent), 1:pos) r2 = char_range(children_nt(cur.parent)[1]) s = string[intersect(r2, 1:pos)] - elseif kind(cur) in KSet"Identifier @" - r = intersect(char_range(cur), 1:pos) - s = string[r] elseif kind(cur) == K"MacroName" # Include the `@` r = intersect(prevind(string, cur.position):char_last(cur), 1:pos) s = string[r] + elseif looks_like_ident || kind(cur) in KSet"Bool Identifier @" + r = intersect(char_range(cur), 1:pos) + s = string[r] else r = nextind(string, pos):pos s = "" @@ -1114,7 +1115,7 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif complete_modules_only = false prefix = node_prefix(cur, context_module) - comp_keywords = prefix === nothing + comp_keywords = prefix === nothing && !isempty(s) # Complete loadable module names: # import Mod TAB diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 43bea08e285f1..2225417ddcec6 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -2720,3 +2720,25 @@ let s = "foo58296(findfi" @test "findfirst" in c @test r == 10:15 end + +# #58931 - only show local names when completing the empty string +let s = "" + c, r = test_complete_foo(s) + @test "test" in c + @test !("rand" in c) +end + +# #58309, #58832 - don't show every name when completing after a full keyword +let s = "true" # bool is a little different (Base.isidentifier special case) + c, r = test_complete(s) + @test "trues" in c + @test "true" in c + @test !("rand" in c) +end + +let s = "for" + c, r = test_complete(s) + @test "for" in c + @test "foreach" in c + @test !("rand" in c) +end From 57def4d19ef79794439ef839e5e3b551323f49a2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 28 Jul 2025 15:47:24 -0400 Subject: [PATCH 583/662] Add builtin function name to add methods error (#59112) ``` julia> Base.throw(x::Int) = 1 ERROR: cannot add methods to builtin function `throw` Stacktrace: [1] top-level scope @ REPL[1]:1 ``` --- src/method.c | 4 ++-- test/core.jl | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/method.c b/src/method.c index 8220178964333..c33b343742dc8 100644 --- a/src/method.c +++ b/src/method.c @@ -1220,7 +1220,7 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, // jl_value_t **ttypes = { jl_builtin_type, jl_tparam0(jl_anytuple_type) }; // jl_value_t *invalidt = jl_apply_tuple_type_v(ttypes, 2); // Tuple{Union{Builtin,OpaqueClosure}, Vararg} // if (!jl_has_empty_intersection(argtype, invalidt)) - // jl_error("cannot add methods to a builtin function"); + // jl_error("cannot add methods to builtin function"); //} assert(jl_is_linenode(functionloc)); @@ -1299,7 +1299,7 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, } ft = jl_rewrap_unionall(ft, argtype); if (!external_mt && !jl_has_empty_intersection(ft, (jl_value_t*)jl_builtin_type)) // disallow adding methods to Any, Function, Builtin, and subtypes, or Unions of those - jl_error("cannot add methods to a builtin function"); + jl_errorf("cannot add methods to builtin function `%s`", jl_symbol_name(name)); m = jl_new_method_uninit(module); m->external_mt = (jl_value_t*)external_mt; diff --git a/test/core.jl b/test/core.jl index 3d79802e33a87..78d777c6ed84c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2661,13 +2661,16 @@ struct D14919 <: Function; end @test B14919()() == "It's a brand new world" @test C14919()() == D14919()() == "Boo." -let ex = ErrorException("cannot add methods to a builtin function") +let ex_t = ErrorException, ex_r = r"cannot add methods to builtin function" for f in (:(Core.Any), :(Core.Function), :(Core.Builtin), :(Base.Callable), :(Union{Nothing,F} where F), :(typeof(Core.getfield)), :(Core.IntrinsicFunction)) - @test_throws ex @eval (::$f)() = 1 + @test_throws ex_t @eval (::$f)() = 1 + @test_throws ex_r @eval (::$f)() = 1 end - @test_throws ex @eval (::Union{Nothing,F})() where {F<:Function} = 1 + @test_throws ex_t @eval (::Union{Nothing,F})() where {F<:Function} = 1 + @test_throws ex_r @eval (::Union{Nothing,F})() where {F<:Function} = 1 for f in (:(Core.getfield),) - @test_throws ex @eval $f() = 1 + @test_throws ex_t @eval $f() = 1 + @test_throws ex_r @eval $f() = 1 end end From f356a40b930b608d8f5c323e7ef61a8794398818 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 28 Jul 2025 15:48:25 -0400 Subject: [PATCH 584/662] better error in juliac for defining main inside a new module (#59106) This is more helpful if the script you try to compile defines a module containing main instead of defining it at the toplevel. --- contrib/juliac/juliac-buildscript.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/juliac/juliac-buildscript.jl b/contrib/juliac/juliac-buildscript.jl index db1cb75339e4d..80a3fd756bcd0 100644 --- a/contrib/juliac/juliac-buildscript.jl +++ b/contrib/juliac/juliac-buildscript.jl @@ -37,7 +37,7 @@ function _main(argc::Cint, argv::Ptr{Ptr{Cchar}})::Cint return Main.main(args) end -let mod = Base.include(Main, ARGS[1]) +let include_result = Base.include(Main, ARGS[1]) Core.@latestworld if ARGS[2] == "--output-exe" have_cmain = false @@ -49,6 +49,11 @@ let mod = Base.include(Main, ARGS[1]) break end end + elseif include_result isa Module && isdefined(include_result, :main) + error(""" + The `main` function must be defined in `Main`. If you are defining it inside a + module, try adding `import .$(nameof(include_result)).main` to $(ARGS[1]). + """) end if !have_cmain if Base.should_use_main_entrypoint() @@ -59,7 +64,7 @@ let mod = Base.include(Main, ARGS[1]) error("`@main` must accept a `Vector{String}` argument.") end else - error("To generate an executable a `@main` function must be defined.") + error("To generate an executable a `@main` function must be defined in the `Main` module.") end end end From f2c283772075e07e9bc5e17ca6b7e72d38cc90d5 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Mon, 28 Jul 2025 19:46:51 -0500 Subject: [PATCH 585/662] Don't call `sort!` and `partialsort!` "in-place" (#58901) "in-place" strongly implies non-allocating. This docs change removes that language from the `sort!` and `sortperm!` docstrings because those functions do sometimes allocate. --- base/sort.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/base/sort.jl b/base/sort.jl index 6cc1420708a35..bfa65d2459680 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -115,7 +115,7 @@ maybeview(v, k::Integer) = v[k] """ partialsort!(v, k; by=identity, lt=isless, rev=false) -Partially sort the vector `v` in place so that the value at index `k` (or +Mutate the vector `v` so that the value at index `k` (or range of adjacent values if `k` is a range) occurs at the position where it would appear if the array were fully sorted. If `k` is a single index, that value is returned; if `k` is a range, an array of values at those indices is @@ -1625,10 +1625,11 @@ defalg(v) = DEFAULT_STABLE """ sort!(v; alg::Base.Sort.Algorithm=Base.Sort.defalg(v), lt=isless, by=identity, rev::Bool=false, order::Base.Order.Ordering=Base.Order.Forward) -Sort the vector `v` in place. A stable algorithm is used by default: the -ordering of elements that compare equal is preserved. A specific algorithm can -be selected via the `alg` keyword (see [Sorting Algorithms](@ref) for available -algorithms). +Mutate the vector `v` so that it is sorted. + +A stable algorithm is used by default: the ordering of elements that +compare equal is preserved. A specific algorithm can be selected via the +`alg` keyword (see [Sorting Algorithms](@ref) for available algorithms). Elements are first transformed with the function `by` and then compared according to either the function `lt` or the ordering `order`. Finally, the @@ -1745,7 +1746,7 @@ end Variant of [`sort!`](@ref) that returns a sorted copy of `v` leaving `v` itself unmodified. When calling `sort` on the [`keys`](@ref) or [`values](@ref) of a dictionary, `v` is -collected and then sorted in place. +collected and then sorted. !!! compat "Julia 1.12" Sorting `NTuple`s requires Julia 1.12 or later. @@ -2289,7 +2290,7 @@ UIntMappable(T::Type, order::ReverseOrdering) = UIntMappable(T, order.fwd) ### Vectors -# Convert v to unsigned integers in place, maintaining sort order. +# Convert v to unsigned integers in-place, maintaining sort order. function uint_map!(v::AbstractVector, lo::Integer, hi::Integer, order::Ordering) u = reinterpret(UIntMappable(eltype(v), order), v) @inbounds for i in lo:hi From 50c895614653e56f30a7995265b93398f89f7da4 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:48:11 -0700 Subject: [PATCH 586/662] [REPL] Fix keyword arguments completions with do block (#59123) In `_complete_methods`, desugar the `:do` Expr into a call with a lambda in the first argument. Fixes #58833. --- stdlib/REPL/src/REPLCompletions.jl | 9 +++++++++ stdlib/REPL/test/replcompletions.jl | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index e05d0435d81d8..385d0fe720b46 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -692,6 +692,15 @@ code_typed(CC.typeinf, (REPLInterpreter, CC.InferenceState)) MAX_METHOD_COMPLETIONS::Int = 40 function _complete_methods(ex_org::Expr, context_module::Module, shift::Bool) isempty(ex_org.args) && return 2, nothing, [], Set{Symbol}() + # Desugar do block call into call with lambda + if ex_org.head === :do && length(ex_org.args) >= 2 + ex_call = ex_org.args[1] + ex_args = [x for x in ex_call.args if !(x isa Expr && x.head === :parameters)] + ex_params = findfirst(x -> x isa Expr && x.head === :parameters, ex_call.args) + new_args = [ex_args[1], ex_org.args[end], ex_args[2:end]...] + ex_params !== nothing && push!(new_args, ex_call.args[ex_params]) + ex_org = Expr(:call, new_args...) + end funct = repl_eval_ex(ex_org.args[1], context_module) funct === nothing && return 2, nothing, [], Set{Symbol}() funct = CC.widenconst(funct) diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 2225417ddcec6..7b2b032af77d3 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -139,6 +139,7 @@ let ex = kwtest4(a::SubString; x23, _something) = pass kwtest5(a::Int, b, x...; somekwarg, somekotherkwarg) = pass kwtest5(a::Char, b; xyz) = pass + kwtest6(f::Function, arg1; somekwarg) = pass const named = (; len2=3) const fmsoebelkv = (; len2=3) @@ -198,6 +199,8 @@ test_scomplete(s) = map_completion_text(@inferred(shell_completions(s, lastinde test_complete_pos(s) = map_completion_text(@inferred(completions(replace(s, '|' => ""), findfirst('|', s)-1))) test_complete_context(s, m=@__MODULE__; shift::Bool=true) = map_completion_text(@inferred(completions(s,lastindex(s), m, shift))) +test_complete_context_pos(s, m=@__MODULE__; shift::Bool=true) = + map_completion_text(@inferred(completions(replace(s, '|' => ""), findfirst('|', s)-1, m, shift))) test_complete_foo(s; shift::Bool=true) = test_complete_context(s, Main.CompletionFoo; shift) test_complete_noshift(s) = map_completion_text(@inferred(completions(s, lastindex(s), Main, false))) @@ -2742,3 +2745,10 @@ let s = "for" @test "foreach" in c @test !("rand" in c) end + +# #58833 - Autocompletion of keyword arguments with do-blocks is broken +let s = "kwtest6(123; som|) do x; x + 3 end" + c, r = test_complete_context_pos(s, Main.CompletionFoo) + @test "somekwarg=" in c + @test r == 14:16 +end From fde79f5bb5c4835189bfe3215521137176ac3eb8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 28 Jul 2025 20:01:22 -0500 Subject: [PATCH 587/662] Fix memory order typo in "src/julia_atomics.h" (#59120) --- src/julia_atomics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/julia_atomics.h b/src/julia_atomics.h index d05f0fafab28f..1d8fba3b44e33 100644 --- a/src/julia_atomics.h +++ b/src/julia_atomics.h @@ -190,7 +190,7 @@ T jl_atomic_exchange_explicit(std::atomic *ptr, S desired, std::memory_order { return std::atomic_exchange_explicit(ptr, desired, order); } -#define jl_atomic_exchange_release(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_reease) +#define jl_atomic_exchange_release(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_release) #define jl_atomic_exchange_relaxed(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_relaxed) extern "C" { #else From 90e3c1a568991bfc9e24276fa9639a3a92463769 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 29 Jul 2025 01:33:39 -0400 Subject: [PATCH 588/662] Update list of COMPILER_SRCS in sysimage.mk (#59125) This list needs to match all source files included in Base_compiler.jl. Written by claude. --- sysimage.mk | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sysimage.mk b/sysimage.mk index bdd69dc6e31c0..2a938de697156 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -31,32 +31,48 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ base/abstractset.jl \ base/iddict.jl \ base/idset.jl \ + base/anyall.jl \ base/array.jl \ + base/baseext.jl \ base/bitarray.jl \ base/bitset.jl \ base/bool.jl \ + base/c.jl \ + base/checked.jl \ + base/cmem.jl \ + base/coreio.jl \ + base/coreir.jl \ base/ctypes.jl \ base/error.jl \ base/essentials.jl \ base/expr.jl \ base/exports.jl \ + base/flfrontend.jl \ + base/float.jl \ + base/gcutils.jl \ base/generator.jl \ + base/genericmemory.jl \ base/int.jl \ base/indices.jl \ base/iterators.jl \ base/invalidation.jl \ base/module.jl \ base/namedtuple.jl \ + base/ntuple.jl \ base/number.jl \ base/operators.jl \ base/options.jl \ + base/ordering.jl \ base/pair.jl \ base/pointer.jl \ base/promotion.jl \ + base/public.jl \ base/range.jl \ + base/refvalue.jl \ + base/rounding.jl \ base/runtime_internals.jl \ + base/strings/lazy.jl \ base/traits.jl \ - base/refvalue.jl \ base/tuple.jl) COMPILER_SRCS += $(shell find $(JULIAHOME)/Compiler/src -name \*.jl -and -not -name verifytrim.jl -and -not -name show.jl) # sort these to remove duplicates From ce600a0945ffedf796df8af2985ed37faa1f265d Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 29 Jul 2025 13:25:50 -0400 Subject: [PATCH 589/662] Enforce ExceptionStack stack type (#59133) The code that processes these requires these to be this NamedTuple, so enforce that so that it fails on construction rather than on access. Also fix the one place that was passing a regular tuple. Fixes #59132. --- base/client.jl | 4 ++-- base/error.jl | 6 +++--- test/runtests.jl | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/client.jl b/base/client.jl index 03248700e7ca0..8ff925d0d9ba4 100644 --- a/base/client.jl +++ b/base/client.jl @@ -71,7 +71,7 @@ function repl_cmd(cmd, out) catch # Julia throws an exception if it can't find the cmd (which may be the shell itself), but the stack trace isn't useful lasterr = current_exceptions() - lasterr = ExceptionStack([(exception = e[1], backtrace = [] ) for e in lasterr]) + lasterr = ExceptionStack(NamedTuple[(exception = e[1], backtrace = [] ) for e in lasterr]) invokelatest(display_error, lasterr) end end @@ -99,7 +99,7 @@ function scrub_repl_backtrace(bt) return bt end scrub_repl_backtrace(stack::ExceptionStack) = - ExceptionStack(Any[(;x.exception, backtrace = scrub_repl_backtrace(x.backtrace)) for x in stack]) + ExceptionStack(NamedTuple[(;x.exception, backtrace = scrub_repl_backtrace(x.backtrace)) for x in stack]) istrivialerror(stack::ExceptionStack) = length(stack) == 1 && length(stack[1].backtrace) ≤ 1 && !isa(stack[1].exception, MethodError) diff --git a/base/error.jl b/base/error.jl index 3ea7210652dad..e5eecf453ee75 100644 --- a/base/error.jl +++ b/base/error.jl @@ -135,8 +135,8 @@ function catch_backtrace() return _reformat_bt(bt::Vector{Ptr{Cvoid}}, bt2::Vector{Any}) end -struct ExceptionStack <: AbstractArray{Any,1} - stack::Array{Any,1} +struct ExceptionStack <: AbstractArray{NamedTuple{(:exception, :backtrace)},1} + stack::Array{NamedTuple{(:exception, :backtrace)},1} end """ @@ -159,7 +159,7 @@ uncaught exceptions. """ function current_exceptions(task::Task=current_task(); backtrace::Bool=true) raw = ccall(:jl_get_excstack, Any, (Any,Cint,Cint), task, backtrace, typemax(Cint))::Vector{Any} - formatted = Any[] + formatted = NamedTuple{(:exception, :backtrace)}[] stride = backtrace ? 3 : 1 for i = reverse(1:stride:length(raw)) exc = raw[i] diff --git a/test/runtests.jl b/test/runtests.jl index 392c1c9828587..ba64c66172126 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -415,7 +415,7 @@ cd(@__DIR__) do # deserialization errors or something similar. Record this testset as Errored. fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(Any[(resp, [])]), LineNumberNode(1), nothing)) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = resp, backtrace = [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() @@ -424,7 +424,7 @@ cd(@__DIR__) do for test in all_tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack(Any[("skipped", [])]), LineNumberNode(1), nothing)) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack(NamedTuple[(;exception = "skipped", backtrace = [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() From 9f6996149a9b56c03cf1ccff051e35c225539c04 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 29 Jul 2025 17:39:46 -0400 Subject: [PATCH 590/662] fix serializer compat with CodeInfos from v1.11 (#58650) --- src/method.c | 3 ++ stdlib/Serialization/src/Serialization.jl | 36 +++++++++++++++++++++-- stdlib/Serialization/test/runtests.jl | 12 ++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/method.c b/src/method.c index c33b343742dc8..46a4f5bc4d56c 100644 --- a/src/method.c +++ b/src/method.c @@ -958,6 +958,9 @@ JL_DLLEXPORT void jl_method_set_source(jl_method_t *m, jl_code_info_t *src) } src = jl_copy_code_info(src); src->isva = m->isva; // TODO: It would be nice to reverse this + // If nargs hasn't been set yet, do it now. This can happen if an old CodeInfo is deserialized. + if (src->nargs == 0) + src->nargs = m->nargs; assert(m->nargs == src->nargs); src->code = copy; jl_gc_wb(src, copy); diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 3c4152bf10598..1b25c59bf3cdf 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -8,6 +8,7 @@ Provide serialization of Julia objects via the functions module Serialization import Base: Bottom, unsafe_convert +import Base.ScopedValues: ScopedValue, with import Core: svec, SimpleVector using Base: unaliascopy, unwrap_unionall, require_one_based_indexing, ntupleany using Core.IR @@ -28,6 +29,8 @@ end Serializer(io::IO) = Serializer{typeof(io)}(io) +const current_module = ScopedValue{Union{Nothing,Module}}(nothing) + ## serializing values ## const n_int_literals = 33 @@ -1064,7 +1067,10 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) nospecializeinfer = false constprop = 0x00 purity = 0x0000 - template_or_is_opaque = deserialize(s) + local template_or_is_opaque, template + with(current_module => mod) do + template_or_is_opaque = deserialize(s) + end if isa(template_or_is_opaque, Bool) is_for_opaque_closure = template_or_is_opaque if format_version(s) >= 24 @@ -1078,7 +1084,9 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) elseif format_version(s) >= 17 purity = UInt16(deserialize(s)::UInt8) end - template = deserialize(s) + with(current_module => mod) do + template = deserialize(s) + end else template = template_or_is_opaque end @@ -1182,6 +1190,22 @@ function deserialize(s::AbstractSerializer, ::Type{PhiNode}) return PhiNode(edges, values) end +# v1.12 disallows bare symbols in IR, but older CodeInfos might still have them +function symbol_to_globalref(@nospecialize(x), m::Module) + mapper(@nospecialize(x)) = symbol_to_globalref(x, m) + if x isa Symbol + return GlobalRef(m, x) + elseif x isa Expr + return Expr(x.head, map(mapper, x.args)...) + elseif x isa ReturnNode + return ReturnNode(mapper(x.val)) + elseif x isa GotoIfNot + return GotoIfNot(mapper(x.cond), x.dest) + else + return x + end +end + function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ()) deserialize_cycle(s, ci) @@ -1200,6 +1224,9 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) end end end + if current_module[] !== nothing + map!(x->symbol_to_globalref(x, current_module[]), code) + end _x = deserialize(s) have_debuginfo = _x isa Core.DebugInfo if have_debuginfo @@ -1248,6 +1275,9 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci.slottypes = deserialize(s) ci.rettype = deserialize(s) ci.parent = deserialize(s) + if format_version(s) < 29 && ci.parent isa MethodInstance && ci.parent.def isa Method + ci.nargs = ci.parent.def.nargs + end world_or_edges = deserialize(s) pre_13 = isa(world_or_edges, Union{UInt, Int}) if pre_13 @@ -1258,7 +1288,7 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci.min_world = deserialize(s)::UInt ci.max_world = deserialize(s)::UInt end - if format_version(s) >= 26 + if format_version(s) >= 29 ci.method_for_inference_limit_heuristics = deserialize(s) end end diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl index f1b83ca947c7e..e341c6e3eb9ec 100644 --- a/stdlib/Serialization/test/runtests.jl +++ b/stdlib/Serialization/test/runtests.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random, Serialization, Base64 +using Base.ScopedValues: with # Check that serializer hasn't gone out-of-frame @test Serialization.sertag(Symbol) == 1 @@ -661,3 +662,14 @@ end @test_broken isempty(undoc) @test undoc == [:AbstractSerializer, :Serializer] end + +# test method definitions from v1.11 +if Int === Int64 + let f_data = "N0pMGgQAAAAWAQEFdGh1bmsbFUbnFgEBBXRodW5rGxVG4DoWAQEGbWV0aG9kAQtmMTExX3RvXzExMhUABuABAAAA4BUAB+AAAAAAThVG4DQQAQxMaW5lSW5mb05vZGUfTptEH04BBE1haW5EAQ90b3AtbGV2ZWwgc2NvcGUBBG5vbmW+vhUAAd8V305GTk4JAQAAAAAAAAAJ//////////9MTExMAwADAAUAAAX//xYBAQZtZXRob2QsBwAWAlYkH06bRAEGVHlwZW9mLAcAFgNWJB9Om0QBBHN2ZWMo4iQfTptETxYBViQfTptEAQRzdmVjFgRWJB9Om0QBBHN2ZWMo4yjkGhfgAQRub25lFgMBBm1ldGhvZCwHACjlGxVG5AEBXhYDViQfTptElyQfTp5EAQNWYWzhFgFWKOEWBFYkH06eRAELbGl0ZXJhbF9wb3co4CXhKOI6KOMVAAbkAQAAAAEAAAABAAAAAQAAAAAAAADkFQAH5AAAAAAAAAAAAAAAAAAAAAAAAAAAThVG4DQsCwAfTgEETWFpbkQBBG5vbmUBBG5vbmW/vhUAAeGifRXhAAhORk5OCQEAAAAAAAAACf//////////TExMTAMAAwAFAAAF//86ThUABucBAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAAAAAOcVAAfnAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABOFUbgNCwLAB9OAQRNYWluRCwNAAEEbm9uZb6+FQAB3xXfTkZOTgkBAAAAAAAAAAn//////////0xMTEwDAAMABQAABf//" + @eval Main function f111_to_112 end + Core.eval(Main, with(Serialization.current_module => Main) do + deserialize(IOBuffer(base64decode(f_data))) + end) + @test @invokelatest(Main.f111_to_112(16)) == 256 + end +end From 1d42e05a44bb1ed52ce2c0c90052436e3360685c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 30 Jul 2025 03:56:09 -0500 Subject: [PATCH 591/662] REPL: improve type inference for `maybe_spawn_cache_PATH` (#59144) --- stdlib/REPL/src/REPLCompletions.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 385d0fe720b46..3a441540c3620 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -334,19 +334,23 @@ PATH_cache_task::Union{Task,Nothing} = nothing PATH_cache_condition::Union{Threads.Condition, Nothing} = nothing # used for sync in tests next_cache_update::Float64 = 0.0 function maybe_spawn_cache_PATH() - global PATH_cache_task, next_cache_update + global PATH_cache_task, PATH_cache_condition, next_cache_update + # Extract to local variables to enable flow-sensitive type inference for these global variables + PATH_cache_task_local = PATH_cache_task + PATH_cache_condition_local = PATH_cache_condition @lock PATH_cache_lock begin - PATH_cache_task isa Task && !istaskdone(PATH_cache_task) && return + PATH_cache_task_local isa Task && !istaskdone(PATH_cache_task_local) && return time() < next_cache_update && return PATH_cache_task = Threads.@spawn begin REPLCompletions.cache_PATH() @lock PATH_cache_lock begin next_cache_update = time() + 10 # earliest next update can run is 10s after PATH_cache_task = nothing # release memory when done - PATH_cache_condition !== nothing && notify(PATH_cache_condition) + PATH_cache_condition_local !== nothing && notify(PATH_cache_condition_local) end end - Base.errormonitor(PATH_cache_task) + PATH_cache_task_local = PATH_cache_task + Base.errormonitor(PATH_cache_task_local) end end From c230f9fe8d52aa391f34780824a5cbc0d1279ae2 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 30 Jul 2025 06:29:37 -0500 Subject: [PATCH 592/662] iterator: improve type inference for `Filter` (#59142) --- base/iterators.jl | 11 ++++++++--- test/iterators.jl | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/base/iterators.jl b/base/iterators.jl index add4c4c0e7bed..9812feb2d62cd 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -534,10 +534,15 @@ filter(flt, itr) = Filter(flt, itr) function iterate(f::Filter, state...) y = iterate(f.itr, state...) while y !== nothing - if f.flt(y[1]) - return y + v, s = y + if f.flt(v) + if y isa Tuple{Any,Any} + return (v, s) # incorporate type information that may be improved by user-provided `f.flt` + else + return y + end end - y = iterate(f.itr, y[2]) + y = iterate(f.itr, s) end nothing end diff --git a/test/iterators.jl b/test/iterators.jl index cebabf9cc07fc..11e7b3756fb3e 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -1201,3 +1201,8 @@ end @testset "Iterators docstrings" begin @test isempty(Docs.undocumented_names(Iterators)) end + +# Filtered list comprehension (`Filter` construct) type inference +@test Base.infer_return_type((Vector{Any},)) do xs + [x for x in xs if x isa Int] +end == Vector{Int} From 21d15ede0729a810458e2045f224e2e8a7db92e8 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Wed, 30 Jul 2025 10:10:02 -0400 Subject: [PATCH 593/662] Refactor stacktrace processing and display pipeline for more clarity and less redundancy (#55841) The current stacktrace display pipeline has 2 different ways to show repetition: image And each of these uses two separate printing paths, leading to other inconsistencies, like sometimes top-level scope is shown and sometimes it isn't, and sometimes the frame counter includes the hidden frames and sometimes it doesn't. I merged the two pathways and display the cycles more nicely using line characters, and extracted some of the filtering into separate functions. I also added a comment to document the stacktrace processing pipeline. ~~Keeping as a draft to look at CI results and get feedback.~~ Current state of PR: image Original idea: image --- base/errorshow.jl | 361 +++++++++++++++++++++++---------------- base/task.jl | 2 +- stdlib/REPL/test/repl.jl | 5 +- test/errorshow.jl | 107 +++++++++++- test/stacktraces.jl | 4 +- 5 files changed, 319 insertions(+), 160 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 7a25c561aa9e9..1ae98378ff542 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -658,112 +658,146 @@ const update_stackframes_callback = Ref{Function}(identity) const STACKTRACE_MODULECOLORS = Iterators.Stateful(Iterators.cycle([:magenta, :cyan, :green, :yellow])) const STACKTRACE_FIXEDCOLORS = IdDict(Base => :light_black, Core => :light_black) -function show_full_backtrace(io::IO, trace::Vector; print_linebreaks::Bool, prefix=nothing) - num_frames = length(trace) - ndigits_max = ndigits(num_frames) - - println(io) - prefix === nothing || print(io, prefix) - println(io, "Stacktrace:") - - for (i, (frame, n)) in enumerate(trace) - print_stackframe(io, i, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) - if i < num_frames - println(io) - print_linebreaks && println(io) - end - end -end - const BIG_STACKTRACE_SIZE = 50 # Arbitrary constant chosen here -function show_reduced_backtrace(io::IO, t::Vector; prefix=nothing) +function _backtrace_find_and_remove_cycles(t) recorded_positions = IdDict{UInt, Vector{Int}}() #= For each frame of hash h, recorded_positions[h] is the list of indices i such that hash(t[i-1]) == h, ie the list of positions in which the frame appears just before. =# + max_nested_cycles = 0 displayed_stackframes = [] - repeated_cycle = Tuple{Int,Int,Int}[] - # First: line to introuce the "cycle repetition" message - # Second: length of the cycle - # Third: number of repetitions + repeated_cycles = Tuple{Int,Int,Int}[] + # First: index into `display_stackframes` to introuce the cycle bracket on + # Second: length of the cycle as a count in the trace + # Third: number of cycle repetitions + + t_curr = 1 frame_counter = 1 - while frame_counter < length(t) - (last_frame, n) = t[frame_counter] - frame_counter += 1 # Indicating the next frame - - current_hash = hash(last_frame) - positions = get(recorded_positions, current_hash, Int[]) - recorded_positions[current_hash] = push!(positions, frame_counter) - - repetitions = 0 - for index_p in length(positions)-1:-1:1 # More recent is more likely - p = positions[index_p] - cycle_length = frame_counter - p - i = frame_counter - j = p - while i < length(t) && t[i] == t[j] - i += 1 - j += 1 + + while t_curr ≤ length(t) + (last_frame, n) = t[t_curr] + current_hash = hash(t[t_curr]) + positions = get(recorded_positions, current_hash, Int[]) + + t_curr += 1 + recorded_positions[current_hash] = push!(positions, t_curr) + + # Check previous positions for cycles + ncycles = 0 + nnested_cycles = n > 0 + for k ∈ reverse(eachindex(positions))[2:end] # More recent is more likely + t_prev = positions[k] + t_cycle_length = t_curr - t_prev + + # walk trace at current and previous matching positions until matching stops + t_curr_end = t_curr + t_prev_end = t_prev + while t_curr_end < length(t) && t[t_curr_end] == t[t_prev_end] + t_curr_end += 1 + t_prev_end += 1 end - if j >= frame_counter-1 + + if t_prev_end ≥ t_curr - 1 #= At least one cycle repeated =# - repetitions = div(i - frame_counter + 1, cycle_length) - push!(repeated_cycle, (length(displayed_stackframes), cycle_length, repetitions)) - frame_counter += cycle_length * repetitions - 1 - break + ncycles = div(t_curr_end - t_prev + 1, t_cycle_length) + push!(repeated_cycles, (length(displayed_stackframes) - 1, t_cycle_length, ncycles)) + t_curr += t_cycle_length * (ncycles - 1) - 1 + nnested_cycles += 1 end end - if repetitions==0 + # ensure an outer cycle comes before a contained inner cycle + sort!(repeated_cycles, by = x -> (x[1], -x[2])) + max_nested_cycles = max(max_nested_cycles, nnested_cycles) + + if ncycles == 0 push!(displayed_stackframes, (last_frame, n)) end end + return displayed_stackframes, repeated_cycles, max_nested_cycles +end - try invokelatest(update_stackframes_callback[], displayed_stackframes) catch end +function _backtrace_print_repetition_closings!(io::IO, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix = nothing) + while !isempty(current_cycles) + start_line = current_cycles[end][1] + cycle_length = current_cycles[end][2] + end_line = start_line + cycle_length - 1 + repetitions = current_cycles[end][3] + frame_counter_advance = current_cycles[end][4] + i != end_line && break + + println(io) + prefix === nothing || print(io, prefix) + line_length = (max_nested_cycles - nactive_cycles) + ndigits_max + 2 + nactive_cycles -= 1 + printstyled(io, " ", "│" ^ nactive_cycles, "╰", "─" ^ (line_length); color = :light_black) + printstyled(io, " repeated $repetitions times"; color = :light_black, italic = true) + + pop!(current_cycles) + + if cycle_length > 1 + # adjust cycle_length in outer cycles to reflect displayed frames consumed by this inner cycle + for j ∈ eachindex(current_cycles) + current_cycles[j] = (current_cycles[j][1], current_cycles[j][2] - cycle_length * (repetitions - 1), current_cycles[j][3:4]...) + end + else + # adjust frame_counter_advance in outer cycles to reflect frames consumed by a single repeated frame + for j ∈ eachindex(current_cycles) + current_cycles[j] = (current_cycles[j][1:3]..., current_cycles[j][4] + (frame_counter_advance * (current_cycles[j][3] - 1))) + end + end + + frame_counter += frame_counter_advance + end + return frame_counter, nactive_cycles +end + +function show_processed_backtrace(io::IO, trace::Vector, num_frames::Int, repeated_cycles::Vector{NTuple{3, Int}}, max_nested_cycles::Int; print_linebreaks::Bool, prefix = nothing) println(io) prefix === nothing || print(io, prefix) println(io, "Stacktrace:") - ndigits_max = ndigits(length(t)) + ndigits_max = ndigits(num_frames) + + push!(repeated_cycles, (0,0,0)) # repeated_cycles is never empty - push!(repeated_cycle, (0,0,0)) # repeated_cycle is never empty frame_counter = 1 - for i in eachindex(displayed_stackframes) - (frame, n) = displayed_stackframes[i] - prefix === nothing || print(io, prefix) - print_stackframe(io, frame_counter, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) + current_cycles = NTuple{4, Int}[] # adding a value to track amount to advance frame_counter when cycle is closed - if i < length(displayed_stackframes) - println(io) - stacktrace_linebreaks() && println(io) - end + for i in eachindex(trace) + (frame, n) = trace[i] - while repeated_cycle[1][1] == i # never empty because of the initial (0,0,0) - cycle_length = repeated_cycle[1][2] - repetitions = repeated_cycle[1][3] - popfirst!(repeated_cycle) - prefix === nothing || print(io, prefix) - printstyled(io, - "--- the above ", cycle_length, " lines are repeated ", - repetitions, " more time", repetitions>1 ? "s" : "", " ---", color = :light_black) - if i < length(displayed_stackframes) - println(io) - stacktrace_linebreaks() && println(io) - end - frame_counter += cycle_length * repetitions + ncycle_starts = 0 + while repeated_cycles[1][1] == i + cycle = popfirst!(repeated_cycles) + push!(current_cycles, (cycle..., cycle[2] * (cycle[3] - 1))) + ncycle_starts += 1 end + if n > 1 + push!(current_cycles, (i, 1, n, n - 1)) + ncycle_starts += 1 + end + nactive_cycles = length(current_cycles) + + print_stackframe(io, frame_counter, frame, ndigits_max, max_nested_cycles, nactive_cycles, ncycle_starts, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) + + frame_counter, nactive_cycles = _backtrace_print_repetition_closings!(io, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix) frame_counter += 1 + + if i < length(trace) + println(io) + print_linebreaks && println(io) + end end end - # Print a stack frame where the module color is determined by looking up the parent module in # `modulecolordict`. If the module does not have a color, yet, a new one can be drawn # from `modulecolorcycler`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolordict, modulecolorcycler; prefix=nothing) +function print_stackframe(io, i, frame::StackFrame, ndigits_max::Int, max_nested_cycles::Int, nactive_cycles::Int, ncycle_starts::Int, modulecolordict, modulecolorcycler; prefix = nothing) m = Base.parentmodule(frame) modulecolor = if m !== nothing m = parentmodule_before_main(m) @@ -771,7 +805,7 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec else :default end - print_stackframe(io, i, frame, n, ndigits_max, modulecolor; prefix) + print_stackframe(io, i, frame, ndigits_max, max_nested_cycles, nactive_cycles, ncycle_starts, modulecolor; prefix) end # Gets the topmost parent module that isn't Main @@ -786,7 +820,7 @@ end parentmodule_before_main(x) = parentmodule_before_main(parentmodule(x)) # Print a stack frame where the module color is set manually with `modulecolor`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolor; prefix=nothing) +function print_stackframe(io, i, frame::StackFrame, ndigits_max::Int, max_nested_cycles::Int, nactive_cycles::Int, ncycle_starts::Int, modulecolor; prefix = nothing) file, line = string(frame.file), frame.line # Used by the REPL to make it possible to open @@ -798,22 +832,29 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec inlined = getfield(frame, :inlined) modul = parentmodule(frame) - digit_align_width = ndigits_max + 2 + digit_align_width = ndigits_max + 2 + max_nested_cycles - nactive_cycles - # frame number + # repeated section bracket line 1 prefix === nothing || print(io, prefix) - print(io, " ", lpad("[" * string(i) * "]", digit_align_width)) + print(io, " ") + printstyled(io, "├" ^ (nactive_cycles - ncycle_starts); color = :light_black) + printstyled(io, "┌" ^ ncycle_starts; color = :light_black) + + # frame number + print(io, lpad("[" * string(i) * "]", digit_align_width)) print(io, " ") + # func name and arguments StackTraces.show_spec_linfo(IOContext(io, :backtrace=>true), frame) - if n > 1 - printstyled(io, " (repeats $n times)"; color=Base.warn_color(), bold=true) - end println(io) + # repeated section bracket line 2 prefix === nothing || print(io, prefix) + print(io, " ") + printstyled(io, "│" ^ nactive_cycles; color = :light_black) + # @ Module path / file : line - print_module_path_file(io, modul, file, line; modulecolor, digit_align_width) + print_module_path_file(io, modul, file, line; modulecolor, digit_align_width = digit_align_width - 1) # inlined printstyled(io, inlined ? " [inlined]" : "", color = :light_black) @@ -840,42 +881,112 @@ function print_module_path_file(io, modul, file, line; modulecolor = :light_blac printstyled(io, basename(file), ":", line; color = :light_black, underline = true) end -function show_backtrace(io::IO, t::Vector; prefix=nothing) +#= + +Stacktrace processing pipeline: +1. Raw traces extracted with `backtrace` or `catch_backtrace` as vector of instruction pointers. +2. IP traces converted to frames with `stacktrace`, which may or may not include C frames. +3. Originator trims frames related to itself (e.g. REPL removes REPL-specific frames) + - CapturedException only keeps a limit of 100 frames by processing before display +4. `process_backtrace` filters a trace for internal implementation or redundant frames and summarizes repeated single frames: + - `kwcall` frames removed + - `include`-related stack frames removed + - Some frames that have the same location info are merged + - Repeated frames are removed and summarized with a count + - Output is an Any[] containing (StackFrame, count) tuple elements and this form is exposed to e.g. Revise +5. If a trace is too long, cycles are identified and summarized +6. `update_stackframes_callback[]` provides e.g. Revise an opportunity to edit line info + +=# + +function show_backtrace(io::IO, t::Vector; prefix = nothing) if haskey(io, :last_shown_line_infos) empty!(io[:last_shown_line_infos]) end - # t is a pre-processed backtrace (ref #12856) + # Process backtrace if it has not yet been. A processed backtrace is a Vector{Any} + # with elements of type Tuple{StackFrame, Int}. (ref #12856) if t isa Vector{Any} && (length(t) == 0 || t[1] isa Tuple{StackFrame,Int}) filtered = t else - filtered = process_backtrace(t) + # t is a raw trace requiring lookup + if t isa Vector{<:Union{Base.InterpreterIP,Ptr{Cvoid}}} + frametrace = stacktrace(t) + else + frametrace = t + end + filtered = process_backtrace(frametrace) end isempty(filtered) && return - if length(filtered) == 1 && StackTraces.is_top_level_frame(filtered[1][1]) + nframes = sum(last(x) for x ∈ filtered) + + # don't show a single top-level frame with no location info + if nframes == 1 && StackTraces.is_top_level_frame(filtered[1][1]) f = filtered[1][1]::StackFrame if f.line == 0 && f.file === :var"" - # don't show a single top-level frame with no location info return end end + # Find repeated cycles if trace is too long if length(filtered) > BIG_STACKTRACE_SIZE - show_reduced_backtrace(IOContext(io, :backtrace => true), filtered; prefix) - return + filtered, repeated_cycles, max_nested_cycles = _backtrace_find_and_remove_cycles(filtered) else - try invokelatest(update_stackframes_callback[], filtered) catch end - # process_backtrace returns a Vector{Tuple{Frame, Int}} - show_full_backtrace(io, filtered; print_linebreaks = stacktrace_linebreaks(), prefix) + repeated_cycles = NTuple{3, Int}[] + max_nested_cycles = any(x -> last(x) > 1, filtered) ? 1 : 0 end + + # Allow external code to edit information in the frames (e.g. line numbers with Revise) + try invokelatest(update_stackframes_callback[], filtered) catch end + + show_processed_backtrace(IOContext(io, :backtrace => true), filtered, nframes, repeated_cycles, max_nested_cycles; print_linebreaks = stacktrace_linebreaks(), prefix) nothing end +function _backtrace_collapse_and_count_repeated_frames(frames::Vector{StackFrame}) + n = 0 + last_frame = StackTraces.UNKNOWN + tracecount = Any[] + for frame in frames + if frame.file != last_frame.file || frame.line != last_frame.line || frame.func != last_frame.func || frame.linfo !== last_frame.linfo + if n > 0 + push!(tracecount, (last_frame, n)) + end + n = 1 + last_frame = frame + else + n += 1 + end + end + if n > 0 + push!(tracecount, (last_frame, n)) + end + return tracecount +end + +function _backtrace_remove_kwcall_frames!(trace) + todelete = findall(trace) do (frame, _) + code = frame.linfo + if code isa MethodInstance + def = code.def + if def isa Method && def.name !== :kwcall && def.sig <: Tuple{typeof(Core.kwcall),NamedTuple,Any,Vararg} + # hide kwcall() methods, which are probably internal keyword sorter methods + # (we print the internal method instead, after demangling + # the argument list, since it has the right line number info) + return true + end + else + frame.func === :kwcall && return true + end + return false + end + deleteat!(trace, todelete) +end # For improved user experience, filter out frames for include() implementation # - see #33065. See also #35371 for extended discussion of internal frames. -function _simplify_include_frames(trace) +function _backtrace_simplify_include_frames!(trace) kept_frames = trues(length(trace)) first_ignored = nothing for i in length(trace):-1:1 @@ -907,11 +1018,11 @@ function _simplify_include_frames(trace) if first_ignored !== nothing kept_frames[1:first_ignored] .= false end - return trace[kept_frames] + keepat!(trace, kept_frames) end # Collapse frames that have the same location (in some cases) -function _collapse_repeated_frames(trace) +function _backtrace_collapse_repeated_locations!(trace) kept_frames = trues(length(trace)) last_frame = nothing for i in eachindex(trace) @@ -972,63 +1083,19 @@ function _collapse_repeated_frames(trace) end last_frame = frame end - return trace[kept_frames] + keepat!(trace, kept_frames) end -function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true) - n = 0 - last_frame = StackTraces.UNKNOWN - count = 0 - ret = Any[] - for i in eachindex(t) - lkups = t[i] - if lkups isa StackFrame - lkups = [lkups] - else - lkups = StackTraces.lookup(lkups) - end - for lkup in lkups - if lkup === StackTraces.UNKNOWN - continue - end - - if (lkup.from_c && skipC) - continue - end - if lkup.linfo isa Union{MethodInstance, CodeInstance} - def = StackTraces.frame_method_or_module(lkup) - if def isa Method && def.name !== :kwcall && def.sig <: Tuple{typeof(Core.kwcall),NamedTuple,Any,Vararg} - # hide kwcall() methods, which are probably internal keyword sorter methods - # (we print the internal method instead, after demangling - # the argument list, since it has the right line number info) - continue - end - elseif !lkup.from_c - lkup.func === :kwcall && continue - end - count += 1 - if count > limit - break - end +function process_backtrace(t::Vector{StackFrame}) + tracecount = _backtrace_collapse_and_count_repeated_frames(t) + process_backtrace(tracecount) +end - if lkup.file != last_frame.file || lkup.line != last_frame.line || lkup.func != last_frame.func || lkup.linfo !== last_frame.linfo - if n > 0 - push!(ret, (last_frame, n)) - end - n = 1 - last_frame = lkup - else - n += 1 - end - end - count > limit && break - end - if n > 0 - push!(ret, (last_frame, n)) - end - trace = _simplify_include_frames(ret) - trace = _collapse_repeated_frames(trace) - return trace +function process_backtrace(tracecount::Vector{Any}) + _backtrace_remove_kwcall_frames!(tracecount) + _backtrace_simplify_include_frames!(tracecount) + _backtrace_collapse_repeated_locations!(tracecount) + return tracecount end function show_exception_stack(io::IO, stack) diff --git a/base/task.jl b/base/task.jl index 1ae4fcc8d22f0..4f17330bb4455 100644 --- a/base/task.jl +++ b/base/task.jl @@ -14,7 +14,7 @@ struct CapturedException <: Exception # Typically the result of a catch_backtrace() # Process bt_raw so that it can be safely serialized - bt_lines = process_backtrace(bt_raw, 100) # Limiting this to 100 lines. + bt_lines = process_backtrace(stacktrace(bt_raw))[1:min(100, end)] # Limiting this to 100 lines. CapturedException(ex, bt_lines) end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index a04ade2a5e78f..5fcef91e2e0a8 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -244,8 +244,9 @@ fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_wri @test occursin("shell> ", s) # check for the echo of the prompt @test occursin("'", s) # check for the echo of the input s = readuntil(stdout_read, "\n\n") - @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || - startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] ")) + @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || + startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] "), + skip = Sys.iswindows() && Sys.WORD_SIZE == 32) write(stdin_write, "\b") wait(t) end diff --git a/test/errorshow.jl b/test/errorshow.jl index 42dc5ad31cb34..04ccc52a7df44 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -794,9 +794,11 @@ backtrace() Base.show_backtrace(io, bt) output = split(String(take!(io)), '\n') length(output) >= 8 || println(output) # for better errors when this fails - @test lstrip(output[3])[1:3] == "[1]" + @test lstrip(output[3])[1] == '┌' + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" @test occursin("g28442", output[3]) - @test lstrip(output[5])[1:3] == "[2]" + @test lstrip(output[5])[1] == '├' + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" @test occursin("f28442", output[5]) is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) if is_windows_32_bit @@ -804,18 +806,107 @@ backtrace() # https://github.com/JuliaLang/julia/issues/55900 # Instead of skipping them entirely, we skip one, and we loosen the other. - # Broken test: @test occursin("the above 2 lines are repeated 5000 more times", output[7]) - @test occursin("the above 2 lines are repeated ", output[7]) - @test occursin(" more times", output[7]) + # Broken test: @test occursin("repeated 5001 times", output[7]) + @test occursin("repeated ", output[7]) + @test occursin(" times", output[7]) # Broken test: @test lstrip(output[8])[1:7] == "[10003]" @test_broken false else - @test occursin("the above 2 lines are repeated 5000 more times", output[7]) + @test occursin("repeated 5001 times", output[7]) @test lstrip(output[8])[1:7] == "[10003]" end end +@testset "Long stacktrace printing - nested repeated single frame" begin + f28442a(n) = n ≤ 0 ? (return backtrace()) : g28442a(n - 1) + g28442a(n) = 80 > n > 20 ? h28442a(n - 1) : f28442a(n - 1) + h28442a(n) = n % 10 == 0 ? g28442a(n - 1) : h28442a(n - 1) + bt = f28442a(100) + io = IOBuffer() + Base.show_backtrace(io, bt) + output = split(String(take!(io)), '\n') + length(output) >= 21 || println(output) # for better errors when this fails + @test startswith(lstrip(output[3]), "┌ ") + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" + @test occursin("f28442a", output[3]) + @test startswith(lstrip(output[5]), "├ ") + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" + @test occursin("g28442a", output[5]) + + @test startswith(lstrip(output[8]), "┌┌ ") + @test occursin("h28442a", output[8]) + @test startswith(lstrip(output[11]), "├ ") + @test occursin("g28442a", output[11]) + + @test startswith(lstrip(output[14]), "┌ ") + @test occursin("f28442a", output[14]) + @test startswith(lstrip(output[16]), "├ ") + @test occursin("g28442a", output[16]) + + @test occursin("f28442a", output[19]) + + is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) + if is_windows_32_bit + # Assuming tests are broken on 32-bit Windows as above, no need to repeat loose tests here. + else + @test occursin("repeated 10 times", output[7]) + @test lstrip(lstrip(output[8])[7:end])[1:4] == "[21]" + @test occursin("repeated 9 times", output[10]) + @test lstrip(lstrip(output[11])[4:end])[1:4] == "[30]" + @test occursin("repeated 6 times", output[13]) + @test lstrip(lstrip(output[14])[4:end])[1:4] == "[81]" + @test lstrip(lstrip(output[16])[4:end])[1:4] == "[82]" + @test lstrip(output[19])[1:5] == "[101]" + @test lstrip(output[21])[1:5] == "[102]" + end +end + +@testset "Long stacktrace printing - nested cycles" begin + f28442b(n) = n ≤ 0 ? (return backtrace()) : g28442b(n - 1) + g28442b(n) = 80 > n > 60 || 40 > n > 20 ? h28442b(n - 1) : f28442b(n - 1) + h28442b(n) = g28442b(n - 1) + bt = f28442b(100) + io = IOBuffer() + Base.show_backtrace(io, bt) + output = split(String(take!(io)), '\n') + length(output) >= 21 || println(output) # for better errors when this fails + @test startswith(lstrip(output[3]), "┌ ") + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" + @test occursin("f28442b", output[3]) + @test startswith(lstrip(output[5]), "├ ") + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" + @test occursin("g28442b", output[5]) + + @test startswith(lstrip(output[8]), "┌┌ ") + @test occursin("h28442b", output[8]) + @test startswith(lstrip(output[10]), "├├ ") + @test occursin("g28442b", output[10]) + + @test startswith(lstrip(output[13]), "├┌ ") + @test occursin("f28442b", output[13]) + @test startswith(lstrip(output[15]), "├├ ") + @test occursin("g28442b", output[15]) + + @test occursin("f28442b", output[19]) + + is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) + if is_windows_32_bit + # Assuming tests are broken on 32-bit Windows as above, no need to repeat loose tests here. + else + @test occursin("repeated 10 times", output[7]) + @test lstrip(lstrip(output[8])[7:end])[1:4] == "[21]" + @test lstrip(lstrip(output[10])[7:end])[1:4] == "[22]" + @test occursin("repeated 10 times", output[12]) + @test lstrip(lstrip(output[13])[7:end])[1:4] == "[41]" + @test lstrip(lstrip(output[15])[7:end])[1:4] == "[42]" + @test occursin("repeated 10 times", output[17]) + @test occursin("repeated 2 times", output[18]) + @test lstrip(output[19])[1:5] == "[101]" + @test lstrip(output[21])[1:5] == "[102]" + end +end + @testset "Line number correction" begin getbt() = backtrace() bt = getbt() @@ -1091,7 +1182,7 @@ if (Sys.isapple() || Sys.islinux()) && Sys.ARCH === :x86_64 catch_backtrace() end bt_str = sprint(Base.show_backtrace, bt) - @test occursin(r"repeats \d+ times", bt_str) + @test occursin(r"repeated \d+ times", bt_str) end let bt = try @@ -1100,7 +1191,7 @@ if (Sys.isapple() || Sys.islinux()) && Sys.ARCH === :x86_64 catch_backtrace() end bt_str = sprint(Base.show_backtrace, bt) - @test occursin(r"the above 2 lines are repeated \d+ more times", bt_str) + @test occursin(r"repeated \d+ times", bt_str) end end end diff --git a/test/stacktraces.jl b/test/stacktraces.jl index f71cf9f616eaa..116e8dd3fe031 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -248,7 +248,7 @@ struct F49231{a,b,c,d,e,f,g} end stacktrace(catch_backtrace()) end str = sprint(Base.show_backtrace, st, context = (:limit=>true, :stacktrace_types_limited => Ref(false), :color=>true, :displaysize=>(50,105))) - @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n\e[90m") + @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n") st = try F49231{Vector,Val{'}'},Vector{Vector{Vector{Vector}}},Tuple{Int,Int,Int,Int,Int,Int,Int},Int,Int,Int}()(1,2,3) @@ -256,7 +256,7 @@ struct F49231{a,b,c,d,e,f,g} end stacktrace(catch_backtrace()) end str = sprint(Base.show_backtrace, st, context = (:limit=>true, :stacktrace_types_limited => Ref(false), :color=>true, :displaysize=>(50,132))) - @test contains(str, "[2] \e[0m\e[1m(::$F49231{Vector, Val{…}, Vector{…}, NTuple{…}, $Int, $Int, $Int})\e[22m\e[0m\e[1m(\e[22m\e[90ma\e[39m::\e[0m$Int, \e[90mb\e[39m::\e[0m$Int, \e[90mc\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n\e[90m") + @test contains(str, "[2] \e[0m\e[1m(::$F49231{Vector, Val{…}, Vector{…}, NTuple{…}, $Int, $Int, $Int})\e[22m\e[0m\e[1m(\e[22m\e[90ma\e[39m::\e[0m$Int, \e[90mb\e[39m::\e[0m$Int, \e[90mc\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n") end @testset "Base.StackTraces docstrings" begin From ebf55c0f2a85b2892104f27afb4d825feda2e8c6 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:02:43 -0700 Subject: [PATCH 594/662] Use ios to support ridiculous DL_LOAD_PATH length (#59150) Use `ios_t` as a string builder for library paths to support really long `DL_LOAD_PATH` entries. `jl_load_dynamic_library` can't use the Julia GC to allocate because we're called from `JuliaOJIT::DLSymOptimizer::lookup` with `symbols_mutex` held. Fixes #59130. --- src/dlload.c | 58 ++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/dlload.c b/src/dlload.c index 2c7ee08229394..8f56ab65c37d9 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -68,8 +68,6 @@ const char *jl_crtdll_name = CRTDLL_BASENAME ".dll"; #undef CRTDLL_BASENAME #endif -#define PATHBUF 4096 - #ifdef _OS_WINDOWS_ void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOINT { @@ -272,7 +270,7 @@ void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err) { JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, int throw_err) { - char path[PATHBUF], relocated[PATHBUF]; + ios_t path, relocated; int i; #ifdef _OS_WINDOWS_ int err; @@ -284,7 +282,6 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, // number of extensions to try — if modname already ends with the // standard extension, then we don't try adding additional extensions int n_extensions = endswith_extension(modname) ? 1 : N_EXTENSIONS; - int ret; // modname == NULL is a sentinel value requesting the handle of libjulia-internal if (modname == NULL) @@ -309,6 +306,9 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, } #endif + ios_mem(&path, IOS_INLSIZE); + ios_mem(&relocated, IOS_INLSIZE); + /* this branch permutes all base paths in DL_LOAD_PATH with all extensions note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH), @@ -325,43 +325,41 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, size_t j; for (j = 0; j < jl_array_nrows(DL_LOAD_PATH); j++) { char *dl_path = jl_string_data(jl_array_ptr_data(DL_LOAD_PATH)[j]); - size_t len = strlen(dl_path); - if (len == 0) + if (*dl_path == 0) continue; + ios_trunc(&relocated, 0); + // Is this entry supposed to be relative to the bindir? - if (len >= 16 && strncmp(dl_path, "@executable_path", 16) == 0) { - snprintf(relocated, PATHBUF, "%s%s", jl_options.julia_bindir, dl_path + 16); - len = len - 16 + strlen(jl_options.julia_bindir); + if (strncmp(dl_path, "@executable_path", 16) == 0) { + ios_printf(&relocated, "%s%s", jl_options.julia_bindir, dl_path + 16); } else { - strncpy(relocated, dl_path, PATHBUF); - relocated[PATHBUF-1] = '\0'; + ios_puts(dl_path, &relocated); } + ios_putc(0, &relocated); for (i = 0; i < n_extensions; i++) { + ios_trunc(&path, 0); const char *ext = extensions[i]; - path[0] = '\0'; - if (relocated[len-1] == PATHSEPSTRING[0]) - snprintf(path, PATHBUF, "%s%s%s", relocated, modname, ext); - else { - ret = snprintf(path, PATHBUF, "%s" PATHSEPSTRING "%s%s", relocated, modname, ext); - if (ret < 0) - jl_errorf("path is longer than %d\n", PATHBUF); - } + if (relocated.buf[relocated.bpos - 2] == PATHSEPSTRING[0]) + ios_printf(&path, "%s%s%s", relocated.buf, modname, ext); + else + ios_printf(&path, "%s" PATHSEPSTRING "%s%s", relocated.buf, modname, ext); + ios_putc(0, &path); #ifdef _OS_WINDOWS_ if (i == 0) { // LoadLibrary already tested the extensions, we just need to check the `stat` result #endif - handle = jl_dlopen(path, flags); + handle = jl_dlopen(path.buf, flags); if (handle && !(flags & JL_RTLD_NOLOAD)) jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, jl_pathname_for_handle(handle)); if (handle) - return handle; + goto success; #ifdef _OS_WINDOWS_ err = GetLastError(); } #endif // bail out and show the error if file actually exists - if (jl_stat(path, (char*)&stbuf) == 0) + if (jl_stat(path.buf, (char*)&stbuf) == 0) goto notfound; } } @@ -370,20 +368,21 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, // now fall back and look in default library paths, for all extensions for (i = 0; i < n_extensions; i++) { + ios_trunc(&path, 0); const char *ext = extensions[i]; - path[0] = '\0'; - snprintf(path, PATHBUF, "%s%s", modname, ext); - handle = jl_dlopen(path, flags); + ios_printf(&path, "%s%s", modname, ext); + ios_putc(0, &path); + handle = jl_dlopen(path.buf, flags); if (handle && !(flags & JL_RTLD_NOLOAD)) jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, jl_pathname_for_handle(handle)); if (handle) - return handle; + goto success; #ifdef _OS_WINDOWS_ err = GetLastError(); break; // LoadLibrary already tested the rest #else // bail out and show the error if file actually exists - if (jl_stat(path, (char*)&stbuf) == 0) + if (jl_stat(path.buf, (char*)&stbuf) == 0) break; #endif } @@ -396,10 +395,15 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, #else const char *reason = dlerror(); #endif + ios_close(&relocated); + ios_close(&path); jl_errorf("could not load library \"%s\"\n%s", modname, reason); } handle = NULL; +success: + ios_close(&relocated); + ios_close(&path); return handle; } From 0d4cdfc3c93f54d42308a1bb1bfa372f828626db Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 30 Jul 2025 13:43:57 -0400 Subject: [PATCH 595/662] build: add --fix option for check-whitespace (#59118) Help agents (and humans) to solve whitespace quicker. New options include to try fixing issues before checking for errors that cannot be automatically fixed (tabs in the middle of lines) and the ability to pass specific files as arguments (particularly for testing this script or for files that aren't yet in git). Partially written by Claude. --- AGENTS.md | 2 +- Makefile | 11 ++++++++++- contrib/check-whitespace.jl | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f8ba930f5a866..6e2c056ee5978 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ will not be reflected, unless you use `Revise`. ## For all changes -1. Run `make check-whitespace` before creating the PR to make sure you're not committing any whitespace errors. +1. Run `make fix-whitespace` before creating the PR to make sure you're not committing any whitespace errors. ## Building Julia diff --git a/Makefile b/Makefile index 40dd1565f8744..9d1351e41b2c7 100644 --- a/Makefile +++ b/Makefile @@ -136,6 +136,15 @@ else $(warn "Skipping whitespace check because git is unavailable") endif +fix-whitespace: +ifneq ($(NO_GIT), 1) + @# Append the directory containing the julia we just built to the end of `PATH`, + @# to give us the best chance of being able to run this check. + @PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) --fix +else + $(warn "Skipping whitespace fix because git is unavailable") +endif + release-candidate: release testall @$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/add_license_to_files.jl #add license headers @#Check documentation @@ -686,7 +695,7 @@ distcleanall: cleanall @-$(MAKE) -C $(BUILDROOT)/doc cleanall .FORCE: -.PHONY: .FORCE default debug release check-whitespace release-candidate \ +.PHONY: .FORCE default debug release check-whitespace fix-whitespace release-candidate \ julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \ julia-cli-release julia-cli-debug julia-src-release julia-src-debug \ julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \ diff --git a/contrib/check-whitespace.jl b/contrib/check-whitespace.jl index fd3106587fb0d..d7e04512e153d 100755 --- a/contrib/check-whitespace.jl +++ b/contrib/check-whitespace.jl @@ -32,10 +32,39 @@ allow_tabs(path) = endswith(path, "test/syntax.jl") || endswith(path, "test/triplequote.jl") -const errors = Set{Tuple{String,Int,String}}() - function check_whitespace() - for path in eachline(`git ls-files -- $patterns`) + # Get file list from ARGS if provided, otherwise use git ls-files + errors = Set{Tuple{String,Int,String}}() + files_to_check = filter(arg -> arg != "--fix", ARGS) + if isempty(files_to_check) + files_to_check = eachline(`git ls-files -- $patterns`) + end + + files_fixed = 0 + if "--fix" in ARGS + for path in files_to_check + content = newcontent = read(path, String) + isempty(content) && continue + if !allow_tabs(path) + tabpattern = r"^([ \t]+)"m => (x -> replace(x, r"((?: {4})*)( *\t)" => s"\1 ")) # Replace tab sequences at start of line after any number of 4-space groups + newcontent = replace(newcontent, tabpattern) + end + newcontent = replace(newcontent, + r"\s*$" => '\n', # Remove trailing whitespace and normalize line ending at eof + r"\s*?[\r\n]" => '\n', # Remove trailing whitespace and normalize line endings on each line + r"\xa0" => ' ' # Replace non-breaking spaces + ) + if content != newcontent + write(path, newcontent) + files_fixed += 1 + end + end + if files_fixed > 0 + println(stderr, "Fixed whitespace issues in $files_fixed files.") + end + end + + for path in files_to_check lineno = 0 non_blank = 0 From 884c5e3fff1f18c28364780a82edee2cb58e25c5 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Wed, 30 Jul 2025 18:28:40 +0000 Subject: [PATCH 596/662] bump slop for twiceprecision division (#59140) division (unlike the other intrinsics) can have table-maker dilema problems since the true result has infinite bits. Bumping the slop for it by 1 reduces failures from 80 in 10^10 to <1 in 10^11. I think accepting an extra bit of inaccuracy is reasonable here (the alternative would be to make the division do an extra loop and do some extra extended precision, which seems unlikely to be worth it. fixes https://github.com/JuliaLang/julia/issues/23497 Co-authored-by: oscarddssmith --- test/ranges.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ranges.jl b/test/ranges.jl index d5bc968a12399..9bad12e6692d2 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -230,7 +230,7 @@ end @test cmp_sn2(Tw(xw+yw), astuple(x+y)..., slopbits) @test cmp_sn2(Tw(xw-yw), astuple(x-y)..., slopbits) @test cmp_sn2(Tw(xw*yw), astuple(x*y)..., slopbits) - @test cmp_sn2(Tw(xw/yw), astuple(x/y)..., slopbits) + @test cmp_sn2(Tw(xw/yw), astuple(x/y)..., slopbits+1) # extra bit because division is hard y = rand(T) yw = widen(widen(y)) @test cmp_sn2(Tw(xw+yw), astuple(x+y)..., slopbits) From 9373bd1dddda8ac73535f7c5d60fc35741741027 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 30 Jul 2025 18:37:44 -0400 Subject: [PATCH 597/662] [test] precompile tests run on master 1, so may need explicit startup-file=no to pass (#59139) Our Makefile passes this flag, which then propagates, and if any workers are launched, they have this flag set also, but when running tests manually via runtests.jl, it may not get set, causing some spurious failures here. --- test/precompile.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index 522ee49efebb7..64797f51a863a 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1884,8 +1884,8 @@ end @testset "Precompile external abstract interpreter" begin dir = @__DIR__ - @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr)) - @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint1.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint2.jl`; dir); stdout, stderr)) end precompile_test_harness("Recursive types") do load_path @@ -2420,7 +2420,7 @@ precompile_test_harness("llvmcall validation") do load_path using LLVMCall LLVMCall.do_llvmcall2() """ - @test readchomp(`$(Base.julia_cmd()) --pkgimages=no -E $(testcode)`) == repr(UInt32(0)) + @test readchomp(`$(Base.julia_cmd()) --startup-file=no --pkgimages=no -E $(testcode)`) == repr(UInt32(0)) # Now the regular way @eval using LLVMCall invokelatest() do @@ -2500,7 +2500,7 @@ end # Issue #58841 - make sure we don't accidentally throw away code for inference let io = IOBuffer() - run(pipeline(`$(Base.julia_cmd()) --trace-compile=stderr -e 'f() = sin(1.) == 0. ? 1 : 0; exit(f())'`, stderr=io)) + run(pipeline(`$(Base.julia_cmd()) --startup-file=no --trace-compile=stderr -e 'f() = sin(1.) == 0. ? 1 : 0; exit(f())'`, stderr=io)) @test isempty(String(take!(io))) end From c9ef3a192d42e616aa98479554a52e7b8bbdbca6 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Wed, 30 Jul 2025 18:39:20 -0400 Subject: [PATCH 598/662] Add stack trace cycle display revamp to NEWS.md (#59156) --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index d46133b6244bc..71ebf8aaead0f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -75,6 +75,7 @@ Standard library changes #### REPL * The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). +* Display of repeated frames and cycles in stack traces has been improved by bracketing them in the trace and treating them consistently ([#55841]). #### Test From 577b8c5c9abcc69d210276c471a9796a8c0f9cee Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 30 Jul 2025 20:14:55 -0400 Subject: [PATCH 599/662] build: instruct AGENTS to run static analysis checks (#59137) --- AGENTS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 6e2c056ee5978..fc282bfdfbf3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,11 @@ If you made changes to the runtime (any files in `src/`), you will need to rebui julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes depending on your changes. +After `make` run these static analysis checks: + - `make -C src clang-sa-` (replace `` with the basename of the file you modified) + - `make -C src clang-sagc-` which may require adding JL_GC_PUSH arguments, or JL_GC_PROMISE_ROOTED statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put the promise as early in the code as is legal. + - `make -C src clang-tidy-` + ## Using Revise If you have made changes to files included in the system image (base/ or stdlib/), From 5ddd7215951d096d4bad0f5e153a18930ed39576 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 31 Jul 2025 02:11:35 -0400 Subject: [PATCH 600/662] build: Fix libssh2 source build (#59164) Addresses the same CI failure that https://github.com/JuliaLang/julia/pull/59141 was intended to address. The issue here is twofold. First, in a source build, we failed to install openssl header files. Second, even if we had done this, libssh needs to be explicitly told where to find openssl, otherwise it might prefer systme files. --- deps/libssh2.mk | 1 + deps/openssl.mk | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 3439ace10da16..05cc12b6e159b 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -19,6 +19,7 @@ ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF else LIBSSH2_OPTS += -DCRYPTO_BACKEND=OpenSSL -DENABLE_ZLIB_COMPRESSION=OFF +LIBSSH2_OPTS += -DOPENSSL_ROOT_DIR=$(build_prefix) endif ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) diff --git a/deps/openssl.mk b/deps/openssl.mk index 734ddb3274e78..ab6bd94921562 100644 --- a/deps/openssl.mk +++ b/deps/openssl.mk @@ -75,16 +75,18 @@ endif # Override bindir and only install runtime libraries, otherwise they'll go into build_depsbindir. OPENSSL_INSTALL = \ mkdir -p $2$$(build_shlibdir) && \ - $$(MAKE) -C $1 install_runtime_libs $$(MAKE_COMMON) bindir=$$(build_shlibdir) $3 DESTDIR="$2" + $$(MAKE) -C $1 install_dev $$(MAKE_COMMON) bindir=$$(build_shlibdir) $3 DESTDIR="$2" + +OPENSSL_POST_INSTALL := \ + $(WIN_MAKE_HARD_LINK) $(build_bindir)/libcrypto-*.dll $(build_bindir)/libcrypto.dll && \ + $(WIN_MAKE_HARD_LINK) $(build_bindir)/libssl-*.dll $(build_bindir)/libssl.dll && \ + $(INSTALL_NAME_CMD)libcrypto.$(SHLIB_EXT) $(build_shlibdir)/libcrypto.$(SHLIB_EXT) && \ + $(INSTALL_NAME_CMD)libssl.$(SHLIB_EXT) $(build_shlibdir)/libssl.$(SHLIB_EXT) && \ + $(INSTALL_NAME_CHANGE_CMD) $(build_shlibdir)/libcrypto.3.dylib @rpath/libcrypto.$(SHLIB_EXT) $(build_shlibdir)/libssl.$(SHLIB_EXT) $(eval $(call staged-install, \ openssl,openssl-$(OPENSSL_VER), \ - OPENSSL_INSTALL,,, \ - $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libcrypto-*.dll $(build_bindir)/libcrypto.dll && \ - $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libssl-*.dll $(build_bindir)/libssl.dll && \ - $$(INSTALL_NAME_CMD)libcrypto.$$(SHLIB_EXT) $$(build_shlibdir)/libcrypto.$$(SHLIB_EXT) && \ - $$(INSTALL_NAME_CMD)libssl.$$(SHLIB_EXT) $$(build_shlibdir)/libssl.$$(SHLIB_EXT) && \ - $$(INSTALL_NAME_CHANGE_CMD) $$(build_shlibdir)/libcrypto.3.dylib @rpath/libcrypto.$$(SHLIB_EXT) $$(build_shlibdir)/libssl.$$(SHLIB_EXT))) + OPENSSL_INSTALL,,,$(OPENSSL_POST_INSTALL))) clean-openssl: -rm -f $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-configured $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-compiled From f5c55e808cf8c3da77fbcc3db712ff2cb88d2c25 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 31 Jul 2025 02:11:51 -0400 Subject: [PATCH 601/662] misc: Don't get confused by GC running during compilation (#59163) Fixes https://github.com/JuliaLang/julia/issues/59162 --- test/misc.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/misc.jl b/test/misc.jl index 2b87a2ad26f0f..86bfc251c826f 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -499,12 +499,12 @@ begin local second = @capture_stdout @time @eval calldouble2(1.0) # these functions were not recompiled - local matches = collect(eachmatch(r"(\d+(?:\.\d+)?)%", first)) + local matches = collect(eachmatch(r"(\d+(?:\.\d+)?)% compilation", first)) @test length(matches) == 1 @test parse(Float64, matches[1][1]) > 0.0 @test parse(Float64, matches[1][1]) <= 100.0 - matches = collect(eachmatch(r"(\d+(?:\.\d+)?)%", second)) + matches = collect(eachmatch(r"(\d+(?:\.\d+)?)% compilation", second)) @test length(matches) == 1 @test parse(Float64, matches[1][1]) > 0.0 @test parse(Float64, matches[1][1]) <= 100.0 From 9a161192b21a70b0fb071f5d205a07b1ce6e0533 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 31 Jul 2025 12:11:43 -0400 Subject: [PATCH 602/662] replcompletions: Try to make the test more robust (#59166) This is an alternative to #59161, attempting to fix the same observed CI behavior. I don't think #59161 is the best way to fix it, as the point of these tests is to make sure that REPL completions looks up the PATH internally. Calling the path update function explicitly defeats that somewhat. The extra synchronization here to make this deterministic is messy, but I do think it makes the test closer to real-world usage. The core attempted fix here is to move the read of the PATH_ locals inside `maybe_spawn_cache_PATH` into the locked region. If they are not under the lock, they could be unconditionally overwritten by a second call to this function, causing issues in the state machine. I do not know whether this is the cause of the observed CI hangs, but it's worth fixing anyway. --- stdlib/REPL/src/REPLCompletions.jl | 22 ++++++++++++---------- stdlib/REPL/test/replcompletions.jl | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 3a441540c3620..4179819e47f1d 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -335,21 +335,23 @@ PATH_cache_condition::Union{Threads.Condition, Nothing} = nothing # used for syn next_cache_update::Float64 = 0.0 function maybe_spawn_cache_PATH() global PATH_cache_task, PATH_cache_condition, next_cache_update - # Extract to local variables to enable flow-sensitive type inference for these global variables - PATH_cache_task_local = PATH_cache_task - PATH_cache_condition_local = PATH_cache_condition @lock PATH_cache_lock begin + # Extract to local variables to enable flow-sensitive type inference for these global variables + PATH_cache_task_local = PATH_cache_task PATH_cache_task_local isa Task && !istaskdone(PATH_cache_task_local) && return time() < next_cache_update && return - PATH_cache_task = Threads.@spawn begin - REPLCompletions.cache_PATH() - @lock PATH_cache_lock begin - next_cache_update = time() + 10 # earliest next update can run is 10s after - PATH_cache_task = nothing # release memory when done - PATH_cache_condition_local !== nothing && notify(PATH_cache_condition_local) + PATH_cache_task = PATH_cache_task_local = Threads.@spawn begin + try + REPLCompletions.cache_PATH() + finally + @lock PATH_cache_lock begin + next_cache_update = time() + 10 # earliest next update can run is 10s after + PATH_cache_task = nothing # release memory when done + PATH_cache_condition_local = PATH_cache_condition + PATH_cache_condition_local !== nothing && notify(PATH_cache_condition_local) + end end end - PATH_cache_task_local = PATH_cache_task Base.errormonitor(PATH_cache_task_local) end end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 7b2b032af77d3..1b6dad2d17a00 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -1095,7 +1095,7 @@ function test_only_arm_cache_refresh() # force the next cache update to happen immediately REPL.REPLCompletions.next_cache_update = 0 end - return REPL.REPLCompletions.PATH_cache_condition + return nothing end function test_only_wait_cache_path_done() From 1f6eff183db0b947632f780b4acc440d9c41a6e2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 31 Jul 2025 19:58:09 -0400 Subject: [PATCH 603/662] Clarify and enhance confusing precompile test (#59170) --- test/precompile.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/precompile.jl b/test/precompile.jl index 64797f51a863a..d1ad0c459932d 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -691,7 +691,10 @@ precompile_test_harness(false) do dir error("the \"break me\" test failed") catch exc isa(exc, ErrorException) || rethrow() - occursin("ERROR: LoadError: break me", exc.msg) && rethrow() + # The LoadError shouldn't be surfaced but is printed to stderr, hence the `@test_warn` capture tests + occursin("LoadError: break me", exc.msg) && rethrow() + # The actual error that is thrown + occursin("Failed to precompile FooBar2", exc.msg) || rethrow() end # Test that trying to eval into closed modules during precompilation is an error From 3de5b9a66d83a9b4dab665d47c4e3afcf5388a63 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Thu, 31 Jul 2025 18:24:37 -0700 Subject: [PATCH 604/662] Fix desugaring of `const x::T = y` for complex `y` (#59155) Fix #59128 Assignment desugaring of `(const (= (|::| x T) rhs))` would pre-expand to, then re-expand `(const x ,(convert-for-type-decl rhs T))`, but two-arg (IR) const is expected to have a simple RHS---closure conversion doesn't recurse here (should it?), giving us partially-lowered IR, and hence our bug. Fix: Pre-expand to the one-arg AST const form `(const (= x ,(convert-for-type-decl rhs T)))` instead. This also gives us a `(latestworld)` we were missing before, so this lowering may have been originally intended. --- src/julia-syntax.scm | 6 +++--- test/syntax.jl | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 052e0000ebe87..69869e3e923ec 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1645,9 +1645,9 @@ (expand-forms ;; TODO: This behaviour (`const _:T = ...` does not call convert, ;; but still evaluates RHS) should be documented. - `(const ,(car e) ,(if (underscore-symbol? (car e)) - rhs - (convert-for-type-decl rhs T #t #f)))) + `(const (= ,(car e) ,(if (underscore-symbol? (car e)) + rhs + (convert-for-type-decl rhs T #t #f))))) (expand-forms `(block ,@(cdr e) ;; TODO: When x is a complex expression, this acts as a diff --git a/test/syntax.jl b/test/syntax.jl index 25a683a3b9b31..35fe05c9c425d 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4305,6 +4305,24 @@ end @test letf_57470(3) == 5 @test letT_57470 === Int64 +# Closure conversion should happen on const assignment rhs +module M59128 +using Test +const x0::Int = (()->1)() +global x1::Int = (()->1)() +global const x2::Int = (()->1)() +const global x3::Int = (()->1)() +@test x0 === x1 === x2 === x3 === 1 +let g = 1 + global x4::Vector{T} where {T<:Number} = let; (()->[g])(); end + const global x5::Vector{T} where {T<:Number} = let; (()->[g])(); end + global const x6::Vector{T} where {T<:Number} = let; (()->[g])(); end +end +@test x4 == x5 == x6 == [1] +const letT_57470{T} = (()->Int64)() +@test letT_57470 == Int64 +end + end # M57470_sub # lowering globaldecl with complex type From 6175761e581553eb1f23b05ebc65ab5a2f7f31a5 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Thu, 31 Jul 2025 21:35:12 -0500 Subject: [PATCH 605/662] Remove unnecessary `@inbounds` in `fill!` (#59174) LLVM can elide this and our effect analysis doesn't know that the `@inbounds` is safe here. ```julia-repl julia> function my_fill(x,y) v = Vector{typeof(x)}(undef, y) for i in eachindex(v) v[i] = y end v end my_fill (generic function with 1 method) julia> Base.infer_effects(fill, Tuple{Int, Int}) (!c,+e,!n,!t,+s,+m,!u,+o,+r) julia> Base.infer_effects(my_fill, Tuple{Int, Int}) (!c,+e,!n,!t,+s,+m,+u,+o,+r) julia> @b fill(10, 10) 12.152 ns (2 allocs: 144 bytes) julia> @b fill(10, 10) 12.179 ns (2 allocs: 144 bytes) julia> @b fill(10, 10) 12.187 ns (2 allocs: 144 bytes) julia> @b my_fill(10, 10) 11.806 ns (2 allocs: 144 bytes) julia> @b my_fill(10, 10) 12.152 ns (2 allocs: 144 bytes) julia> @b my_fill(10, 10) 12.190 ns (2 allocs: 144 bytes) ``` --- base/array.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 48f197d429740..d62436ca30497 100644 --- a/base/array.jl +++ b/base/array.jl @@ -336,7 +336,7 @@ function fill!(dest::Array{T}, x) where T end function _fill!(dest::Array{T}, x::T) where T for i in eachindex(dest) - @inbounds dest[i] = x + dest[i] = x end return dest end From 8ba3b11f9bb0898f39025e4c5909f26f953c01e0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Aug 2025 03:41:02 -0400 Subject: [PATCH 606/662] Try to remove `readavailable` sections from REPL precompile (#59179) The `readavailable` is problematic, because it can read partial error messages. The `readuntil`s should be safe because they properly fence between each prompt. If there is some other reason these readavailables are required, let's find out and replace them with proper readuntils if we can. Fixes #59099 (or tries to at least) --- stdlib/REPL/src/precompile.jl | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl index f9f411566a719..3b69484ee3996 100644 --- a/stdlib/REPL/src/precompile.jl +++ b/stdlib/REPL/src/precompile.jl @@ -24,7 +24,8 @@ function repl_workload() function check_errors(out) str = String(out) if occursin("ERROR:", str) && !any(occursin(e, str) for e in allowed_errors) - @error "Unexpected error (Review REPL precompilation with debug_output on):\n$str" + @error "Unexpected error (Review REPL precompilation with debug_output on):\n$str" exception=( + Base.PrecompilableError(), Base.backtrace()) exit(1) end end @@ -40,6 +41,7 @@ function repl_workload() # This is notified as soon as the first prompt appears repl_init_event = Base.Event() + repl_init_done_event = Base.Event() atreplinit() do repl # Main is closed so we can't evaluate in it, but atreplinit runs at @@ -48,6 +50,7 @@ function repl_workload() t = @async begin wait(repl_init_event) REPL.activate(REPL.Precompile; interactive_utils=false) + notify(repl_init_done_event) end Base.errormonitor(t) end @@ -80,6 +83,9 @@ function repl_workload() """ JULIA_PROMPT = "julia> " + # The help text for `reinterpret` has example `julia>` prompts in it, + # so use the longer prompt to avoid desychronization. + ACTIVATED_JULIA_PROMPT = "(REPL.Precompile) julia> " PKG_PROMPT = "pkg> " SHELL_PROMPT = "shell> " HELP_PROMPT = "help?> " @@ -142,18 +148,24 @@ function repl_workload() end schedule(repltask) # wait for the definitive prompt before start writing to the TTY - check_errors(readuntil(output_copy, JULIA_PROMPT)) + check_errors(readuntil(output_copy, JULIA_PROMPT, keep=true)) + + # Switch to the activated prompt + notify(repl_init_event) + wait(repl_init_done_event) + write(ptm, "\n") + # The prompt prints twice - once for the restatement of the input, once + # to indicate ready for the new prompt. + check_errors(readuntil(output_copy, ACTIVATED_JULIA_PROMPT, keep=true)) + check_errors(readuntil(output_copy, ACTIVATED_JULIA_PROMPT, keep=true)) + write(debug_output, "\n#### REPL STARTED ####\n") - sleep(0.01) - check_errors(readavailable(output_copy)) # Input our script precompile_lines = split(repl_script::String, '\n'; keepempty=false) curr = 0 for l in precompile_lines sleep(0.01) # try to let a bit of output accumulate before reading again curr += 1 - # consume any other output - bytesavailable(output_copy) > 0 && check_errors(readavailable(output_copy)) # push our input write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") # If the line ends with a CTRL_C, don't write an extra newline, which would @@ -166,13 +178,12 @@ function repl_workload() strbuf = "" while !eof(output_copy) strbuf *= String(readavailable(output_copy)) - occursin(JULIA_PROMPT, strbuf) && break + occursin(ACTIVATED_JULIA_PROMPT, strbuf) && break occursin(PKG_PROMPT, strbuf) && break occursin(SHELL_PROMPT, strbuf) && break occursin(HELP_PROMPT, strbuf) && break sleep(0.01) # try to let a bit of output accumulate before reading again end - notify(repl_init_event) check_errors(strbuf) end write(debug_output, "\n#### COMPLETED - Closing REPL ####\n") From f5b97a8d7582b7d6eff68aa3d1055291c7e5522f Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 1 Aug 2025 09:10:23 -0700 Subject: [PATCH 607/662] Add ThreadSanitzer hooks for jl_mutex_t (#59034) Lets us detect lock order inversions on `jl_mutex_t`s, races on mutex initialization, and shows the list of locked mutexes when some other ThreadSanitizer warning is shown. --- src/gc-mmtk.c | 4 ++++ src/julia_internal.h | 6 ----- src/julia_locks.h | 10 ++++++++ src/task.c | 4 ++++ src/threading.c | 57 +++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index edf74dcc65443..9e3dc6f8d87ed 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -3,6 +3,10 @@ #include "mmtkMutator.h" #include "threading.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + // File exists in the binding #include "mmtk.h" diff --git a/src/julia_internal.h b/src/julia_internal.h index 1312f3d5cb497..4bf6acd17f4c8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -99,12 +99,6 @@ static inline void msan_unpoison(const volatile void *a, size_t size) JL_NOTSAFE static inline void msan_allocated_memory(const volatile void *a, size_t size) JL_NOTSAFEPOINT {} static inline void msan_unpoison_string(const volatile char *a) JL_NOTSAFEPOINT {} #endif -#ifdef _COMPILER_TSAN_ENABLED_ -JL_DLLIMPORT void *__tsan_create_fiber(unsigned flags); -JL_DLLIMPORT void *__tsan_get_current_fiber(void); -JL_DLLIMPORT void __tsan_destroy_fiber(void *fiber); -JL_DLLIMPORT void __tsan_switch_to_fiber(void *fiber, unsigned flags); -#endif #ifndef _OS_WINDOWS_ #if defined(_CPU_ARM_) || defined(_CPU_PPC_) || defined(_CPU_WASM_) diff --git a/src/julia_locks.h b/src/julia_locks.h index 92d67b34b1692..a4b5fd96b8fb4 100644 --- a/src/julia_locks.h +++ b/src/julia_locks.h @@ -3,6 +3,10 @@ #ifndef JL_LOCKS_H #define JL_LOCKS_H +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -34,7 +38,13 @@ static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT JL_NOTSA // Hide this body from the analyzer, otherwise it complains that we're calling // a non-safepoint from this function. The 0 arguments guarantees that we do // not reach the safepoint, but the analyzer can't figure that out +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_write_reentrant); +#endif jl_mutex_wait(lock, 0); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, __tsan_mutex_write_reentrant, 1); +#endif #endif } diff --git a/src/task.c b/src/task.c index f8367a514572c..ffef3441be62f 100644 --- a/src/task.c +++ b/src/task.c @@ -37,6 +37,10 @@ #include "threading.h" #include "julia_assert.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef __cplusplus extern "C" { #endif diff --git a/src/threading.c b/src/threading.c index 6cd8eef558eb6..15763978b4611 100644 --- a/src/threading.c +++ b/src/threading.c @@ -8,6 +8,10 @@ #include "julia_internal.h" #include "julia_assert.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef USE_ITTAPI #include "ittapi/ittnotify.h" #endif @@ -931,7 +935,16 @@ void _jl_mutex_init(jl_mutex_t *lock, const char *name) JL_NOTSAFEPOINT { jl_atomic_store_relaxed(&lock->owner, (jl_task_t*)NULL); lock->count = 0; +#if defined(_COMPILER_TSAN_ENABLED_) && defined(ENABLE_TIMINGS) + __tsan_mutex_pre_divert(lock, 0); +#endif jl_profile_lock_init(lock, name); +#ifdef _COMPILER_TSAN_ENABLED_ +#ifdef ENABLE_TIMINGS + __tsan_mutex_post_divert(lock, 0); +#endif + __tsan_mutex_create(lock, __tsan_mutex_write_reentrant); +#endif } void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) @@ -941,11 +954,17 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) lock->count++; return; } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_divert(lock, 0); +#endif // Don't use JL_TIMING for instant acquires, results in large blowup of events jl_profile_lock_start_wait(lock); if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; jl_profile_lock_acquired(lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif return; } JL_TIMING(LOCK_SPIN, LOCK_SPIN); @@ -953,6 +972,9 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; jl_profile_lock_acquired(lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif return; } if (jl_running_under_rr(0)) { @@ -973,6 +995,9 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) jl_cpu_suspend(); owner = jl_atomic_load_relaxed(&lock->owner); } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif } static void jl_lock_frame_push(jl_task_t *self, jl_mutex_t *lock) @@ -998,23 +1023,43 @@ static void jl_lock_frame_pop(jl_task_t *self) void _jl_mutex_lock(jl_task_t *self, jl_mutex_t *lock) { +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_write_reentrant); +#endif JL_SIGATOMIC_BEGIN_self(); _jl_mutex_wait(self, lock, 1); jl_lock_frame_push(self, lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, __tsan_mutex_write_reentrant, 1); +#endif } int _jl_mutex_trylock_nogc(jl_task_t *self, jl_mutex_t *lock) { +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_try_lock | __tsan_mutex_write_reentrant); +#endif jl_task_t *owner = jl_atomic_load_acquire(&lock->owner); + int ret = 0; if (owner == self) { lock->count++; - return 1; + ret = 1; + goto done; } if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; - return 1; + ret = 1; + goto done; } - return 0; +done: +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, + __tsan_mutex_try_lock | + (ret ? 0 : __tsan_mutex_try_lock_failed) | + __tsan_mutex_write_reentrant, + 1); +#endif + return ret; } int _jl_mutex_trylock(jl_task_t *self, jl_mutex_t *lock) @@ -1030,6 +1075,9 @@ int _jl_mutex_trylock(jl_task_t *self, jl_mutex_t *lock) void _jl_mutex_unlock_nogc(jl_mutex_t *lock) { #ifndef __clang_gcanalyzer__ +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_unlock(lock, 0); +#endif assert(jl_atomic_load_relaxed(&lock->owner) == jl_current_task && "Unlocking a lock in a different thread."); if (--lock->count == 0) { @@ -1044,6 +1092,9 @@ void _jl_mutex_unlock_nogc(jl_mutex_t *lock) } jl_profile_lock_release_end(lock); } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_unlock(lock, 0); +#endif #endif } From 2eb2bf9cdf6ab65c304996f96f5d8e86373e0320 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 1 Aug 2025 12:35:12 -0400 Subject: [PATCH 608/662] convert Tarjan algorithms to iterative structure (#59143) This ensures that we do not use too much stack space. Conversion implemented by Claude. --- base/staticdata.jl | 429 ++++++++++++++++++++++++++--------------- src/gf.c | 336 ++++++++++++++++++++++---------- src/staticdata_utils.c | 282 +++++++++++++++++++++------ 3 files changed, 735 insertions(+), 312 deletions(-) diff --git a/base/staticdata.jl b/base/staticdata.jl index c6be79c1f6a15..a9ba58f3d82aa 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -8,6 +8,61 @@ const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) debug_method_invalidation(onoff::Bool) = _jl_debug_method_invalidation[] = onoff ? Any[] : nothing +# Immutable structs for different categories of state data +struct VerifyMethodInitialState + codeinst::CodeInstance + mi::MethodInstance + def::Method + callees::Core.SimpleVector +end + +struct VerifyMethodWorkState + depth::Int + cause::CodeInstance + recursive_index::Int + stage::Symbol +end + +struct VerifyMethodResultState + child_cycle::Int + result_minworld::UInt + result_maxworld::UInt +end + +# Container for all the work arrays +struct VerifyMethodWorkspace + # Arrays of different state categories + initial_states::Vector{VerifyMethodInitialState} + work_states::Vector{VerifyMethodWorkState} + result_states::Vector{VerifyMethodResultState} + + # Tarjan's algorithm working data + stack::Vector{CodeInstance} + visiting::IdDict{CodeInstance,Int} + + function VerifyMethodWorkspace() + new(VerifyMethodInitialState[], VerifyMethodWorkState[], VerifyMethodResultState[], + CodeInstance[], IdDict{CodeInstance,Int}()) + end +end + +# Helper functions to create default states +function VerifyMethodInitialState(codeinst::CodeInstance) + mi = get_ci_mi(codeinst) + def = mi.def::Method + callees = codeinst.edges + VerifyMethodInitialState(codeinst, mi, def, callees) +end + +function VerifyMethodWorkState(dummy_cause::CodeInstance) + VerifyMethodWorkState(0, dummy_cause, 1, :init_and_process_callees) +end + +function VerifyMethodResultState() + VerifyMethodResultState(0, 0, 0) +end + + # Restore backedges to external targets # `edges` = [caller1, ...], the list of worklist-owned code instances internally # `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally @@ -16,19 +71,18 @@ function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{ # to enable any applicable new codes backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) Base.scan_new_methods!(extext_methods, internal_methods, backedges_only) - stack = CodeInstance[] - visiting = IdDict{CodeInstance,Int}() - _insert_backedges(edges, stack, visiting) + workspace = VerifyMethodWorkspace() + _insert_backedges(edges, workspace) if ext_ci_list !== nothing - _insert_backedges(ext_ci_list, stack, visiting, #=external=#true) + _insert_backedges(ext_ci_list, workspace, #=external=#true) end end -function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, external::Bool=false) +function _insert_backedges(edges::Vector{Any}, workspace::VerifyMethodWorkspace, external::Bool=false) for i = 1:length(edges) codeinst = edges[i]::CodeInstance validation_world = get_world_counter() - verify_method_graph(codeinst, stack, visiting, validation_world) + verify_method_graph(codeinst, validation_world, workspace) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @@ -52,11 +106,13 @@ function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visi end end -function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, validation_world::UInt) - @assert isempty(stack); @assert isempty(visiting); - child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, validation_world) +function verify_method_graph(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) + @assert isempty(workspace.stack); @assert isempty(workspace.visiting); + @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) + child_cycle, minworld, maxworld = verify_method(codeinst, validation_world, workspace) @assert child_cycle == 0 - @assert isempty(stack); @assert isempty(visiting); + @assert isempty(workspace.stack); @assert isempty(workspace.visiting); + @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) nothing end @@ -109,175 +165,230 @@ end # - Visit the entire call graph, starting from edges[idx] to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable # and slightly modified with an early termination option once the computation reaches its minimum -function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, validation_world::UInt) - world = codeinst.min_world - let max_valid2 = codeinst.max_world - if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL - return 0, world, max_valid2 - end - end - mi = get_ci_mi(codeinst) - def = mi.def::Method - if needs_instrumentation(codeinst, mi, def, validation_world) - return 0, world, UInt(0) - end +function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) + # Initialize root state + push!(workspace.initial_states, VerifyMethodInitialState(codeinst)) + push!(workspace.work_states, VerifyMethodWorkState(codeinst)) + push!(workspace.result_states, VerifyMethodResultState()) - # Implicitly referenced bindings in the current module do not get explicit edges. - # If they were invalidated, they'll have the flag set in did_scan_source. If they weren't, they imply a minworld - # of `get_require_world`. In principle, this is only required for methods that do reference - # an implicit globalref. However, we already don't perform this validation for methods that - # don't have any (implicit or explicit) edges at all. The remaining corner case (some explicit, - # but no implicit edges) is rare and there would be little benefit to lower the minworld for it - # in any case, so we just always use `get_require_world` here. - local minworld::UInt, maxworld::UInt = Base.get_require_world(), validation_world - if haskey(visiting, codeinst) - return visiting[codeinst], minworld, maxworld - end - push!(stack, codeinst) - depth = length(stack) - visiting[codeinst] = depth - # TODO JL_TIMING(VERIFY_IMAGE, VERIFY_Methods) - callees = codeinst.edges - # Check for invalidation of the implicit edges from GlobalRef in the Method source - if (def.did_scan_source & 0x1) == 0x0 - backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) - Base.scan_new_method!(def, backedges_only) - end - if (def.did_scan_source & 0x4) != 0x0 - maxworld = 0 - invalidations = _jl_debug_method_invalidation[] - if invalidations !== nothing - push!(invalidations, def, "method_globalref", codeinst, nothing) - end - end - # verify current edges - if isempty(callees) - # quick return: no edges to verify (though we probably shouldn't have gotten here from WORLD_AGE_REVALIDATION_SENTINEL) - elseif maxworld == Base.get_require_world() - # if no new worlds were allocated since serializing the base module, then no new validation is worth doing right now either - else - matches = [] - j = 1 - while j ≤ length(callees) - local min_valid2::UInt, max_valid2::UInt - edge = callees[j] - @assert !(edge isa Method) # `Method`-edge isn't allowed for the optimized one-edge format - if edge isa CodeInstance - edge = get_ci_mi(edge) - end - if edge isa MethodInstance - sig = edge.specTypes - min_valid2, max_valid2 = verify_call(sig, callees, j, 1, world, true, matches) - j += 1 - elseif edge isa Int - sig = callees[j+1] - # Handle negative counts (fully_covers=false) - nmatches = abs(edge) - fully_covers = edge > 0 - min_valid2, max_valid2 = verify_call(sig, callees, j+2, nmatches, world, fully_covers, matches) - j += 2 + nmatches - edge = sig - elseif edge isa Core.Binding - j += 1 - min_valid2 = minworld - max_valid2 = maxworld - if !Base.binding_was_invalidated(edge) - if isdefined(edge, :partitions) - min_valid2 = edge.partitions.min_world - max_valid2 = edge.partitions.max_world - end - else - # Binding was previously invalidated - min_valid2 = 1 - max_valid2 = 0 - end - else - callee = callees[j+1] - if callee isa Core.MethodTable # skip the legacy edge (missing backedge) - j += 2 + current_depth = 1 # == length(workspace._states) == end + while true + # Get current state indices + initial = workspace.initial_states[current_depth] + work = workspace.work_states[current_depth] + + if work.stage == :init_and_process_callees + # Initialize state and handle early returns + world = initial.codeinst.min_world + let max_valid2 = initial.codeinst.max_world + if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL + workspace.result_states[current_depth] = VerifyMethodResultState(0, world, max_valid2) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) continue end - if callee isa CodeInstance - callee = get_ci_mi(callee) - end - if callee isa MethodInstance - meth = callee.def::Method - else - meth = callee::Method + end + + if needs_instrumentation(initial.codeinst, initial.mi, initial.def, validation_world) + workspace.result_states[current_depth] = VerifyMethodResultState(0, world, UInt(0)) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + continue + end + + minworld, maxworld = Base.get_require_world(), validation_world + + if haskey(workspace.visiting, initial.codeinst) + workspace.result_states[current_depth] = VerifyMethodResultState(workspace.visiting[initial.codeinst], minworld, maxworld) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + continue + end + + push!(workspace.stack, initial.codeinst) + depth = length(workspace.stack) + workspace.visiting[initial.codeinst] = depth + + # Check for invalidation of GlobalRef edges + if (initial.def.did_scan_source & 0x1) == 0x0 + backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) + Base.scan_new_method!(initial.def, backedges_only) + end + if (initial.def.did_scan_source & 0x4) != 0x0 + maxworld = 0 + invalidations = _jl_debug_method_invalidation[] + if invalidations !== nothing + push!(invalidations, initial.def, "method_globalref", initial.codeinst, nothing) end - min_valid2, max_valid2 = verify_invokesig(edge, meth, world, matches) - j += 2 end - if minworld < min_valid2 - minworld = min_valid2 + + # Process all non-CodeInstance edges + if !isempty(initial.callees) && maxworld != Base.get_require_world() + matches = [] + j = 1 + while j <= length(initial.callees) + local min_valid2::UInt, max_valid2::UInt + edge = initial.callees[j] + @assert !(edge isa Method) + + if edge isa CodeInstance + # Convert CodeInstance to MethodInstance for validation (like original) + edge = get_ci_mi(edge) + end + + if edge isa MethodInstance + sig = edge.specTypes + min_valid2, max_valid2 = verify_call(sig, initial.callees, j, 1, world, true, matches) + j += 1 + elseif edge isa Int + sig = initial.callees[j+1] + nmatches = abs(edge) + fully_covers = edge > 0 + min_valid2, max_valid2 = verify_call(sig, initial.callees, j+2, nmatches, world, fully_covers, matches) + j += 2 + nmatches + edge = sig + elseif edge isa Core.Binding + j += 1 + min_valid2 = minworld + max_valid2 = maxworld + if !Base.binding_was_invalidated(edge) + if isdefined(edge, :partitions) + min_valid2 = edge.partitions.min_world + max_valid2 = edge.partitions.max_world + end + else + min_valid2 = 1 + max_valid2 = 0 + end + else + callee = initial.callees[j+1] + if callee isa Core.MethodTable + j += 2 + continue + end + if callee isa CodeInstance + callee = get_ci_mi(callee) + end + if callee isa MethodInstance + meth = callee.def::Method + else + meth = callee::Method + end + min_valid2, max_valid2 = verify_invokesig(edge, meth, world, matches) + j += 2 + end + + if minworld < min_valid2 + minworld = min_valid2 + end + if maxworld > max_valid2 + maxworld = max_valid2 + end + invalidations = _jl_debug_method_invalidation[] + if max_valid2 ≠ typemax(UInt) && invalidations !== nothing + push!(invalidations, edge, "insert_backedges_callee", initial.codeinst, copy(matches)) + end + if max_valid2 == 0 && invalidations === nothing + break + end + end end - if maxworld > max_valid2 - maxworld = max_valid2 + + # Store computed minworld/maxworld in result state and transition to recursive phase + workspace.result_states[current_depth] = VerifyMethodResultState(depth, minworld, maxworld) + workspace.work_states[current_depth] = VerifyMethodWorkState(depth, work.cause, 1, :recursive_phase) + + elseif work.stage == :recursive_phase + # Find next CodeInstance edge that needs processing + recursive_index = work.recursive_index + found_child = false + while recursive_index ≤ length(initial.callees) + edge = initial.callees[recursive_index] + recursive_index += 1 + + if edge isa CodeInstance + # Create child state and add to stack + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, recursive_index, :recursive_phase) + push!(workspace.initial_states, VerifyMethodInitialState(edge)) + push!(workspace.work_states, VerifyMethodWorkState(edge)) + push!(workspace.result_states, VerifyMethodResultState()) + current_depth += 1 + found_child = true + break + end end - invalidations = _jl_debug_method_invalidation[] - if max_valid2 ≠ typemax(UInt) && invalidations !== nothing - push!(invalidations, edge, "insert_backedges_callee", codeinst, copy(matches)) + + if !found_child + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, recursive_index, :cleanup) end - if max_valid2 == 0 && invalidations === nothing - break + + elseif work.stage == :cleanup + # If we are the top of the current cycle, now mark all other parts of + # our cycle with what we found. + # Or if we found a failed edge, also mark all of the other parts of the + # cycle as also having a failed edge. + result = workspace.result_states[current_depth] + if result.result_maxworld == 0 || result.child_cycle == work.depth + while length(workspace.stack) ≥ work.depth + child = pop!(workspace.stack) + if result.result_maxworld ≠ 0 + @atomic :monotonic child.min_world = result.result_minworld + end + @atomic :monotonic child.max_world = result.result_maxworld + if result.result_maxworld == validation_world && validation_world == get_world_counter() + Compiler.store_backedges(child, child.edges) + end + @assert workspace.visiting[child] == length(workspace.stack) + 1 + delete!(workspace.visiting, child) + invalidations = _jl_debug_method_invalidation[] + if invalidations !== nothing && result.result_maxworld < validation_world + push!(invalidations, child, "verify_methods", work.cause) + end + end + + workspace.result_states[current_depth] = VerifyMethodResultState(0, result.result_minworld, result.result_maxworld) end - end - end - # verify recursive edges (if valid, or debugging) - cycle = depth - cause = codeinst - if maxworld ≠ 0 || _jl_debug_method_invalidation[] !== nothing - for j = 1:length(callees) - edge = callees[j] - if !(edge isa CodeInstance) - continue + + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + + elseif work.stage == :return_to_parent + # Pass results to parent and process them + pop!(workspace.initial_states) + pop!(workspace.work_states) + result = pop!(workspace.result_states) + current_depth -= 1 + if current_depth == 0 # Return results from the root call + return (result.child_cycle, result.result_minworld, result.result_maxworld) end - callee = edge - local min_valid2::UInt, max_valid2::UInt - child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, validation_world) - if minworld < min_valid2 - minworld = min_valid2 + # Propagate results to parent + parent_work = workspace.work_states[current_depth] + parent_result = workspace.result_states[current_depth] + callee = initial.codeinst + child_cycle, min_valid2, max_valid2 = result.child_cycle, result.result_minworld, result.result_maxworld + parent_cycle = parent_result.child_cycle + parent_minworld = parent_result.result_minworld + parent_maxworld = parent_result.result_maxworld + parent_cause = parent_work.cause + parent_stage = parent_work.stage + if parent_minworld < min_valid2 + parent_minworld = min_valid2 end - if minworld > max_valid2 + if parent_minworld > max_valid2 max_valid2 = 0 end - if maxworld > max_valid2 - cause = callee - maxworld = max_valid2 + if parent_maxworld > max_valid2 + parent_cause = callee + parent_maxworld = max_valid2 end if max_valid2 == 0 # found what we were looking for, so terminate early - break - elseif child_cycle ≠ 0 && child_cycle < cycle + # The parent should break out of its loop in :recursive_phase + parent_stage = :cleanup + elseif child_cycle ≠ 0 && child_cycle < parent_cycle # record the cycle will resolve at depth "cycle" - cycle = child_cycle + parent_cycle = child_cycle end + workspace.work_states[current_depth] = VerifyMethodWorkState(parent_work.depth, parent_cause, parent_work.recursive_index, parent_stage) + workspace.result_states[current_depth] = VerifyMethodResultState(parent_cycle, parent_minworld, parent_maxworld) end end - if maxworld ≠ 0 && cycle ≠ depth - return cycle, minworld, maxworld - end - # If we are the top of the current cycle, now mark all other parts of - # our cycle with what we found. - # Or if we found a failed edge, also mark all of the other parts of the - # cycle as also having a failed edge. - while length(stack) ≥ depth - child = pop!(stack) - if maxworld ≠ 0 - @atomic :monotonic child.min_world = minworld - end - @atomic :monotonic child.max_world = maxworld - if maxworld == validation_world && validation_world == get_world_counter() - Compiler.store_backedges(child, child.edges) - end - @assert visiting[child] == length(stack) + 1 - delete!(visiting, child) - invalidations = _jl_debug_method_invalidation[] - if invalidations !== nothing && maxworld < validation_world - push!(invalidations, child, "verify_methods", cause) - end - end - return 0, minworld, maxworld end function get_method_from_edge(@nospecialize t) diff --git a/src/gf.c b/src/gf.c index d454110313881..a83587cfde13f 100644 --- a/src/gf.c +++ b/src/gf.c @@ -4467,113 +4467,253 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio // * `*found_minmax`: whether there is a minmax method already found, so future fully_covers matches should be ignored // Outputs: // * `*has_ambiguity`: whether there are any ambiguities that mean the sort order is not exact +// Stack frame for iterative sort_mlmatches implementation +enum sort_state { + STATE_VISITING, // Initial visit and setup + STATE_PROCESSING_INTERFERENCES, // Processing interference loop + STATE_CHECK_COVERS, // Check coverage conditions + STATE_FINALIZE_SCC // SCC processing and cleanup +}; + +typedef struct { + size_t idx; // Current method match index + size_t interference_index; // Current position in interferences loop + size_t interference_count; // Total interferences count + size_t depth; // Stack depth when frame created + size_t cycle; // Cycle depth tracking + jl_method_match_t *matc; // Current method match + jl_method_t *m; // Current method + jl_value_t *ti; // Type intersection + int subt; // Subtype flag + jl_genericmemory_t *interferences; // Method interferences + int child_result; // Result from child recursive call + enum sort_state state; +} sort_stack_frame_t; + // Returns: // * -1: too many matches for lim, other outputs are undefined // * 0: the child(ren) have been added to the output // * 1+: the children are part of this SCC (up to this depth) -// TODO: convert this function into an iterative call, rather than recursive static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, arraylist_t *stack, arraylist_t *result, arraylist_t *recursion_stack, int lim, int include_ambiguous, int *has_ambiguity, int *found_minmax) { - size_t cycle = (size_t)visited->items[idx]; - if (cycle != 0) - return cycle - 1; // depth remaining - arraylist_push(stack, (void*)idx); - size_t depth = stack->len; - { // scope block - visited->items[idx] = (void*)(1 + depth); - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, idx); - jl_method_t *m = matc->method; - jl_value_t *ti = (jl_value_t*)matc->spec_types; - int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); - cycle = depth; - // Iterate over the interferences set to get the morespecific methods - for (size_t i = 0; i < interferences->length; i++) { - jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); - if (m2 == NULL) - continue; - int childidx = find_method_in_matches(t, m2); - if (childidx < 0 || (size_t)childidx == idx) - continue; - int child_cycle = (size_t)visited->items[childidx]; - if (child_cycle == 1) - continue; // already handled - if (child_cycle != 0 && child_cycle - 1 >= cycle) - continue; // already part of this cycle - if (method_in_interferences(m, m2)) - continue; - // m2 is morespecific, so attempt to visit it first - child_cycle = sort_mlmatches(t, childidx, visited, stack, result, recursion_stack, lim, include_ambiguous, has_ambiguity, found_minmax); - if (child_cycle == -1) - return -1; - // record the cycle will resolve at depth "cycle" - if (child_cycle && child_cycle < cycle) - cycle = child_cycle; - } - // There is some probability that this method is already fully covered - // now, and we can delete this vertex now without anyone noticing. - if (subt && *found_minmax) { - if (*found_minmax == 2) - visited->items[idx] = (void*)1; - } - else if (check_interferences_covers(m, ti, t, visited, recursion_stack)) { - visited->items[idx] = (void*)1; - } - else if (check_fully_ambiguous(m, ti, t, include_ambiguous, has_ambiguity)) { - visited->items[idx] = (void*)1; - } - - // If there were no cycles hit either, then we can potentially delete all of its edges too. - if ((size_t)visited->items[idx] == 1 && stack->len == depth) { - // n.b. cycle might be < depth, if we had a cycle with a child - // idx, but since we are on the top of the stack, nobody - // observed that and so we are content to ignore this - size_t childidx = (size_t)arraylist_pop(stack); - assert(childidx == idx); (void)childidx; - return 0; + // Use arraylist_t for explicit stack of processing frames + arraylist_t frame_stack; + arraylist_new(&frame_stack, 0); + + // Push initial frame + sort_stack_frame_t initial_frame = { + .idx = idx, + .interference_index = 0, + .interference_count = 0, + .depth = 0, + .cycle = 0, + .matc = NULL, + .m = NULL, + .ti = NULL, + .subt = 0, + .interferences = NULL, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(sort_stack_frame_t)), &initial_frame, sizeof(sort_stack_frame_t))); + + int final_result = 0; + + while (1) { + sort_stack_frame_t *current = (sort_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + JL_GC_PROMISE_ROOTED(current->m); + JL_GC_PROMISE_ROOTED(current->interferences); + JL_GC_PROMISE_ROOTED(current->ti); + + switch (current->state) { + case STATE_VISITING: { + size_t cycle = (size_t)visited->items[current->idx]; + if (cycle != 0) { + final_result = cycle - 1; + goto propagate_to_parent; + } + + arraylist_push(stack, (void*)current->idx); + current->depth = stack->len; + visited->items[current->idx] = (void*)(1 + current->depth); + current->matc = (jl_method_match_t*)jl_array_ptr_ref(t, current->idx); + current->m = current->matc->method; + current->ti = (jl_value_t*)current->matc->spec_types; + current->subt = current->matc->fully_covers != NOT_FULLY_COVERS; + current->interferences = jl_atomic_load_relaxed(¤t->m->interferences); + current->cycle = current->depth; + current->interference_count = current->interferences->length; + current->interference_index = 0; + current->state = STATE_PROCESSING_INTERFERENCES; + break; + } + + case STATE_PROCESSING_INTERFERENCES: { + // If we have a child result to process, handle it first + if (current->child_result != 0) { + if (current->child_result == -1) { + final_result = -1; + goto propagate_to_parent; + } + // record the cycle will resolve at depth "cycle" + if (current->child_result && current->child_result < current->cycle) + current->cycle = current->child_result; + current->child_result = 0; // Clear after processing + } + + // Process interferences iteratively + while (current->interference_index < current->interference_count) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(current->interferences, current->interference_index); + current->interference_index++; + + if (m2 == NULL) + continue; + + int childidx = find_method_in_matches(t, m2); + if (childidx < 0 || (size_t)childidx == current->idx) + continue; + + int child_cycle = (size_t)visited->items[childidx]; + if (child_cycle == 1) + continue; // already handled + if (child_cycle != 0 && child_cycle - 1 >= current->cycle) + continue; // already part of this cycle + if (method_in_interferences(current->m, m2)) + continue; + + // m2 is morespecific, so attempt to visit it first + if (child_cycle != 0) { + // Child already being processed, use cached result + int child_result = child_cycle - 1; + if (child_result == -1) { + final_result = -1; + goto propagate_to_parent; + } + if (child_result && child_result < current->cycle) + current->cycle = child_result; + } + else { + // Need to process child - push new frame and pause current processing + sort_stack_frame_t child_frame = { + .idx = childidx, + .interference_index = 0, + .interference_count = 0, + .depth = 0, + .cycle = 0, + .matc = NULL, + .m = NULL, + .ti = NULL, + .subt = 0, + .interferences = NULL, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(sort_stack_frame_t)), &child_frame, sizeof(sort_stack_frame_t))); + goto continue_main_loop; // Resume processing after child completes + } + } + + current->state = STATE_CHECK_COVERS; + break; + } + + case STATE_CHECK_COVERS: { + // There is some probability that this method is already fully covered + // now, and we can delete this vertex now without anyone noticing. + if (current->subt && *found_minmax) { + if (*found_minmax == 2) + visited->items[current->idx] = (void*)1; + } + else if (check_interferences_covers(current->m, current->ti, t, visited, recursion_stack)) { + visited->items[current->idx] = (void*)1; + } + else if (check_fully_ambiguous(current->m, current->ti, t, include_ambiguous, has_ambiguity)) { + visited->items[current->idx] = (void*)1; + } + + // If there were no cycles hit either, then we can potentially delete all of its edges too. + if ((size_t)visited->items[current->idx] == 1 && stack->len == current->depth) { + // n.b. cycle might be < depth, if we had a cycle with a child + // idx, but since we are on the top of the stack, nobody + // observed that and so we are content to ignore this + size_t childidx = (size_t)arraylist_pop(stack); + assert(childidx == current->idx); (void)childidx; + final_result = 0; + goto propagate_to_parent; + } + + if (current->cycle != current->depth) { + final_result = current->cycle; + goto propagate_to_parent; + } + + current->state = STATE_FINALIZE_SCC; + break; + } + + case STATE_FINALIZE_SCC: { + // If this is in an SCC group, do some additional checks before returning or setting has_ambiguity + if (current->depth != stack->len) { + int scc_count = 0; + for (size_t i = current->depth - 1; i < stack->len; i++) { + size_t childidx = (size_t)stack->items[i]; + if (visited->items[childidx] == (void*)1) + continue; + scc_count++; + } + if (scc_count > 1) + *has_ambiguity = 1; + } + + // copy this cycle into the results + for (size_t i = current->depth - 1; i < stack->len; i++) { + size_t childidx = (size_t)stack->items[i]; + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers != NOT_FULLY_COVERS; + if (subt && *found_minmax) + visited->items[childidx] = (void*)1; + if ((size_t)visited->items[childidx] == 1) + continue; + assert(visited->items[childidx] == (void*)(2 + i)); + visited->items[childidx] = (void*)1; + if (lim == -1 || result->len < lim) + arraylist_push(result, (void*)childidx); + else { + final_result = -1; + goto propagate_to_parent; + } + } + + // now finally cleanup the stack + while (stack->len >= current->depth) { + size_t childidx = (size_t)arraylist_pop(stack); + // always remove fully_covers matches after the first minmax ambiguity group is handled + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers == FULLY_COVERS; + if (subt && *found_minmax == 1) + *found_minmax = 2; + assert(visited->items[childidx] == (void*)1); + } + + final_result = 0; + goto propagate_to_parent; + } } - if (cycle != depth) - return cycle; - } - // If this is in an SCC group, do some additional checks before returning or setting has_ambiguity - if (depth != stack->len) { - int scc_count = 0; - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - if (visited->items[childidx] == (void*)1) - continue; - scc_count++; - } - if (scc_count > 1) - *has_ambiguity = 1; - } - // copy this cycle into the results - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - int subt = matc->fully_covers != NOT_FULLY_COVERS; - if (subt && *found_minmax) - visited->items[childidx] = (void*)1; - if ((size_t)visited->items[childidx] == 1) + + continue_main_loop: continue; - assert(visited->items[childidx] == (void*)(2 + i)); - visited->items[childidx] = (void*)1; - if (lim == -1 || result->len < lim) - arraylist_push(result, (void*)childidx); - else - return -1; - } - // now finally cleanup the stack - while (stack->len >= depth) { - size_t childidx = (size_t)arraylist_pop(stack); - // always remove fully_covers matches after the first minmax ambiguity group is handled - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - int subt = matc->fully_covers == FULLY_COVERS; - if (subt && *found_minmax == 1) - *found_minmax = 2; - assert(visited->items[childidx] == (void*)1); + + propagate_to_parent: + // Propagate result to parent if exists + free(arraylist_pop(&frame_stack)); + if (frame_stack.len == 0) + break; + sort_stack_frame_t *parent = (sort_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + parent->child_result = final_result; } - return 0; + assert(frame_stack.len == 0); + arraylist_free(&frame_stack); + return final_result; } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index e676eabbddad3..69b60a4314a5a 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -195,68 +195,240 @@ static int type_in_worklist(jl_value_t *v, jl_query_cache *cache) JL_NOTSAFEPOIN return result; } +// Stack frame for iterative has_backedge_to_worklist implementation +enum backedge_state { + STATE_VISITING, // Initial visit, setup phase + STATE_PROCESSING_EDGES, // Processing backedges loop + STATE_FINISHING // Cleanup and result propagation +}; + +typedef struct { + jl_method_instance_t *mi; // Current method instance + size_t edge_index; // Current position in backedges array + size_t backedges_len; // Total backedges count + jl_array_t *backedges; // Backedges array + int depth; // Stack depth when this frame was created + int cycle; // Cycle depth tracking + int found; // Result found flag + int child_result; // Result from child recursive call + enum backedge_state state; +} backedge_stack_frame_t; + // When we infer external method instances, ensure they link back to the // package. Otherwise they might be, e.g., for external macros. // Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_t *stack, jl_query_cache *query_cache) { - jl_module_t *mod = mi->def.module; - if (jl_is_method(mod)) - mod = ((jl_method_t*)mod)->module; - assert(jl_is_module(mod)); - uint8_t is_precompiled = jl_atomic_load_relaxed(&mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; - if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(mi->specTypes, query_cache)) { - return 1; - } - if (!mi->backedges) { - return 0; - } - void **bp = ptrhash_bp(visited, mi); - // HT_NOTFOUND: not yet analyzed - // HT_NOTFOUND + 1: no link back - // HT_NOTFOUND + 2: does link back - // HT_NOTFOUND + 3: does link back, and included in new_ext_cis already - // HT_NOTFOUND + 4 + depth: in-progress - int found = (char*)*bp - (char*)HT_NOTFOUND; - if (found) - return found - 1; - arraylist_push(stack, (void*)mi); - int depth = stack->len; - *bp = (void*)((char*)HT_NOTFOUND + 4 + depth); // preliminarily mark as in-progress - jl_array_t *backedges = jl_mi_get_backedges(mi); - size_t i = 0, n = jl_array_nrows(backedges); - int cycle = depth; - while (i < n) { - jl_code_instance_t *be; - i = get_next_edge(backedges, i, NULL, &be); - if (!be) - continue; - JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here - int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack, query_cache); - if (child_found == 1 || child_found == 2) { - // found what we were looking for, so terminate early - found = 1; - break; - } - else if (child_found >= 3 && child_found - 3 < cycle) { - // record the cycle will resolve at depth "cycle" - cycle = child_found - 3; - assert(cycle); + // Use arraylist_t for explicit stack of processing frames + arraylist_t frame_stack; + arraylist_new(&frame_stack, 0); + + // Push initial frame + backedge_stack_frame_t initial_frame = { + .mi = mi, + .edge_index = 0, + .backedges_len = 0, + .backedges = NULL, + .depth = 0, + .cycle = 0, + .found = 0, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(backedge_stack_frame_t)), &initial_frame, sizeof(backedge_stack_frame_t))); + + int final_result = 0; + while (1) { + backedge_stack_frame_t *current = (backedge_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + JL_GC_PROMISE_ROOTED(current->mi); + JL_GC_PROMISE_ROOTED(current->backedges); + + switch (current->state) { + case STATE_VISITING: { + jl_module_t *mod = current->mi->def.module; + if (jl_is_method(mod)) + mod = ((jl_method_t*)mod)->module; + assert(jl_is_module(mod)); + uint8_t is_precompiled = jl_atomic_load_relaxed(¤t->mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; + + if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(current->mi->specTypes, query_cache)) { + if (frame_stack.len > 1) { + final_result = 1; + goto propagate_to_parent; + } + current->found = 1; + // Continue to setup below, then go to finishing + } + else if (!current->mi->backedges) { + if (frame_stack.len > 1) { + final_result = 0; + goto propagate_to_parent; + } + current->found = 0; + // Setup minimal state for cleanup, skip backedges processing + arraylist_push(stack, (void*)current->mi); + current->depth = stack->len; + void **bp = ptrhash_bp(visited, current->mi); + *bp = (void*)((char*)HT_NOTFOUND + 4 + current->depth); + current->cycle = current->depth; + current->state = STATE_FINISHING; + break; + } + + void **bp = ptrhash_bp(visited, current->mi); + // HT_NOTFOUND: not yet analyzed + // HT_NOTFOUND + 1: no link back + // HT_NOTFOUND + 2: does link back + // HT_NOTFOUND + 3: does link back, and included in new_ext_cis already + // HT_NOTFOUND + 4 + depth: in-progress + int found = (char*)*bp - (char*)HT_NOTFOUND; + if (found) { + if (frame_stack.len > 1) { + final_result = found - 1; + goto propagate_to_parent; + } + current->found = found - 1; + } + + // Setup for processing + arraylist_push(stack, (void*)current->mi); + current->depth = stack->len; + *bp = (void*)((char*)HT_NOTFOUND + 4 + current->depth); // preliminarily mark as in-progress + current->backedges = jl_mi_get_backedges(current->mi); + current->backedges_len = current->backedges ? jl_array_nrows(current->backedges) : 0; + current->cycle = current->depth; + current->edge_index = 0; + // Don't reset current->found if it was already set by early termination logic above + if (current->found == 0) { + current->state = STATE_PROCESSING_EDGES; + } + else { + // Early termination case - skip processing and go straight to finishing + current->state = STATE_FINISHING; + } + break; + } + + case STATE_PROCESSING_EDGES: { + // If we have a child result to process, handle it first + if (current->child_result != 0) { + if (current->child_result == 1 || current->child_result == 2) { + // found what we were looking for, so terminate early + current->found = 1; + current->state = STATE_FINISHING; + break; + } + else if (current->child_result >= 3 && current->child_result - 3 < current->cycle) { + // record the cycle will resolve at depth "cycle" + current->cycle = current->child_result - 3; + assert(current->cycle); + } + current->child_result = 0; // Clear after processing + } + + // Process backedges iteratively + while (current->edge_index < current->backedges_len && current->backedges) { + jl_code_instance_t *be; + current->edge_index = get_next_edge(current->backedges, current->edge_index, NULL, &be); + if (!be) + continue; + JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here + + jl_method_instance_t *child_mi = jl_get_ci_mi(be); + + // Check if we need to recurse (push new frame) or handle result + jl_module_t *child_mod = child_mi->def.module; + if (jl_is_method(child_mod)) + child_mod = ((jl_method_t*)child_mod)->module; + assert(jl_is_module(child_mod)); + uint8_t child_is_precompiled = jl_atomic_load_relaxed(&child_mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; + + // Early termination check for child + if (child_is_precompiled || !jl_object_in_image((jl_value_t*)child_mod) || type_in_worklist(child_mi->specTypes, query_cache)) { + // found what we were looking for, so terminate early + current->found = 1; + break; + } + + if (!child_mi->backedges) { + // This child returns 0, continue with next edge + continue; + } + + void **child_bp = ptrhash_bp(visited, child_mi); + int child_found = (char*)*child_bp - (char*)HT_NOTFOUND; + if (child_found) { + int child_result = child_found - 1; + if (child_result == 1 || child_result == 2) { + // found what we were looking for, so terminate early + current->found = 1; + break; + } + else if (child_result >= 3 && child_result - 3 < current->cycle) { + // record the cycle will resolve at depth "cycle" + current->cycle = child_result - 3; + assert(current->cycle); + } + } + else { + // Need to process child - push new frame and pause current processing + backedge_stack_frame_t child_frame = { + .mi = child_mi, + .edge_index = 0, + .backedges_len = 0, + .backedges = NULL, + .depth = 0, + .cycle = 0, + .found = 0, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(backedge_stack_frame_t)), &child_frame, sizeof(backedge_stack_frame_t))); + goto continue_main_loop; // Resume processing after child completes + } + } + + current->state = STATE_FINISHING; + break; + } + + case STATE_FINISHING: { + if (!current->found && current->cycle != current->depth) { + final_result = current->cycle + 3; + goto propagate_to_parent; + } + + // If we are the top of the current cycle, now mark all other parts of + // our cycle with what we found. + // Or if we found a backedge, also mark all of the other parts of the + // cycle as also having an backedge. + while (stack->len >= current->depth) { + void *mi_ptr = arraylist_pop(stack); + void **bp = ptrhash_bp(visited, mi_ptr); + assert((char*)*bp - (char*)HT_NOTFOUND == 5 + stack->len); + *bp = (void*)((char*)HT_NOTFOUND + 1 + current->found); + } + + final_result = current->found; + goto propagate_to_parent; + } } + + continue_main_loop: + continue; + + propagate_to_parent: + // Propagate result to parent + free(arraylist_pop(&frame_stack)); + if (frame_stack.len == 0) + break; + backedge_stack_frame_t *parent = (backedge_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + parent->child_result = final_result; } - if (!found && cycle != depth) - return cycle + 3; - // If we are the top of the current cycle, now mark all other parts of - // our cycle with what we found. - // Or if we found a backedge, also mark all of the other parts of the - // cycle as also having an backedge. - while (stack->len >= depth) { - void *mi = arraylist_pop(stack); - bp = ptrhash_bp(visited, mi); - assert((char*)*bp - (char*)HT_NOTFOUND == 5 + stack->len); - *bp = (void*)((char*)HT_NOTFOUND + 1 + found); - } - return found; + // Cleanup remaining frames + assert(frame_stack.len == 0); + arraylist_free(&frame_stack); + return final_result; } // Given the list of CodeInstances that were inferred during the build, select From 38aff47dea4fd2c3b2092b5c8d462cd832f845f4 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 1 Aug 2025 12:48:10 -0500 Subject: [PATCH 609/662] Require `n::Integer` for `sizehint!(a, n)` (#59168) Currently we're a bit hit-and-miss about any type-restriction on the second argument to `sizehint!`: of 11 methods, 4 require `n::Integer` and the remainder are unrestricted. Arguably, our dispatch is "prettier" if methods differ only in their first argument and are consistent about type-restrictions on `n`. This imposes `::Integer` restrictions on all methods that specify a non-abstract type for the first argument. Only the fallback methods for `AbstractDict`, `AbstractSet`, and `AbstractArray` are left unrestricted. It also makes similar changes to a `rehash!` and `hashindex` method. On 1.12-rc1, JET shows 40 runtime dispatches for `report_opt(merge!, (AbstractDict, AbstractDict))`, most of which are triggered from a branch on `sizehint!(::WeakKeyDict, ::Any)`. This seems likely to silence most of those, but it's not possible to check due to JET currently throwing errors on master. Because of these unrestricted fallbacks, there is one funny thing which applies to both this PR and master: ```julia julia> v = [1, 2, 3]; julia> sizehint!(v, 7.0); # no-op ``` This is mitigated by the fact that `sizehint!` is just a hint. To me it still seems like it might be better to throw an error in such cases, but that is not implemented by this PR. --------- Co-authored-by: Shuhei Kadowaki --- base/dict.jl | 5 +++-- base/iddict.jl | 4 ++-- base/idset.jl | 2 +- base/set.jl | 2 +- base/weakkeydict.jl | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 68aa4f1a7a543..6cbf4429ca9e0 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -124,9 +124,10 @@ _shorthash7(hsh::UInt) = (hsh >> (8sizeof(UInt)-7))%UInt8 | 0x80 # hashindex (key, sz) - computes optimal position and shorthash7 # idx - optimal position in the hash table # sh::UInt8 - short hash (7 highest hash bits) -function hashindex(key, sz) +function hashindex(key, sz::Integer) + sz = Int(sz)::Int hsh = hash(key)::UInt - idx = (((hsh % Int) & (sz-1)) + 1)::Int + idx = ((hsh % Int) & (sz-1)) + 1 return idx, _shorthash7(hsh) end diff --git a/base/iddict.jl b/base/iddict.jl index b12fbfcca107f..ec5392cf7b5b8 100644 --- a/base/iddict.jl +++ b/base/iddict.jl @@ -58,12 +58,12 @@ IdDict(kv) = dict_with_eltype((K, V) -> IdDict{K, V}, kv, eltype(kv)) empty(d::IdDict, ::Type{K}, ::Type{V}) where {K, V} = IdDict{K,V}() -function rehash!(d::IdDict, newsz = length(d.ht)%UInt) +function rehash!(d::IdDict, newsz::Integer = length(d.ht)%UInt) d.ht = ccall(:jl_idtable_rehash, Memory{Any}, (Any, Csize_t), d.ht, newsz) d end -function sizehint!(d::IdDict, newsz) +function sizehint!(d::IdDict, newsz::Integer) newsz = _tablesz(newsz*2) # *2 for keys and values in same array oldsz = length(d.ht) # grow at least 25% diff --git a/base/idset.jl b/base/idset.jl index 963da8e7c0f92..4ad9426172909 100644 --- a/base/idset.jl +++ b/base/idset.jl @@ -78,7 +78,7 @@ pop!(s::IdSet, @nospecialize(x)) = _pop!(s, x) == -1 ? throw(KeyError(x)) : x pop!(s::IdSet, @nospecialize(x), @nospecialize(default)) = _pop!(s, x) == -1 ? default : x delete!(s::IdSet, @nospecialize(x)) = (_pop!(s, x); s) -function sizehint!(s::IdSet, newsz) +function sizehint!(s::IdSet, newsz::Integer) # TODO: grow/compact list and perform rehash, if profitable? # TODO: shrink? # s.list = resize(s.list, newsz) diff --git a/base/set.jl b/base/set.jl index 3c7dc9487ac8a..8b8f3d44603c8 100644 --- a/base/set.jl +++ b/base/set.jl @@ -169,7 +169,7 @@ copymutable(s::Set{T}) where {T} = Set{T}(s) # Set is the default mutable fall-back copymutable(s::AbstractSet{T}) where {T} = Set{T}(s) -sizehint!(s::Set, newsz; shrink::Bool=true) = (sizehint!(s.dict, newsz; shrink); s) +sizehint!(s::Set, newsz::Integer; shrink::Bool=true) = (sizehint!(s.dict, newsz; shrink); s) empty!(s::Set) = (empty!(s.dict); s) rehash!(s::Set) = (rehash!(s.dict); s) diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index 191cca485ab3c..1283dc9cbc8cb 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -76,7 +76,7 @@ function _cleanup_locked(h::WeakKeyDict) return h end -sizehint!(d::WeakKeyDict, newsz; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) +sizehint!(d::WeakKeyDict, newsz::Integer; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) empty(d::WeakKeyDict, ::Type{K}, ::Type{V}) where {K, V} = WeakKeyDict{K, V}() IteratorSize(::Type{<:WeakKeyDict}) = SizeUnknown() From 7cbea5fc047756fb7e9f5dee032a2a612ac383fb Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 1 Aug 2025 12:12:10 -0700 Subject: [PATCH 610/662] Use relaxed atomics to load/update jl_lineno and jl_filename (#58939) This is another small change to avoid ThreadSanitizer false positives when we run it on CI. --- src/interpreter.c | 2 +- src/julia_internal.h | 4 ++-- src/method.c | 6 +++--- src/module.c | 8 ++++---- src/signal-handling.c | 2 +- src/toplevel.c | 46 +++++++++++++++++++++---------------------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/interpreter.c b/src/interpreter.c index c4de34da37ce9..8b1b005349d9d 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -702,7 +702,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, s->locals[n - 1] = NULL; } else if (toplevel && jl_is_linenode(stmt)) { - jl_lineno = jl_linenode_line(stmt); + jl_atomic_store_relaxed(&jl_lineno, jl_linenode_line(stmt)); } else { eval_stmt_value(stmt, s); diff --git a/src/julia_internal.h b/src/julia_internal.h index 4bf6acd17f4c8..e93dae8312d51 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -424,8 +424,8 @@ extern _Atomic(jl_typemap_entry_t*) call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; void free_stack(void *stkbuf, size_t bufsz) JL_NOTSAFEPOINT; -JL_DLLEXPORT extern int jl_lineno; -JL_DLLEXPORT extern const char *jl_filename; +JL_DLLEXPORT extern _Atomic(int) jl_lineno; +JL_DLLEXPORT extern _Atomic(const char *) jl_filename; jl_value_t *jl_gc_small_alloc_noinline(jl_ptls_t ptls, int offset, int osize); diff --git a/src/method.c b/src/method.c index 46a4f5bc4d56c..459c8088ea0f1 100644 --- a/src/method.c +++ b/src/method.c @@ -720,7 +720,7 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t jl_code_instance_t *ci = NULL; JL_GC_PUSH5(&ex, &func, &uninferred, &ci, &kind); jl_task_t *ct = jl_current_task; - int last_lineno = jl_lineno; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); int last_in = ct->ptls->in_pure_callback; size_t last_age = ct->world_age; @@ -818,12 +818,12 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t } ct->ptls->in_pure_callback = last_in; - jl_lineno = last_lineno; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); ct->world_age = last_age; } JL_CATCH { ct->ptls->in_pure_callback = last_in; - jl_lineno = last_lineno; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); jl_rethrow(); } JL_GC_POP(); diff --git a/src/module.c b/src/module.c index 59968e2fd6cca..d7ba12609ee80 100644 --- a/src/module.c +++ b/src/module.c @@ -1145,8 +1145,8 @@ JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *var) return b && jl_binding_kind(bpart) == PARTITION_KIND_IMPORTED; } -extern const char *jl_filename; -extern int jl_lineno; +extern _Atomic(const char *) jl_filename; +extern _Atomic(int) jl_lineno; static char const dep_message_prefix[] = "_dep_message_"; @@ -1872,8 +1872,8 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_binding_dep_message(b); if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) { - if (jl_lineno != 0) { - jl_printf(JL_STDERR, " likely near %s:%d\n", jl_filename, jl_lineno); + if (jl_atomic_load_relaxed(&jl_lineno) != 0) { + jl_printf(JL_STDERR, " likely near %s:%d\n", jl_atomic_load_relaxed(&jl_filename), jl_atomic_load_relaxed(&jl_lineno)); } } diff --git a/src/signal-handling.c b/src/signal-handling.c index 6e19028ca7940..70299104d751e 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -637,7 +637,7 @@ void jl_critical_error(int sig, int si_code, bt_context_t *context, jl_task_t *c else jl_safe_printf("\n[%d] signal %d: %s\n", getpid(), sig, strsignal(sig)); } - jl_safe_printf("in expression starting at %s:%d\n", jl_filename, jl_lineno); + jl_safe_printf("in expression starting at %s:%d\n", jl_atomic_load_relaxed(&jl_filename), jl_atomic_load_relaxed(&jl_lineno)); if (context && ct) { // Must avoid extended backtrace frames here unless we're sure bt_data // is properly rooted. diff --git a/src/toplevel.c b/src/toplevel.c index 4628842619d05..ff5aabec748ec 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -26,9 +26,9 @@ extern "C" { #endif // current line number in a file -JL_DLLEXPORT int jl_lineno = 0; // need to update jl_critical_error if this is TLS +JL_DLLEXPORT _Atomic(int) jl_lineno = 0; // need to update jl_critical_error if this is TLS // current file name -JL_DLLEXPORT const char *jl_filename = "none"; // need to update jl_critical_error if this is TLS +JL_DLLEXPORT _Atomic(const char *) jl_filename = "none"; // need to update jl_critical_error if this is TLS htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; @@ -619,8 +619,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val *toplevel_filename = jl_symbol_name((jl_sym_t*)file); } // Not thread safe. For debugging and last resort error messages (jl_critical_error) only. - jl_filename = *toplevel_filename; - jl_lineno = *toplevel_lineno; + jl_atomic_store_relaxed(&jl_filename, *toplevel_filename); + jl_atomic_store_relaxed(&jl_lineno, *toplevel_lineno); return jl_nothing; } return jl_interpret_toplevel_expr_in(m, e, NULL, NULL); @@ -780,8 +780,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_module_t *m, jl_value_t *v) { - const char *filename = jl_filename; - int lineno = jl_lineno; + const char *filename = jl_atomic_load_relaxed(&jl_filename); + int lineno = jl_atomic_load_relaxed(&jl_lineno); return jl_toplevel_eval_flex(m, v, 1, 0, &filename, &lineno); } @@ -819,23 +819,23 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex) { jl_check_top_level_effect(m, "eval"); jl_value_t *v = NULL; - int last_lineno = jl_lineno; - const char *last_filename = jl_filename; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); + const char *last_filename = jl_atomic_load_relaxed(&jl_filename); jl_task_t *ct = jl_current_task; - jl_lineno = 1; - jl_filename = "none"; + jl_atomic_store_relaxed(&jl_lineno, 1); + jl_atomic_store_relaxed(&jl_filename, "none"); size_t last_age = ct->world_age; JL_TRY { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); v = jl_toplevel_eval(m, ex); } JL_CATCH { - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); jl_rethrow(); } - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); ct->world_age = last_age; assert(v); return v; @@ -867,12 +867,12 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, } jl_task_t *ct = jl_current_task; - int last_lineno = jl_lineno; - const char *last_filename = jl_filename; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); + const char *last_filename = jl_atomic_load_relaxed(&jl_filename); int lineno = 0; - jl_lineno = 0; + jl_atomic_store_relaxed(&jl_lineno, 0); const char *filename_str = jl_string_data(filename); - jl_filename = filename_str; + jl_atomic_store_relaxed(&jl_filename, filename_str); JL_TRY { size_t last_age = ct->world_age; @@ -882,7 +882,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, if (jl_is_linenode(expression)) { // filename is already set above. lineno = jl_linenode_line(expression); - jl_lineno = lineno; + jl_atomic_store_relaxed(&jl_lineno, lineno); continue; } expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0); @@ -893,16 +893,16 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, } JL_CATCH { result = jl_box_long(lineno); // (ab)use result to root error line - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); if (jl_loaderror_type == NULL) jl_rethrow(); else jl_rethrow_other(jl_new_struct(jl_loaderror_type, filename, result, jl_current_exception(ct))); } - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); JL_GC_POP(); return result; } From 9473ef73eccfe2485411ce140047e6285f3a5cad Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 2 Aug 2025 04:21:56 +0900 Subject: [PATCH 611/662] inference: avoid `LimitedAccuracy` within slot wrappers (#59182) I looked at all the usage sites of these constructors within the compiler, and fortunately it appears that none of the usage sites require that the return values of these constructors be objects of their respective types. So in cases where `LimitedAccuracy` is given as a wrapped element, these slot type refinements simply will not be performed. - fixes JuliaLang/julia#59004 --- Compiler/src/abstractinterpretation.jl | 2 +- Compiler/src/typelattice.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 100cdd97e511a..47e90e22c4d57 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3255,7 +3255,7 @@ function abstract_eval_copyast(interp::AbstractInterpreter, e::Expr, sstate::Sta return RTEffects(rt, Any, effects) end -function abstract_eval_isdefined_expr(interp::AbstractInterpreter, e::Expr, sstate::StatementState, +function abstract_eval_isdefined_expr(::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) sym = e.args[1] if isa(sym, SlotNumber) && sstate.vtypes !== nothing diff --git a/Compiler/src/typelattice.jl b/Compiler/src/typelattice.jl index 2bf4954343b37..aae0241d69aa1 100644 --- a/Compiler/src/typelattice.jl +++ b/Compiler/src/typelattice.jl @@ -40,6 +40,9 @@ struct Conditional isdefined::Bool=false) assert_nested_slotwrapper(thentype) assert_nested_slotwrapper(elsetype) + if thentype isa LimitedAccuracy || elsetype isa LimitedAccuracy + return Bool + end return new(slot, thentype, elsetype, isdefined) end end @@ -83,6 +86,9 @@ struct MustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" + if vartyp isa LimitedAccuracy || fldtyp isa LimitedAccuracy + return fldtyp + end return new(slot, vartyp, fldidx, fldtyp) end end @@ -104,6 +110,9 @@ struct InterMustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" + if vartyp isa LimitedAccuracy || fldtyp isa LimitedAccuracy + return fldtyp + end return new(slot, vartyp, fldidx, fldtyp) end end From f9b7d27b753020b828d6a050a341b3c2af49000c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 1 Aug 2025 15:23:19 -0400 Subject: [PATCH 612/662] rename `GlobalMethods` to `methodtable` :bike: :house: (#59158) --- Compiler/src/abstractinterpretation.jl | 4 ++-- Compiler/src/stmtinfo.jl | 4 ++-- Compiler/src/tfuncs.jl | 2 +- src/builtins.c | 2 +- src/jltypes.c | 2 +- src/rtutils.c | 2 +- stdlib/Serialization/src/Serialization.jl | 2 +- stdlib/Test/src/Test.jl | 4 ++-- test/misc.jl | 4 ++-- test/worlds.jl | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 47e90e22c4d57..75a7bcaa4eede 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -369,7 +369,7 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: end valid_worlds = intersect(valid_worlds, thismatches.valid_worlds) thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches) - mt = Core.GlobalMethods + mt = Core.methodtable thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) for idx = 1:length(thismatches) @@ -390,7 +390,7 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a return FailedMethodMatch("Too many methods matched") end fullmatch = any(match::MethodMatch->match.fully_covers, matches) - mt = Core.GlobalMethods + mt = Core.methodtable info = MethodMatchInfo(matches, mt, atype, fullmatch) applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] return MethodMatches(applicable, info, matches.valid_worlds) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index d6a63d8f71abf..525fe0cc222b7 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -49,14 +49,14 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo if !fully_covering(info) exists = false for i in 2:length(edges) - if edges[i] === Core.GlobalMethods && edges[i-1] == info.atype + if edges[i] === Core.methodtable && edges[i-1] == info.atype exists = true break end end if !exists push!(edges, info.atype) - push!(edges, Core.GlobalMethods) + push!(edges, Core.methodtable) end end nmatches = length(info.results) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index fb173759ab122..71719d75144b3 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -3208,7 +3208,7 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv if match === nothing rt = Const(false) vresults = MethodLookupResult(Any[], valid_worlds, true) - mt = Core.GlobalMethods + mt = Core.methodtable vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) diff --git a/src/builtins.c b/src/builtins.c index 79953001662fd..2fcdd4281f371 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2545,7 +2545,7 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("Module", (jl_value_t*)jl_module_type); add_builtin("MethodTable", (jl_value_t*)jl_methtable_type); - add_builtin("GlobalMethods", (jl_value_t*)jl_method_table); + add_builtin("methodtable", (jl_value_t*)jl_method_table); add_builtin("MethodCache", (jl_value_t*)jl_methcache_type); add_builtin("Method", (jl_value_t*)jl_method_type); add_builtin("CodeInstance", (jl_value_t*)jl_code_instance_type); diff --git a/src/jltypes.c b/src/jltypes.c index b3434b2cbb95b..d33c4050c8e20 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2956,7 +2956,7 @@ void jl_init_types(void) JL_GC_DISABLED XX(simplevector); jl_methcache_type = jl_new_uninitialized_datatype(); jl_methtable_type = jl_new_uninitialized_datatype(); - jl_method_table = jl_new_method_table(jl_symbol("GlobalMethods"), core); + jl_method_table = jl_new_method_table(jl_symbol("methodtable"), core); jl_emptysvec = (jl_svec_t*)jl_gc_permobj(sizeof(void*), jl_simplevector_type, 0); jl_set_typetagof(jl_emptysvec, jl_simplevector_tag, GC_OLD_MARKED); diff --git a/src/rtutils.c b/src/rtutils.c index a64f2e2e757c0..2976e56b4195d 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -1081,7 +1081,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, "nothing"); } else if (v == (jl_value_t*)jl_method_table) { - n += jl_printf(out, "Core.GlobalMethods"); + n += jl_printf(out, "Core.methodtable"); } else if (vt == jl_string_type) { n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1, 0); diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 1b25c59bf3cdf..ee40ebdd4abad 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -1129,7 +1129,7 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) meth.recursion_relation = recursion_relation end if !is_for_opaque_closure - mt = Core.GlobalMethods + mt = Core.methodtable if nothing === ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, Base.get_world_counter()) # XXX: quite sketchy? ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, meth, C_NULL) end diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 09594e5be0025..1fd02de9eb769 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -2243,7 +2243,7 @@ function detect_ambiguities(mods::Module...; end end end - examine(Core.GlobalMethods) + examine(Core.methodtable) return collect(ambs) end @@ -2291,7 +2291,7 @@ function detect_unbound_args(mods...; push!(ambs, m) end end - examine(Core.GlobalMethods) + examine(Core.methodtable) return collect(ambs) end diff --git a/test/misc.jl b/test/misc.jl index 86bfc251c826f..8782941adcc1e 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1632,10 +1632,10 @@ end let errs = IOBuffer() run(`$(Base.julia_cmd()) -e ' using Test - @test !isempty(Core.GlobalMethods.backedges) + @test !isempty(Core.methodtable.backedges) Base.Experimental.disable_new_worlds() @test_throws "disable_new_worlds" @eval f() = 1 - @test isempty(Core.GlobalMethods.backedges) + @test isempty(Core.methodtable.backedges) @test_throws "disable_new_worlds" Base.delete_method(which(+, (Int, Int))) @test 1+1 == 2 using Dates diff --git a/test/worlds.jl b/test/worlds.jl index e1e86882306e4..caededd329eff 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -198,7 +198,7 @@ z26506 = Any["ABC"] f26506(x::Int) = 2 g26506(z26506) # Places an entry for f26506(::String) in MethodTable cache w26506 = Base.get_world_counter() -cache26506 = ccall(:jl_mt_find_cache_entry, Any, (Any, Any, UInt), Core.GlobalMethods.cache, Tuple{typeof(f26506),String}, w26506)::Core.TypeMapEntry +cache26506 = ccall(:jl_mt_find_cache_entry, Any, (Any, Any, UInt), Core.methodtable.cache, Tuple{typeof(f26506),String}, w26506)::Core.TypeMapEntry @test cache26506.max_world === typemax(UInt) w26506 = Base.get_world_counter() f26506(x::String) = 3 From f5668d6fdd831f65e4c2494011d3d53c7edc6d22 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Fri, 1 Aug 2025 21:26:16 +0200 Subject: [PATCH 613/662] more accurate `rad2deg` and `deg2rad` for `Float16` and `Float32` (#59097) Both `deg2rad(::AbstractFloat)` and `rad2deg(::AbstractFloat)` just multiply the input argument with a constant-foldable factor of the same type as of the input argument. The generic code for computing the factor involves a double rounding, so the factor is not guaranteed to be correctly rounded. Introduce special cases for `Float16` and `Float32` to make the factor correctly rounded. This is not necessary for `Float64`, as the factor happens to be correctly rounded in that case. Direct improvements: * improve accuracy of `deg2rad(::Float16)` * improve accuracy of `rad2deg(::Float16)` * improve accuracy of `rad2deg(::Float32)` Example indirect improvements: * improving accuracy of `rad2deg(::Float32)` also improves accuracy of `asind(::Float32)`, `acosd(::Float32)`, `atand(::Float32)` --- base/math.jl | 25 +++++++++++++++++++++++-- test/math.jl | 12 +++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/base/math.jl b/base/math.jl index 5122676958405..d6efadf56eccd 100644 --- a/base/math.jl +++ b/base/math.jl @@ -228,6 +228,27 @@ end return hi, lo end +# generic, but involves double rounding +function _180_over_pi(z::AbstractFloat) + 180 / oftype(z, pi) +end +function _pi_over_180(z::AbstractFloat) + oftype(z, pi) / 180 +end + +# rounded to closest representable number where necessary +function _180_over_pi(z::Union{Float16, Float32}) + if z isa Float16 + r = Float16(57.28) + elseif z isa Float32 + r = 57.29578f0 + end + r +end +function _pi_over_180(::Float16) + Float16(0.01746) +end + """ rad2deg(x) @@ -241,7 +262,7 @@ julia> rad2deg(pi) 180.0 ``` """ -rad2deg(z::AbstractFloat) = z * (180 / oftype(z, pi)) +rad2deg(z::AbstractFloat) = z * _180_over_pi(z) """ deg2rad(x) @@ -256,7 +277,7 @@ julia> deg2rad(90) 1.5707963267948966 ``` """ -deg2rad(z::AbstractFloat) = z * (oftype(z, pi) / 180) +deg2rad(z::AbstractFloat) = z * _pi_over_180(z) rad2deg(z::Real) = rad2deg(float(z)) deg2rad(z::Real) = deg2rad(float(z)) rad2deg(z::Number) = (z/pi)*180 diff --git a/test/math.jl b/test/math.jl index fa8dd172b0c2d..dea1c8035d5eb 100644 --- a/test/math.jl +++ b/test/math.jl @@ -448,7 +448,7 @@ end end @testset "deg2rad/rad2deg" begin - @testset "$T" for T in (Int, Float64, BigFloat) + @testset "$T" for T in (Int, Float16, Float32, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi @test deg2rad.(T[45, 60]) ≈ [pi/T(4), pi/T(3)] @test rad2deg.([pi/T(4), pi/T(3)]) ≈ [45, 60] @@ -456,6 +456,16 @@ end @test rad2deg(T(1)) ≈ rad2deg(true) @test deg2rad(T(1)) ≈ deg2rad(true) end + @testset "accuracy" begin + @testset "$T" for T in (Float16, Float32, Float64) + @test rad2deg(T(1)) === setprecision(BigFloat, 500) do + T(180 / BigFloat(pi)) + end + @test deg2rad(T(1)) === setprecision(BigFloat, 500) do + T(BigFloat(pi) / 180) + end + end + end @test deg2rad(180 + 60im) ≈ pi + (pi/3)*im @test rad2deg(pi + (pi/3)*im) ≈ 180 + 60im end From ca4981c9e3ae9f7b0301ea1bbcfa4209bd8cce77 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 1 Aug 2025 12:38:45 -0700 Subject: [PATCH 614/662] Update lock hierarchy devdocs (#58884) Brings the devdocs up to date for the lock hierarchy we have now, documenting the deadlock avoidance algorithm we use for method locks. The information about which locks to take to update global data has been removed because it was out of date, and some basic comments about locking has been added to `julia.h`. Also removes an unused mutex, `extern_c_lock`, which I found while looking for locks. TODO: - A few fields that have questionable locking (`usings_backedges`, `jl_binding_t.backedges`) will be fixed later. - Document how the atomic fields are updated/read. --------- Co-authored-by: Jameson Nash --- doc/src/devdocs/img/invalidation-example.svg | 234 +++++++++++++ doc/src/devdocs/img/typeinf-promotion.svg | 55 +++ doc/src/devdocs/locks.md | 345 +++++++++++-------- src/jitlayers.cpp | 3 - src/julia.h | 27 +- 5 files changed, 525 insertions(+), 139 deletions(-) create mode 100644 doc/src/devdocs/img/invalidation-example.svg create mode 100644 doc/src/devdocs/img/typeinf-promotion.svg diff --git a/doc/src/devdocs/img/invalidation-example.svg b/doc/src/devdocs/img/invalidation-example.svg new file mode 100644 index 0000000000000..678f5100ba5df --- /dev/null +++ b/doc/src/devdocs/img/invalidation-example.svg @@ -0,0 +1,234 @@ + + + + + + +G + + + +mt + +MethodTable + + + +f_m + +Method +f() + + + +mt->f_m + + + + + +g_m + +Method +g() + + + +mt->g_m + + + + + +m_mod + +Module +M + + + +c2_bind + +Binding +M.c2 + + + +m_mod->c2_bind + + +bindings + + + +f_mi + +MethodInstance +f() + + + +f_m->f_mi + + +specializations + + + +g_mi + +MethodInstance +g() + + + +g_m->g_mi + + +specializations + + + +f_ci2 + +CodeInstance +f() for [2, ∞) + + + +f_mi->f_ci2 + + +cache + + + +g_ci1 + +CodeInstance +g() for [1, 1] + + + +f_mi->g_ci1 + + +backedges[1] + + + +g_ci2 + +CodeInstance +g() for [2, ∞) + + + +f_mi->g_ci2 + + +backedges[2] + + + +g_mi->g_ci2 + + +cache + + + +f_ci1 + +CodeInstance +f() for [1, 1] + + + +f_ci1->c2_bind + + +edges[1] + + + +f_ci2->f_ci1 + + +next + + + +f_ci2->c2_bind + + +edges[1] + + + +g_ci1->f_ci1 + + +edges[1] + + + +g_ci2->f_ci2 + + +edges[1] + + + +g_ci2->g_ci1 + + +next + + + +c2_bind->f_ci1 + + +backedges[1] + + + +c2_bind->f_ci2 + + +backedges[2] + + + +c2_bpart2 + +BindingPartition +constant 3 for [2, ∞) + + + +c2_bind->c2_bpart2 + + +partitions + + + +c2_bpart1 + +BindingPartition +constant 2 for [1, 1] + + + +c2_bpart2->c2_bpart1 + + +next + + + diff --git a/doc/src/devdocs/img/typeinf-promotion.svg b/doc/src/devdocs/img/typeinf-promotion.svg new file mode 100644 index 0000000000000..2bf859a70d06c --- /dev/null +++ b/doc/src/devdocs/img/typeinf-promotion.svg @@ -0,0 +1,55 @@ + + + + + + + + + T1 + + + + + acq W + promote + new method + read W + + + + + + + + + + promotefail + + + + + + + + + + + + T2 + + T3 + + + + + acq W + read W + read W+1 + rel W+1 + + + + + + diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index c89499a4162a9..5854c4cb22ff4 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -3,158 +3,233 @@ The following strategies are used to ensure that the code is dead-lock free (generally by addressing the 4th Coffman condition: circular wait). -> 1. structure code such that only one lock will need to be acquired at a time -> 2. always acquire shared locks in the same order, as given by the table below -> 3. avoid constructs that expect to need unrestricted recursion - -## Locks - -Below are all of the locks that exist in the system and the mechanisms for using them that avoid -the potential for deadlocks (no Ostrich algorithm allowed here): - -The following are definitely leaf locks (level 1), and must not try to acquire any other lock: - -> * safepoint -> -> > Note that this lock is acquired implicitly by `JL_LOCK` and `JL_UNLOCK`. use the `_NOGC` variants -> > to avoid that for level 1 locks. -> > -> > While holding this lock, the code must not do any allocation or hit any safepoints. Note that -> > there are safepoints when doing allocation, enabling / disabling GC, entering / restoring exception -> > frames, and taking / releasing locks. -> * shared_map -> * finalizers -> * pagealloc -> * gc_perm_lock -> * flisp -> * jl_in_stackwalk (Win32) -> * ResourcePool::mutex -> * RLST_mutex -> * llvm_printing_mutex -> * jl_locked_stream::mutex -> * debuginfo_asyncsafe -> * inference_timing_mutex -> * ExecutionEngine::SessionLock -> -> > flisp itself is already threadsafe, this lock only protects the `jl_ast_context_list_t` pool -> > likewise, the ResourcePool::mutexes just protect the associated resource pool - -The following is a leaf lock (level 2), and only acquires level 1 locks (safepoint) internally: - -> * global_roots_lock -> * Module->lock -> * JLDebuginfoPlugin::PluginMutex -> * newly_inferred_mutex - -The following is a level 3 lock, which can only acquire level 1 or level 2 locks internally: - -> * Method->writelock -> * typecache - -The following is a level 4 lock, which can only recurse to acquire level 1, 2, or 3 locks: - -> * MethodTable->writelock +1. structure code such that only one lock will need to be acquired at a time +2. always acquire shared locks in the same order, as given by the table below +3. avoid constructs that expect to need unrestricted recursion + +## Types of locks + +`uv_mutex_t` (or `std::mutex`) is a wrapper around platform-specific locks +(`pthread_mutex_t` on Unix, `CRITICAL_SECTION` on Windows). It may cause the +current OS thread to block, is not reentrant, and is not a safepoint. + +`jl_mutex_t` is a reentrant spinlock. `jl_mutex_t`s acquired in a `try` block +will be unlocked when we leave the block, either by reaching the end or catching +an exception. `JL_LOCK`/`JL_UNLOCK` are safepoints, while +`JL_LOCK_NOGC`/`JL_UNLOCK_NOGC` are not. `jl_mutex_t` must not be held across +task switches. + +## Lock hierarchy + +Below are all of the locks that exist in the system and the mechanisms for using +them that avoid the potential for deadlocks (no Ostrich algorithm allowed here). +Except in the special cases where a rule for avoiding deadlock is given, no lock +of a lower level may acquire a lock at a higher level. + +### Level 1 + +No other lock may be acquired when one of these locks is held. As a result, the +code must not do any allocation or hit any safepoints. Note that there are +safepoints when doing allocation, enabling/disabling GC, entering/restoring +exception frames, and taking/releasing locks. + +* `safepoint_lock` (`uv_mutex_t`) + !!! danger + + This lock is acquired implicitly by `JL_LOCK` and `JL_UNLOCK`. Use the + `_NOGC` variants to avoid that for level 1 locks. + +* `shared_map_lock.mtx` (`uv_mutex_t`) +* `finalizers_lock` (`jl_mutex_t`) +* `gc_pages_lock` (`uv_mutex_t`) +* `gc_perm_lock` (`uv_mutex_t`) +* `gc_queue_observer_lock` (`uv_mutex_t`) +* `gc_threads_lock` (`uv_mutex_t`) +* `flisp_lock` (`uv_mutex_t`) + !!! note + flisp itself is already threadsafe; this lock only protects the + `jl_ast_context_list_t` pool. Likewise, the `ResourcePool::mutexes` + just protect the associated resource pool. + +* `jl_in_stackwalk` (`uv_mutex_t`, Win32 only) +* `ResourcePool.mutex` (`std::mutex`) +* `RLST_mutex` (`std::mutex`) +* `llvm_printing_mutex` (`std::mutex`) +* `jl_locked_stream.mutex` (`std::mutex`) +* `debuginfo_asyncsafe` (`uv_rwlock_t`) +* `profile_show_peek_cond_lock` (`jl_mutex_t`) +* `trampoline_lock` (`uv_mutex_t`) +* `bt_data_prof_lock` (`uv_mutex_t`) +* `jl_ptls_t.sleep_lock` (`uv_mutex_t`) +* `tls_lock` (`uv_mutex_t`) +* `page_profile_lock` (`uv_mutex_t`) +* `symtab_lock` (`uv_mutex_t`) +* `engine_lock` (`std::mutex`) + +### Level 2 + +* `global_roots_lock` +* `jl_module_t.lock` +* `newly_inferred_mutex` +* `JLDebuginfoPlugin.PluginMutex` (`std::mutex`) +* `precompile_field_replace_lock` +* `live_tasks_lock` (`uv_mutex_t`) +* `heapsnapshot_lock` +* `jitlock` +* `jl_safepoint_suspend_all_threads` and `jl_safepoint_resume_all_threads` + !!! note + Inside a region protected by these functions, all other threads are + blocked inside a safepoint. It is unsafe to take locks that may safepoint + in this region. + +### Level 3 + +* `jl_method_t.writelock` +* `typecache_lock` +* `libmap_lock` + +### Level 4 + +* `jl_methcache_t.writelock` + +### Level 5 + +* `jl_methtable_t.writelock` + +### Level 6 No Julia code may be called while holding a lock above this point. -orc::ThreadSafeContext (TSCtx) locks occupy a special spot in the locking hierarchy. They are used to -protect LLVM's global non-threadsafe state, but there may be an arbitrary number of them. By default, -all of these locks may be treated as level 5 locks for the purposes of comparing with the rest of the -hierarchy. Acquiring a TSCtx should only be done from the JIT's pool of TSCtx's, and all locks on -that TSCtx should be released prior to returning it to the pool. If multiple TSCtx locks must be -acquired at the same time (due to recursive compilation), then locks should be acquired in the order -that the TSCtxs were borrowed from the pool. +* `world_counter_lock` -The following is a level 5 lock +### Level 7 -> * JuliaOJIT::EmissionMutex +* `JuliaOJIT::EmissionMutex` (`std::recursive_mutex`) -The following are a level 6 lock, which can only recurse to acquire locks at lower levels: +* `jl_modules_mutex` -> * jl_modules_mutex +* `jl_uv_mutex` (known as `iolock` from Julia) + !!! danger + Doing any I/O (for example, printing warning messages or debug information) + while holding any other lock listed above may result in pernicious and + hard-to-find deadlocks. -The following lock synchronizes IO operation. Be aware that doing any I/O (for example, -printing warning messages or debug information) while holding any other lock listed above -may result in pernicious and hard-to-find deadlocks. BE VERY CAREFUL! +* Individual `ThreadSynchronizer` locks + !!! danger + This may continue to be held after releasing the iolock, or acquired + without it, but be very careful to never attempt to acquire the iolock + while holding it. -> * iolock -> * Individual ThreadSynchronizers locks -> -> > this may continue to be held after releasing the iolock, or acquired without it, -> > but be very careful to never attempt to acquire the iolock while holding it -> -> * Libdl.LazyLibrary lock +* `Libdl.LazyLibrary.lock` (`ReentrantLock`) -The following is a level 7 lock, which can only be acquired when not holding any other locks: +* `orc::ThreadSafeContext` -> * world_counter_lock +* `cfun_lock` +### Level 8 -The following is the root lock, meaning no other lock shall be held when trying to acquire it: +* `precomp_statement_out_lock` +* `dispatch_statement_out_lock` -> * toplevel -> -> > this should be held while attempting a top-level action (such as making a new type or defining -> > a new method): trying to obtain this lock inside a staged function will cause a deadlock condition! -> > -> > -> > additionally, it's unclear if *any* code can safely run in parallel with an arbitrary toplevel -> > expression, so it may require all threads to get to a safepoint first +## Exceptions to the lock hierarchy -## Broken Locks +Ordinarily, it is forbidden to acquire locks of equal level to a lock already +held. In these specific cases we use a special protocol for acquiring locks at +the same level: -The following locks are broken: - - * toplevel - - > doesn't exist right now - > - > fix: create it - - * Module->lock - - > This is vulnerable to deadlocks since it can't be certain it is acquired in sequence. - > Some operations (such as `import_module`) are missing a lock. - > - > fix: replace with `jl_modules_mutex`? - - * loading.jl: `require` and `register_root_module` - - > This file potentially has numerous problems. - > - > fix: needs locks - -## Shared Global Data Structures - -These data structures each need locks due to being shared mutable global state. It is the inverse -list for the above lock priority list. This list does not include level 1 leaf resources due to -their simplicity. +- `jl_method_t.writelock` -MethodTable modifications (def, cache) : MethodTable->writelock + Invalidation acquires the lock for every method during its depth-first search + for backedges. To avoid deadlocks, we must already hold `world_counter_lock` + before acquiring multiple `jl_method_t.writelock`s. -Type declarations : toplevel lock +### Broken locks -Type application : typecache lock - -Global variable tables : Module->lock - -Module serializer : toplevel lock - -JIT & type-inference : codegen lock - -MethodInstance/CodeInstance updates : Method->writelock - -> * These are set at construction and immutable: -> * specTypes -> * sparam_vals -> * def -> * owner - -> * Function pointers: -> * these transition once, from `NULL` to a value, which is coordinated internal to the JIT -> - -Method : Method->writelock +The following locks are broken: - * roots array (serializer and codegen) - * invoke / specializations / tfunc modifications +* `loading.jl`: `require` and `register_root_module` + + This file potentially has numerous problems. (fix: needs locks) + +## Updates to the world counter + +Thanks to the [world age](@ref man-world-age) mechanism, Julia can allow the +replacement of both methods and bindings, yet remain amenable to optimization. +Every compiled `CodeInstance` has a range of valid world ages; we could +conservatively assume all CIs are stale after a world age increment. However, +to avoid spurious recompilation, we track dependencies, called "edges", while +maintaining the following invariant: + +For every published `CodeInstance`, either: +- `min_world` and `max_world` are finite, and the CI is valid for every world + in that range. +- `max_world` is ∞ (`-1`), and this CI is ready for invalidation, meaning + for every forward edge: + - If the edge is a `CodeInstance` that is invoked or inlined into this CI, + the edge's `MethodInstance` `backedge` array has an entry pointing back. + - If the edge is a `Binding`: + - If the binding is in another module, it has an entry for this CI in its + `backedges` array. + - If the binding is in the same module, the `Method` for this CI is in the + module's `scanned_methods` array. + +For example, the following code replaces a constant in another module, causing a +chain of invalidations: +```julia +const c1 = 1 +module M const c2 = 2 end +f() = getfield(M, :c2) +g() = f() + c1 + +g() # compile g + +@eval M const c2 = 3 # invalidate f, g +g() # recompile g +``` + +After compiling the two versions of `g()`, the global cache looks like this: +![Global cache state after invalidation](./img/invalidation-example.svg) + +The maximum world age, `jl_world_counter`, is protected by the +`world_counter_lock`. Julia uses a form of optimistic concurrency control to +allow type inference without holding `world_counter_lock`. + +Publishing a new method or binding follows these steps: +- Acquire `world_counter_lock`. +- Relaxed-load `jl_world_counter` and let `new_world = jl_world_counter + 1`. +- Publish the new binding partitions or method table entries with world range + `[new_world, ∞)`. This step is described in the section on the [lock free + data structures](@ref man-lock-free-data). +- Release-store `new_world` to `jl_world_counter`. +- Release `world_counter_lock`. + +Type inference proceeds like so: +- Acquire-load `jl_world_counter` (call this `validation_world`). +- Perform type inference in that world, reading the bindings and method table in + that world using the lock-free data structures. +- Store back edges for every inferred `CodeInstance`: + - For non-local bindings, this acquires the binding's module's lock. + - For CIs, this acquires the method's lock. +- Acquire `world_counter_lock`. +- Relaxed-load `jl_world_counter` and compare it to `validation_world`: + - If it is different, leave the valid world ranges for the inferred CIs + unchanged. + - If it is unchanged, our optimism was rewarded. We can promote all the + inferred CIs valid in `validation_world` to `[validation_world, ∞)` and rely + on the backedges for invalidation. +- Release `world_counter_lock`. + +![Two threads doing type inference while another adds a method](./img/typeinf-promotion.svg) + +In the above diagram, threads 1 and 2 are doing type inference (the dotted +line), while thread 3 is activating a new method. The solid boxes represent +critical sections where the `world_counter_lock` is held. `acq`, `rel`, and +`read`, are acquire loads, release stores, and relaxed loads respectively. + +T1 promotes its CI in time, but T2 takes too long, blocking on +`world_counter_lock` until T3 has finished publishing the new method and +incrementing the world counter. It reads `W+1` and fails to promote its CI, +leaving it with a maximum world of `W`. + +## [Lock free data structures](@id man-lock-free-data) +TODO diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 460ba016c95be..7eee739f4f550 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -332,8 +332,6 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, // lock for places where only single threaded behavior is implemented, so we need GC support static jl_mutex_t jitlock; - // locks for adding external code to the JIT atomically -static std::mutex extern_c_lock; // locks and barriers for this state static std::mutex engine_lock; static std::condition_variable engine_wait; @@ -361,7 +359,6 @@ static DenseMap> incompl // jitlock is outermost, can contain others and allows GC // engine_lock is next // ThreadSafeContext locks are next, they should not be nested (unless engine_lock is also held, but this may make TSAN sad anyways) -// extern_c_lock is next // jl_ExecutionEngine internal locks are exclusive to this list, since OrcJIT promises to never hold a lock over a materialization unit: // construct a query object from a query set and query handler // lock the session diff --git a/src/julia.h b/src/julia.h index a5564670479db..7d0900a761421 100644 --- a/src/julia.h +++ b/src/julia.h @@ -317,6 +317,11 @@ typedef struct _jl_code_info_t { // This type describes a single method definition, and stores data // shared by the specializations of a function. +// +// Reading or writing requires `writelock` or exclusive ownership: +// roots, root_blocks, nroots_sysimg, ccallable +// No lock is required to read these fields, set once on construction: +// all other fields typedef struct _jl_method_t { JL_DATA_TYPE jl_sym_t *name; // for error reporting @@ -382,13 +387,19 @@ typedef struct _jl_method_t { _jl_purity_overrides_t purity; // hidden fields: - // lock for modifications to the method jl_mutex_t writelock; } jl_method_t; // This type is a placeholder to cache data for a specType signature specialization of a Method // can can be used as a unique dictionary key representation of a call to a particular Method // with a particular set of argument types +// +// Reading or writing requires `def.method->writelock` or exclusive ownership: +// backedges +// Reading or writing requires the associated jl_methcache_t's `writelock`: +// cache_with_orig +// No lock is required to read these fields, set once on construction: +// def, specTypes, sparam_vals struct _jl_method_instance_t { JL_DATA_TYPE union { @@ -425,6 +436,11 @@ typedef struct _jl_opaque_closure_t { } jl_opaque_closure_t; // This type represents an executable operation +// +// No lock is required to read these fields, which are set while we have +// exclusive ownership of the CodeInstance: +// def, owner, rettype, exctype, rettype_const, analysis_results, +// time_infer_total, time_infer_self typedef struct _jl_code_instance_t { JL_DATA_TYPE jl_value_t *def; // MethodInstance or ABIOverride @@ -795,6 +811,15 @@ typedef struct { uint64_t lo; } jl_uuid_t; +// Reading or writing requires `lock`: +// scanned_methods, usings +// Reading or writing requires `Base.require_lock`: +// uuid +// Reading or writing requires `world_counter_lock`: +// usings_backedges (TODO) +// No lock is required to read these fields, set once on construction: +// name, parent, file, line, build_id, uuid, nospecialize, optlevel, compile, +// infer, iistopmod, max_methods typedef struct _jl_module_t { JL_DATA_TYPE jl_sym_t *name; From 698ae27883f200b546289d5f04059015125fa0a2 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 1 Aug 2025 12:46:11 -0700 Subject: [PATCH 615/662] Add CROSS_BOOTSTRAP_JULIA option for building the sysimg with another Julia (#59033) Adds makefile variables called `CROSS_BOOTSTRAP_JULIA` and `CROSS_BOOTSTRAP_SYSBASE` that control the Julia executable and `sysbase.so` used to build `sys.so`. This is really useful for building a ThreadSanitizer-enabled version of Julia in a reasonable amount of time. It is used like this: ``` make O=bootstrap julia-src-release julia-sysbase-release make CROSS_BOOTSTRAP_JULIA=bootstrap/usr/bin/julia CROSS_BOOTSTRAP_SYSBASE=bootstrap/usr/lib/julia/sysbase.so julia-src-release julia-sysimg-release ``` This also adds a faster way to create a debug build of Julia (documented in the devdocs). --- Make.inc | 11 +++++++++++ Makefile | 4 ++++ contrib/tsan/Make.user.tsan | 2 ++ contrib/tsan/build.sh | 22 ++++++++++++++++++++-- doc/src/devdocs/build/build.md | 14 ++++++++++++++ sysimage.mk | 6 +++++- 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/Make.inc b/Make.inc index 4a2e75c40bf3f..049d0bebe05de 100644 --- a/Make.inc +++ b/Make.inc @@ -798,6 +798,8 @@ ifneq ($(OS), Darwin) endif endif +bootstrap_julia_flags := + ifeq ($(SANITIZE),1) SANITIZE_OPTS := SANITIZE_LDFLAGS := @@ -815,6 +817,9 @@ endif ifeq ($(SANITIZE_THREAD),1) SANITIZE_OPTS += -fsanitize=thread SANITIZE_LDFLAGS += -fsanitize=thread +ifneq ($(CROSS_BOOTSTRAP_JULIA),) +bootstrap_julia_flags += --target-sanitize=thread +endif endif ifeq ($(SANITIZE_OPTS),) $(error SANITIZE=1, but no sanitizer selected, set either SANITIZE_MEMORY, SANITIZE_THREAD, or SANITIZE_ADDRESS) @@ -1761,9 +1766,15 @@ JULIA_BUILD_MODE := debug endif endif +ifneq ($(CROSS_BOOTSTRAP_JULIA),) +JULIA_EXECUTABLE_debug := $(CROSS_BOOTSTRAP_JULIA) +JULIA_EXECUTABLE_release := $(CROSS_BOOTSTRAP_JULIA) +JULIA_EXECUTABLE := $(CROSS_BOOTSTRAP_JULIA) +else JULIA_EXECUTABLE_debug := $(build_bindir)/julia-debug$(EXE) JULIA_EXECUTABLE_release := $(build_bindir)/julia$(EXE) JULIA_EXECUTABLE := $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) +endif JULIA_SYSIMG_debug := $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT) diff --git a/Makefile b/Makefile index 9d1351e41b2c7..3e4fc1356b08d 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,10 @@ julia-cli-release julia-cli-debug: julia-cli-% : julia-deps julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$* +# Useful for cross-bootstrapping +julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysbase-$* + julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \ julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache diff --git a/contrib/tsan/Make.user.tsan b/contrib/tsan/Make.user.tsan index e4107222aba29..252a17ba86497 100644 --- a/contrib/tsan/Make.user.tsan +++ b/contrib/tsan/Make.user.tsan @@ -10,3 +10,5 @@ USE_BINARYBUILDER_LLVM=1 override SANITIZE=1 override SANITIZE_THREAD=1 +override CROSS_BOOTSTRAP_JULIA=$(BUILDROOT)/../bootstrap/usr/bin/julia +override CROSS_BOOTSTRAP_SYSBASE=$(BUILDROOT)/../bootstrap/usr/lib/julia/sysbase.$(SHLIB_EXT) diff --git a/contrib/tsan/build.sh b/contrib/tsan/build.sh index 2c4ba3b1bde95..de3951fbe8bd5 100755 --- a/contrib/tsan/build.sh +++ b/contrib/tsan/build.sh @@ -3,7 +3,7 @@ # # Usage: -# contrib/tsan/build.sh [...] +# contrib/tsan/build.sh [-j] [...] # # Build TSAN-enabled julia. Given a workspace directory , build # TSAN-enabled julia in /tsan. Required toolss are install under @@ -13,6 +13,16 @@ # make target is `debug`. set -ue +set -x + +JOBS=1 +while getopts j: opt +do + case $opt in + j) JOBS="$OPTARG";; + esac +done +shift $((OPTIND-1)) # `$WORKSPACE` is a directory in which we create `toolchain` and `tsan` # sub-directories. @@ -44,6 +54,12 @@ fi make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools +echo +echo "Building bootstrap Julia..." +BUILD="$WORKSPACE/bootstrap" + +make -j "$JOBS" O="$BUILD" julia-src-release julia-sysbase-release + echo echo "Building Julia..." @@ -54,4 +70,6 @@ if [ ! -d "$BUILD" ]; then fi cd "$BUILD" # so that we can pass `-C src` to `make` -make "$@" +# Reporting tsan warnings will interfere with bootstrapping. +export TSAN_OPTIONS="report_bugs=0 exitcode=0" +make -j "$JOBS" "$@" diff --git a/doc/src/devdocs/build/build.md b/doc/src/devdocs/build/build.md index ff7e64926f1f7..b6a949c74e74a 100644 --- a/doc/src/devdocs/build/build.md +++ b/doc/src/devdocs/build/build.md @@ -304,6 +304,20 @@ LLVM_ASSERTIONS=1 Please note that assert builds of Julia will be slower than regular (non-assert) builds. +## Building a debug build of Julia + +A full debug build of Julia can be built with `make debug`. This builds a debug +version of `libjulia` and uses it to bootstrap the compiler, before creating a +system image with debug symbols enabled. This can take more than 15 minutes. + +Although it may result in some differences, a debug build can be built much +quicker by bootstrapping from a release build: + +```sh +$ make julia-src-release julia-sysbase-release +$ make julia-sysimg-debug CROSS_BOOTSTRAP_JULIA=$PWD/usr/bin/julia CROSS_BOOTSTRAP_SYSBASE=$PWD/usr/lib/julia/sysbase.so +``` + ## Building 32-bit Julia on a 64-bit machine Occasionally, bugs specific to 32-bit architectures may arise, and when this happens it is useful to be able to debug the problem on your local machine. Since most modern 64-bit systems support running programs built for 32-bit ones, if you don't have to recompile Julia from source (e.g. you mainly need to inspect the behavior of a 32-bit Julia without having to touch the C code), you can likely use a 32-bit build of Julia for your system that you can obtain from the [official downloads page](https://julialang.org/downloads/). diff --git a/sysimage.mk b/sysimage.mk index 2a938de697156..e98dceb6a0128 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -11,6 +11,8 @@ sysimg-ji: $(build_private_libdir)/sysbase.ji sysimg-bc: $(build_private_libdir)/sys-bc.a sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT) sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) +sysbase-release: $(build_private_libdir)/sysbase.$(SHLIB_EXT) +sysbase-debug: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) @@ -124,7 +126,8 @@ $$(build_private_libdir)/sysbase$1-o.a $$(build_private_libdir)/sysbase$1-bc.a : false; \ fi ) @mv $$@.tmp $$@ -$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT) $$(JULIAHOME)/contrib/generate_precompile.jl +build_sysbase_$1 := $$(or $$(CROSS_BOOTSTRAP_SYSBASE),$$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT)) +$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_sysbase_$1) $$(JULIAHOME)/contrib/generate_precompile.jl @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \ WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ @@ -133,6 +136,7 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil JULIA_DEPOT_PATH=':' \ JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ + $(bootstrap_julia_flags) \ --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ From 81352c136378f34481d01f180dc2e5f740fd29d7 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 2 Aug 2025 06:01:56 +0900 Subject: [PATCH 616/662] Test: remove unnecessary allocation (#59186) --- stdlib/Test/src/Test.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 1fd02de9eb769..604bd82217240 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1351,7 +1351,7 @@ function finish(ts::DefaultTestSet; print_results::Bool=TESTSET_PRINT_ENABLE[]) end # return the testset so it is returned from the @testset macro - ts + return ts end # Recursive function that finds the column that the result counts @@ -1375,15 +1375,15 @@ get_alignment(ts, depth::Int) = 0 # Recursive function that fetches backtraces for any and all errors # or failures the testset and its children encountered function filter_errors(ts::DefaultTestSet) - efs = [] + efs = Any[] for t in ts.results if isa(t, DefaultTestSet) append!(efs, filter_errors(t)) elseif isa(t, Union{Fail, Error}) - append!(efs, [t]) + push!(efs, t) end end - efs + return efs end """ From d00f9d07ab73da78cd661aa1ae09ee8d263c457e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Aug 2025 20:14:27 -0400 Subject: [PATCH 617/662] Test: `@test_throws` add 3-arg form for testing exception type and message (#59117) --- NEWS.md | 1 + stdlib/Test/src/Test.jl | 142 ++++++++++++++++++++++++++--------- stdlib/Test/test/runtests.jl | 60 +++++++++++++++ 3 files changed, 167 insertions(+), 36 deletions(-) diff --git a/NEWS.md b/NEWS.md index 71ebf8aaead0f..ea4d8337beeb6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -81,6 +81,7 @@ Standard library changes * Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). * Transparent test sets (`@testset let`) now show context when tests error ([#58727]). +* `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]). #### InteractiveUtils diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 604bd82217240..7f73662c87938 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -799,17 +799,29 @@ end """ @test_throws exception expr + @test_throws extype pattern expr Tests that the expression `expr` throws `exception`. The exception may specify either a type, a string, regular expression, or list of strings occurring in the displayed error message, a matching function, or a value (which will be tested for equality by comparing fields). + +In the two-argument form, `@test_throws exception expr`, the `exception` can be a type or a pattern. + +In the three-argument form, `@test_throws extype pattern expr`, both the exception type and +a message pattern are tested. The `extype` must be a type, and `pattern` may be +a string, regular expression, or list of strings occurring in the displayed error message, +a matching function, or a value. + Note that `@test_throws` does not support a trailing keyword form. !!! compat "Julia 1.8" The ability to specify anything other than a type or a value as `exception` requires Julia v1.8 or later. +!!! compat "Julia 1.13" + The three-argument form `@test_throws extype pattern expr` requires Julia v1.12 or later. + # Examples ```jldoctest julia> @test_throws BoundsError [1, 2, 3][4] @@ -823,13 +835,19 @@ Test Passed julia> @test_throws "Try sqrt(Complex" sqrt(-1) Test Passed Message: "DomainError with -1.0:\\nsqrt was called with a negative real argument but will only return a complex result if called with a complex argument. Try sqrt(Complex(x))." + +julia> @test_throws ErrorException "error foo" error("error foo 1") +Test Passed + Thrown: ErrorException ``` -In the final example, instead of matching a single string it could alternatively have been performed with: +In the third example, instead of matching a single string it could alternatively have been performed with: - `["Try", "Complex"]` (a list of strings) - `r"Try sqrt\\([Cc]omplex"` (a regular expression) - `str -> occursin("complex", str)` (a matching function) + +In the final example, both the exception type (`ErrorException`) and message pattern (`"error foo"`) are tested. """ macro test_throws(extype, ex) orig_ex = Expr(:inert, ex) @@ -847,6 +865,22 @@ macro test_throws(extype, ex) return :(do_test_throws($result, $orig_ex, $(esc(extype)))) end +macro test_throws(extype, pattern, ex) + orig_ex = Expr(:inert, ex) + ex = Expr(:block, __source__, esc(ex)) + result = quote + try + Returned($ex, nothing, $(QuoteNode(__source__))) + catch _e + if $(esc(extype)) != InterruptException && _e isa InterruptException + rethrow() + end + Threw(_e, Base.current_exceptions(), $(QuoteNode(__source__))) + end + end + return :(do_test_throws($result, $orig_ex, $(esc(extype)), $(esc(pattern)))) +end + const MACROEXPAND_LIKE = Symbol.(("@macroexpand", "@macroexpand1", "macroexpand")) function isequalexception(@nospecialize(a), @nospecialize(b)) @@ -864,49 +898,78 @@ end # An internal function, called by the code generated by @test_throws # to evaluate and catch the thrown exception - if it exists -function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), extype) +function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), extype, pattern=nothing) if isa(result, Threw) # Check that the right type of exception was thrown success = false message_only = false exc = result.exception - # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit - # the breakage in package tests we add extra logic here. - from_macroexpand = - orig_expr isa Expr && - orig_expr.head in (:call, :macrocall) && - orig_expr.args[1] in MACROEXPAND_LIKE - if isa(extype, Type) - success = - if from_macroexpand && extype == LoadError && exc isa Exception - Base.depwarn("macroexpand no longer throws a LoadError so `@test_throws LoadError ...` is deprecated and passed without checking the error type!", :do_test_throws) - true - elseif extype == ErrorException && isa(exc, FieldError) - Base.depwarn(lazy"Using ErrorException to test field access is deprecated; use FieldError instead.", :do_test_throws) - true - else - isa(exc, extype) - end - elseif isa(extype, Exception) || !isa(exc, Exception) - if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) - extype = extype.error # deprecated + + # Handle three-argument form (type + pattern) + if pattern !== nothing + # In 3-arg form, first argument must be a type + if !isa(extype, Type) + testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, false, "First argument must be an exception type in three-argument form") + record(get_testset(), testres) + return end - # Support `UndefVarError(:x)` meaning `UndefVarError(:x, scope)` for any `scope`. - # Retains the behaviour from pre-v1.11 when `UndefVarError` didn't have `scope`. - if isa(extype, UndefVarError) && !isdefined(extype, :scope) - success = exc isa UndefVarError && exc.var == extype.var - else isa(exc, typeof(extype)) - success = isequalexception(exc, extype) + + # Format combined expected value for display + pattern_str = isa(pattern, AbstractString) ? repr(pattern) : + isa(pattern, Function) ? "< match function >" : + string(pattern) + combined_expected = string(extype) * " with pattern " * pattern_str + + # Check both type and pattern + type_success = isa(exc, extype) + if type_success + exc_msg = sprint(showerror, exc) + pattern_success = contains_warn(exc_msg, pattern) + success = pattern_success + else + success = false end + extype = combined_expected # Use combined format for all results else - message_only = true - exc = sprint(showerror, exc) - success = contains_warn(exc, extype) - exc = repr(exc) - if isa(extype, AbstractString) - extype = repr(extype) - elseif isa(extype, Function) - extype = "< match function >" + # Original two-argument form logic + # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit + # the breakage in package tests we add extra logic here. + from_macroexpand = + orig_expr isa Expr && + orig_expr.head in (:call, :macrocall) && + orig_expr.args[1] in MACROEXPAND_LIKE + if isa(extype, Type) + success = + if from_macroexpand && extype == LoadError && exc isa Exception + Base.depwarn("macroexpand no longer throws a LoadError so `@test_throws LoadError ...` is deprecated and passed without checking the error type!", :do_test_throws) + true + elseif extype == ErrorException && isa(exc, FieldError) + Base.depwarn(lazy"Using ErrorException to test field access is deprecated; use FieldError instead.", :do_test_throws) + true + else + isa(exc, extype) + end + elseif isa(extype, Exception) || !isa(exc, Exception) + if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) + extype = extype.error # deprecated + end + # Support `UndefVarError(:x)` meaning `UndefVarError(:x, scope)` for any `scope`. + # Retains the behaviour from pre-v1.11 when `UndefVarError` didn't have `scope`. + if isa(extype, UndefVarError) && !isdefined(extype, :scope) + success = exc isa UndefVarError && exc.var == extype.var + else isa(exc, typeof(extype)) + success = isequalexception(exc, extype) + end + else + message_only = true + exc = sprint(showerror, exc) + success = contains_warn(exc, extype) + exc = repr(exc) + if isa(extype, AbstractString) + extype = repr(extype) + elseif isa(extype, Function) + extype = "< match function >" + end end end if success @@ -927,6 +990,13 @@ function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), extyp testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, message_only, bt_str) end else + # Handle no exception case - need to format extype properly for 3-arg form + if pattern !== nothing + pattern_str = isa(pattern, AbstractString) ? repr(pattern) : + isa(pattern, Function) ? "< match function >" : + string(pattern) + extype = string(extype) * " with pattern " * pattern_str + end testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, nothing, result.source, false) end record(get_testset(), testres) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 0d6486a26e33e..1b571286abd8c 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -107,6 +107,25 @@ end @test_throws "\"" throw("\"") @test_throws Returns(false) throw(Returns(false)) end + +@testset "Pass - exception with pattern (3-arg form)" begin + # Test 3-argument form: @test_throws ExceptionType pattern expr + @test_throws ErrorException "error foo" error("error foo 1") + @test_throws DomainError r"sqrt.*negative" sqrt(-1) + @test_throws BoundsError "at index [2]" [1][2] + @test_throws ErrorException ["error", "foo"] error("error foo bar") + + # Test with function pattern + @test_throws ErrorException (s -> occursin("foo", s)) error("error foo bar") + + # Test output format + let result = @test_throws ErrorException "error foo" error("error foo 1") + output = sprint(show, result) + @test occursin("Test Passed", output) + @test occursin("Thrown: ErrorException", output) + end +end + # Test printing of Fail results include("nothrow_testset.jl") @@ -402,6 +421,47 @@ let retval_tests = @testset NoThrowTestSet begin end end +@testset "Fail - exception with pattern (3-arg form)" begin + # Test type mismatch + let fails = @testset NoThrowTestSet begin + @test_throws ArgumentError "error foo" error("error foo 1") # Wrong type + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + @test occursin("ArgumentError with pattern \"error foo\"", fails[1].data) + end + + # Test pattern mismatch + let fails = @testset NoThrowTestSet begin + @test_throws ErrorException "wrong pattern" error("error foo 1") # Wrong pattern + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + @test occursin("ErrorException with pattern \"wrong pattern\"", fails[1].data) + end + + # Test no exception thrown + let fails = @testset NoThrowTestSet begin + @test_throws ErrorException "error foo" 1 + 1 # No exception + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_nothing + @test occursin("ErrorException with pattern \"error foo\"", fails[1].data) + end + + # Test first argument must be a type + let fails = @testset NoThrowTestSet begin + @test_throws "not a type" "error foo" error("error foo 1") # First arg not a type + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + end +end + @testset "printing of a TestSetException" begin tse_str = sprint(show, Test.TestSetException(1, 2, 3, 4, Vector{Union{Test.Error, Test.Fail}}())) @test occursin("1 passed", tse_str) From a60814fb590eceb654a1eabb73e56067dde36387 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 1 Aug 2025 20:28:13 -0700 Subject: [PATCH 618/662] Add PNG versions of decdocs diagrams to fix PDF build (#59189) Documenter.jl's LaTeX output doesn't like SVGs, so export PNG versions of these diagrams to fix the build. --- doc/src/devdocs/img/invalidation-example.png | Bin 0 -> 39317 bytes doc/src/devdocs/img/typeinf-promotion.png | Bin 0 -> 4627 bytes doc/src/devdocs/locks.md | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 doc/src/devdocs/img/invalidation-example.png create mode 100644 doc/src/devdocs/img/typeinf-promotion.png diff --git a/doc/src/devdocs/img/invalidation-example.png b/doc/src/devdocs/img/invalidation-example.png new file mode 100644 index 0000000000000000000000000000000000000000..4a80869063761b0919e73652b021ee29aecc3fdd GIT binary patch literal 39317 zcmb??WmH>F6lZ_}1qxN%THGCq(-sZx5F}WFyB4R=7KaiDE-g|lNJDXLDMf;`NN_7w z2oAwz)Bo8$yI=Opp1tSfyq9-p-rRXJbLYx$Vs*7u$w}!*0RR9wP)$i60Kf+U00ez^ zaSs3^3t#$im#21W`dR=$5C;Gd@fHBMz&(oi3jp{D0RUUp0DyER06^`T+oC6nW8m9r zs44+&qmu_aasNm>)j&SDE3Opox9{?51b0L13)E63-n@%PB}H~zt=a(qWI}*SFAZPM zY|jTqXh8{j4;I_&Z29b{r!%&JccJ|crtLoHo9QCgdbom1INS{Zh zUF{Y#z6{HshAk@W4rbcC|8}oVwljRtZ&&to&PU#7*eCxecjF-khttC>Eo)my77d6) z#Tb2-t#F&hg#6Yw;OnIqq!!G|d(?j+aCp-@**EYOXuGZo62Zo zE33sxKo}N_4ZZz&>W;!k|F2HjxxivKORt$;EN36m4O5cEC z`PygIiX|E&5o@D5{ZzgxbTRB58IA`o8e5yoP0Y9HpXos${q*QC{E9x&FNo05w`qM_ zs~8NXkRdktpS0|9pI>@@Gi%Ey^ns;O6;vfu1}(CfWf`a!C}Z#ZBtZ(G4D{-`(A zr(-@`h3@VPe>l<7)?tlE~ZLb0A7B5JRJ>N(1s6h z?^v?=@!j!MNj_(g*1_t+InV#G)Z?c^3{Lhz{a zn6y;Z@SNdOV)4^#bswo)C0o!@NkNxMHN_TiZ?-5%tve67kV0h}u6KbO<5x(ct*E~Y z(?xNAoru&?k&!pZvP*H$<8Ql#!-Hv`@!yR8<kCK#Ad+xRb zp(mkipj>1=fU6=-8?^RkfYI(>&E7PRw)grp ziy~Kx52L#IX&DtH37xppSDCF1qBQz3@{c^4Q{e9)dKMGMqRPi&tR->}7#TuNR6w2T z9(bw6HMZUr2uA!zs6)63*H{06Ls{}$_4!<+8IO*b1vy2BD+J08vSdTi;yyrw={V$c zbF#R6J2fcRm+b?k<*VsVseMKHz`UZI<1GaR%4V?f$=KEUGi3uE=l#4Iwghw%$ zGqNBJTA2ceLqY}MQ7l1pL{gQCiFQq^RI~Y%obfLLfUq@sNi$uTdMgJEo0x`qNtrRM zr$Z>mUf$fz&0A4kH}~fA#Iwr2nbnme;@QxUg(z)#4+0-&Q^VCNA#3|~WJ$Ng`56Mw z>@0x@E#{S4f5H+muT_OsximWqA$*}x;2vS;D-X}6`n&S@g+f6Xs-lVNZvdtL+1VAT z-c3m{sg+a$0>Q!?`_>8=2GUf6%q8pT0k3zp2HLY7Kc&F2kog>%|L0Y%g1&}h*CQ_r zdJ_?vT&O`0M@R{L3U0%myK)L9H)J6K}JRt?@Z5gy76WXZEu)Ar--DLyGVjGmk3>tE+b-I7KpGFJf7*;ZN z&X*><##CN}(}6g=XyI~_6G>q@0}gd5&_A;Afg#y(C4^|?*{%dqvkTm5yagU7G;X{g zq6cw2wrnhJn212`%IVO6K(f6)op~4URHozXPsxOlNZ*-sGL2M>miedLungSy$_Y$g z94%Yry$L+LjHU&F#HgA7rue@rI7NY7LyA-2>_^_qVpZgM2S;34;_jSupxC46wP>-M zw+L^U-+k(7wMhQ48^xQWddQpESQX>Kvo+};L3#VunynfUBvQAbS*u00;A?}IeP^B{ zgk4Q@sCWBxyBnXm{%jW~jf%YYVxXXcfX||~7_xfp<78&=<(sSv|H2eM{voE8jS(fK zX{Npupa-tugDm&!OZgCICHr>Vt9sU4-*lXAMAe*a4DT_!L!dLVmt8%V*xwshNIA!N z38aWa`qswf*=DKe;*Jt4Dyy=jWp0sDa(t}B`*REK1yW;zd*zqyq^%+Reqo|W5s*Yv zNDc*>#b6#M3ZRhLJxyL)$%U@pAwC5jc<9~`x(PMHcd>`zdP!B=d(?egCz7N*VbRJc zgcMO^&@oO{*TLA`=UQ1f!hN*^(P?(Byed8WCR|Hi--A6b%*%fr5VosYiXcUNXENHd znM`;Oc77x}Bp?#fdoINDszH&nu}P{GU@@eKu>j>xewfLwsY0@T1I-d=GnyPHpkdIu z&g2H|#*R`jJ9V=>tqlbF5rG`k+Hh$?O&(6^#8Ij^Nrj#s{nf7ojS65I^; zq2=7U{IhAlXMry%a(18^BHxDZd48SlW={hu&(ZFwf$)I#Dp>EKgGY5UA@E{ZdPi;` zl$KK*Rvg`JV<`aCZ(i{{c?PU-fRxyW=@Au_VuC=O<#)%&_C|5bqvGmq3Ydp7a3^z3 zet!ivYVkb|wU!8UD>nXRSuxj{pMWqNTq7=E_36K;X^Lc(^)7(DA5}$P3xUjhtMakW zu2@0`H8(9i#WGGTu!hZ*l4dxuGM^vj4Z9+xF|C~w+X-T6y=kbCkv;a!JQrj`m%t`t z5b#^qJ*OFIrzb+Qed_Iqfb*n54wtr|#Ie7kqq;mlu^FctzHUwCbjqBYq_q|oXVgw2 zwdg+edL`qq(UeVyn=ZV;%u9t%-3@-1^dCuk7a=A~+Pe&$E|^DUx?bT1QAGMOOe7Cc z=8Bd@$2$~%I!;6Bo%Jq;b1JCwKJL%vZ6`c=kGxY5U{rFKVFx!o2HZz<`a+PeUsVjy zI&tM7c#bxkz7P22zSE!0B7rKW!PlDa)2TBqOO>PVGW)n?ntf5Y@$|J%6Ia?=OY50r zf~C1VWbjUQ8;MKpQ6OXkPKt!m_P7b=xo=S^kBiOK5N`S z*{_eJ3nKXSk-~S38xXXanF61YGTA3brX&2*J-ZHGF#hG-nBvOgOHna)-e@KkfCz*3 zTm)(j{j3Xf-HNpibLc{x9kQrkJ&?SCf`FeVPJwcmE|rZCo}`eSM1-NX`HE-}VI|O)d2F z`8T4}lN`c)WzWfQdcwY@Q~{(%m9%o3+cIREh?tEj0@jxyxG@yCQB`pj$PvPgs^JK5 z?7|WY=;9{AbdT4s_`KtYv4#Yk9sOo8DugG3UsP8)5uY4;0pAy>I zsK0Xfr@F9p!_u=|B1nCOhwyBA6BT1Rh|-Dm%0Qp`_0@*(LhV1!$wj^GJziYdqh<_M z(LWZ$u+-c~5#;Dow}EJSL)_~elGXz3{&8{z$59{tappx)5ydLT7X#|35Y~U3w{R!u zi+={rof4u%-BaA|H&|h zaJk3vk0TKtZilNB<;xv5ZTgR+fvx)lfs}+*r*Ap4{d<2$pbkASfUC0aF^Sc686o0# zDX?Sgp*LlNizXGUegWF@#=j7&oG4I~b^rSClvd0jn%R(qn)NliOrutkgDB>;m$ z`}`@2EDMNsA4$GfRGcJ~9vx?{!Larb%$?F&OX@)&@9oFE1#^)Xn(VGhi-soD<@>c1 z#wo0BNt~L=xm4s%4`iwHfPYW8QHZiTVf+){xG5tjna8_1m#4oIz!ifC3q$zUG+Lf4 zWCTK{nQNG!4D*>f;tV(5dp$3m48>4LBV?W0&MQg%Zq`h>e52%t`jcySl=CBO(g{7w z)ZV!u1&DFfS@ToxUgKbNxAH)5K6#_dy=5rr4H=n5&i6W==67~${G4YhWKX$NGX_~* zU>N&uuGr)8-mbo}S~R}TXL~vj=iwQxGH|lT=fvEbP_nE85HG%2SZ{1? zZN223;SeYm>|1CYJ;qx~-k)xYjYSeZ+A#i6jC6n)5kDCNU7kJ zqswH~)n{}{ZcX+Pk8=Lf*KFz9&dS_L2tQIjM1&1ThY2CuQ;Vt8N@n?91Jc_E`|PGW zGbwkF;DL_E+f_(qds0yNLT_vJrku>`^x->^bPuUAX{LH+pP_cz?{wu`WAFG2@S}|W zl6W-k+#yU4Y!;(Sh?)4v;ib%Zlqn{*=JLt zWqvCr8TLso-Hkp>!V&KJQ2{r>y4|~2RR>?t>^ujZ0!~Bq;iDAKK(>I#2psv#rTEKZ%~-GB+DIc zo=??06jA&fMVny?s!mNnq^(mv2l6yd?l*(7e~fxdxj{5*hli54VXLEq z<4XqFW9d^XZP?{1B3)$j_U@e8A}xq}=t>e)4ZRhOfQ^V&!&bv^3e;)AH(3r+-{yEW zwj$I-C$^QV9Vp@-Daon?QHXnDxLSZFctA^&1=HRk@GzB^XxTuS$RuH(%2@Q zqZ4w?*G&3%I^={i@Sus_8GJR_Zx%iBGTZb&rojJ*)i$+++D#R+g~A1Y;@T>dHFAxA znOcZ23deQCJ}Nu^xxGG>>LymjIPNI6pt&^lkES1PuMI{xrKZqYyrOoI)@jM8H^1>2 zeR`i_WCZqmPx%m+?ZQd|Gu!e6wS=u%id;I;DT$G0A>51#na z%BR4>H8<{W=4kTpmKMXiY=ER%XAOBTg8_nm8Xd*&dGN3$eQ&50Gw6 zES1}++5~zyEiuTa+*}sE8!00Ls1i-opwF%~04DOUTd-f@Ss-H{7bcl;U0&`duM|A6 zN|WcFKaDll+=37E8yN)^rr95K$(M#=%>)tn){+reW-RexN zh|L$f%h9T)UtRw-rC2|NMf^iCk4>$723z^Do}>v7#)KL~N*hi13$#AjWqs%Uxq_{{ zGBlOVi3nVgr3iy^Ooiqpi=Go$nX;`3QVU|V*8Z{q#fz4efw0~hSw0~7^d>)$Cp^V% zG+qEUH~B4V09G9IP)Q8VHgO+SiwrY(Nv!iX!0EwlOU2xe-_6I?KkDt$XS;ONHAG{% zxH0rr^%>dSmHKOV$-D-rt&*C%hL;`sT#V2ix#n0%tN);W)d)C!;9G|N#KGQLhM(N! z+}g>Nixm7tdYfv8`HPEqqtRJSEHkvP z+=yc$hWB0mmo4QOzH&NTRPVgz1>;%-av=C2TEmi-d)hjW{SyL^$r9TA+Ud~9{+v9y z(gAxAdcQ$quU5+$&&IuCgqB*|+jB;%ihkq=fW@`VDb*lY^U#;nL3Q4I`^+<8ILrIra40Ir0J2TYZL&e6$Kb!TW9Cz za#SX>BQeO1UzG)pXE*_jR*1qbtO{MauSf5;s8dJ0=nsE)%wGG<0Oh?pnPRcKjB9;w zMI!$DBTIUD>8DIV(91!L2f(7e{ZA-3<#3Z~19da*ark=NCKeOKL(A}w{sjcDi}SkfZRSjjs#$w|PFLn7AG14#zsgWA zkZFv+CIUGWR|9&C;yBl+2*=g*sWR66hd7(_JO^5G6)CB~(!RLh?r@_!+00`;Y(SX3 z(t9{%UN@}wkvEv`2R)MM?NC_QOTM%)EjEm*(}O8oJaA9>x!jYam8?O27epJD1J|j? zRDJ}##9=ERTk_S|;^PMV=)y9|CjzBX;NO*vt% z|8JDPhq!OoWS9qZE{H0zSUop`=pZ>>Jyf>3)|<&LcHBAW6-bY=SYYotY^FP2f5=8l z^tAq2xZ1@V+>CZB*_E(z5|S;PvkTx%6W8LA^ZuA&(Rf(UDhm!4Ex@&_zjZQ}OXS_z zl`(omMgi2RE&yrLrRY*=6%XIkLqH=B%CW1H{LZOfHRORBK|S2@9&+scKF=7i3IF+% zC$C_|Tcyh(M!>E7W~ikfJV^$cSIG)=rX#B9ZE)y8c^`WxeV3p;8(`V|#sTan^`dC7 zJ%1ztDtZ4EUc4bUA(es%Rc7ly6j1t_!Db`}7lyhuP7l{j+Rv)I9crME6hiprEED7T zhlw03Hq5$GFT;i!NSgM&hzmA4V(;9&F5B@T6PR{w6)i_NHN42QyQI2lm@a%4v>QLv zLOk`RFU#&qLUgF%(b>Gc51_Mip>5*92Ah%#*iH0!b+2K1zi!Chp=yYVfYYrZ{F&>3 z3x6na6@jV3u!l&>eLQ)=ib?uYHcw2^iVG1wf~2b8J!|g8tFwqoM9B0fr? zdwTs-<8l2^oG&Uw-j3&}OWe{wjrTbti53(Kz5etCFsK)i0{{NC?kzzQ@zx849d9bv zSTW=ylr)EfPxqZF38a4JeKBg&3TYv|aJ*k^o#o%GIOA=}0k64(n{X9#c|HVOG?f46 zKZg=6^_?c9__ZVC^OpgX`TzF+n_!CpiR!JDrPsBL|M}3>>>Og(?~dBCVk90R zS9p|X@$A|K@Vk$|B1rcbkHRJ7B3KKpCScDv;r*_)Xsw&9XvrU>!02EuJxN= zjJ6_XV`up^cZJdePj`wE|C?08|7&t3@IC%N=Gy5i6yQmzuH#AV}Wj*NtB#fD>ibg{!N(fLjU-c37aR z9yL*OnS62DLY@tUyff#c)G&)@{1XG_$!d)uy&(wsvvu)NV;EqaA8qb*A0^}Ki4$Y`lu%2UE2Jerxels<%Fw%aek% z7c82wWSGr1pVh{hoOqtoW&ZsG8km>Hv&fn7JxHI8Q=%8c^j%me+9dX(7*bS}9%r_P z-0L)*0WX>IpSEvf5g`d=I;P3`2!VvrWY4dTS^v|`y{xo(QiLU7;5U;#ac$Blis3T1 zmf`bvxi@df5|c{FubNrSyi}0Oh;!7g0;6JrKpJPdi{JY7m95(~u8(pNF#abd?KI!j z8&%k*K7!PYJ=*UNo+YvJcn}k9-rrhpz2iFWN#smeJ8(%HH}Q@;3NRDI-XSrU+dd9D z3wrRPm4O5@G!_jIbmE*p78L?_iWr9D9d-0GAXo3_qMk=?UIX&$UzxZO;J2A9n{m;V z8ARNn7bEL8D`Nd)}q5)nr>mOG87`^u6k_pKQ^idg9% zd)fr>6F|Y4bRL3hL7uPL0q#=wQIiGZx~=m~$82@tBa#w15tlJR`|%wyiG(BC*-uh@ zWU_=!HMe)G03uUv{Omu;&)(&9$e5^V$v-{5GpT1l=StHSwsc`Ek`)8m$EQylBrker_%GPwns0Wu? zzWZs%qG3O}1bSzqa;-`8OG~baoy^IXfiNw!txNsmf1ONu<_zh*=|H|rsINCbe{nSu z?4oNtVf!#K<|8vT`}(~DDTs+u=kfH*oSK&L#BT@XeE#d3$}+tgPhy)nc)t-1y+Yok zI>ft|h{IVcR?Zz6CoOWG7RK^4a=D)0eQi!yu^4Co(2#G02E;T-Vcyq$7)%3Omi`w-Bzg^xVgFzQqCKs~qKp`V<8 zN`}!fNzHcWhZn~GzULOGn&15L_<}L1&!d~SYC#(Bfn1eh{DKSMtjgq{Zw?}RwqBXk zVf7Xq7W`uTu&T@fxpbUZW2AB4@|0)wwZ~@tO9TH?7Aj9@%XMe4v2=-#b0l9 zs;)-&?+mnrQ{Ts__W!^U#%6t_(v9wOy`Gb@v);2s>jio>TKnZ`Ig-UP3xhKlJ0F5dO__e@Q@BK8kT_U@}xmFj8w@HBu39(y0nx(T^r&T z>I)7UegTaki`1?k8=|6rsw$l_!C|?FJf~Gci*DK*{Y!kI`UWDAEU#X@+MWczfA`ix z2#NfF=;dA94!GoKO;pg^=a)Zq0jk<8_EnE0$x(QyH{-TVUL%MWd4jcF$axcD)d|2~Brehuhm>D}QLBg{H&vALo zYWCz}t2cJAG)K+9mM>f1$OcKA!}1OF9G|jJ_Wae<)qU-Z;sKezl@b`1LhfaE^1s>4dEj7QrIL?_ZL`u6~JB^C=i7(INemcUz)|bXwa$$|g zjx_($2EGZIzTeIl@hGN50UPG(!2P$gEDU=XyNiK`^F4=zZ{hrB&wuYZ(|kpSQ9V3q zcrti@>(QgSar2vBPiI3I7C|zKiKH2^v}K*S$XpWcT;8n)h*XFjefN=Ty9_A0BpGJ` z;~(N(RKEE6VJ!&Xw2FxVbN;+mub>V%kEi->_}HkpW`am_;$FnoJH}R=Vor)I&)MMF z&$B^(+Ju zuv$i}qxHo)@ULZu_4AQBqUS|7Z^rcMCg$wdAktS8r&p|Hg(}Be?~Ld z=+d)Dt~#IaZPNZR(#srK7}p0Scwx7BKPG<$>0FhO*G`izj~TmdGC=Efa?-bTaBHLS z6gorWWg7m5Kd{zHRv~Y>C9RBB*SU;-?0$rGx*f(_QtWJwz7PxEbpTLxE{*ly%683U zFnh@VQIgKUYLdIHSGc^n8HElTr?Q>T+@`Q;u_ime297pb@BNYe(@Cs}R2wiC0Hdt~ z{ZopgiAdgje5JAlIO|(P^o0 z=m4}TBb(sC>R#tNu8q?#A%qDj#kR^u1jZZbY#G2W$eXklb2jKAg@R;(6Jr+_P+!eg z4hY-u&^6Pc$|zp$s)VuaRqcOKhP6E}^#OS};`2_}NBCw9Bir0vctQh*jV+*Vt^KTW z(jBFSbQ~H+3gMj*eBqC`pG|I9ja1!1ra7!my&By^q%X{f&6e^-8P1B>gUMGBzezh-tRik#M*xJ<)T($%JIqFtWu~Ayi-D@7>#{UdHVb+C$(CjL z3ALR!%8m1-F*JB{!d0LAdmPuu6iLb2g;g6L3RN%xU-*%)qjhuBo`mR)l>Z<-!(tD# z+(_^<;pO6&yPC#O1$Hx(|Cy@Re~dOaf{w;8@KEP z-aTlUpibG-J+cRo6v3hr-~)&jmy3XADu<|E_Um_2VIDHJ7|?ad;ycDsHGlP?r{5L* zB=;(aD2AIWn~Od;%XF!N^|&h>@(9q*l9qlHnUx28=(|rxTOWt6S+}%&^K@9edMIaF z@n+cntu+%LBS8pXc*g`az{;Ou9&`Ep!RGvTZp`Z>Gm|Da5VQKEisl#UUs21o7F${# zo2sS)6HCFHGE?m2aT!>K+!Ql!-7Vi-{;;rMoJ^u2t#54vE9rF2z|Vf^OLspW@P9-% zAbv&J%=qqLH13$6*R;p+lMz8NsuMVLeRahIW9I-HlPv{uuP2OCer#j<2)FqDW`G2m z_g;LI^NzZA@SmutN#}j)jrIxQPO->5n1b$0qjN1fVhhq{U!{tjjSUmQ>W1)2m+tBm z3%BiugBUaN;DIdz7LE1AH{CjQo4mwH@6CRKPtBg~x0l@4WB1;R*ekJKK;I<}4@MPe zoU1cLcFLK1{w~sD+xFVoNW*@?kBs_9tRAZ9Qg7{pP1mZFIx=Kqg@JkC&I`N=>zLz~ zf`WqR?4PI$qQ1(XN~#mJrn@Sm~gN8PyOfzTxoUvf?8z^^6 zs4MZ!2tGh*`t{$AcLyEwAPx>+BhxQ^NIB+A0yRKjwB(t%>M)0&g*sDIfMJV9&NiWr zB-Ag?JY&{*B%61m_>%<%MaNe(=y6FTQ!#=EZ(N@|s0VH#2u53cjG$ArCIOC${{DhI z$;dDi=`=tabUoPA%^a6MIa2R|i}U26oJ$*x;_*h;tV(op!|Q~f_el#n$d#lq6_;DE zyfFg~{24f%CsnIf%|%wPjw_N0GPi1@^*0TygMaNR>N9URyf1x$9*G}LaGZ1@?+R}> z$1Z*stReoM@-XjG5bve|kFRd$J%4+SoJ%}lYO)lRoOtP$5izkMZ`!G4MevOv=Q7Ou zq4ffl58Qm;}+kyd`t1 z9@UA_Z6%kJIs$<{%zgcqE*2_fF+n5A@Z^bs7ChH{I|QemKebI1;QBU3)qW@G{`J+T z&s!WYWy-+3w+h7@G(1}Gs?f!d#A7GBUU_xWRShT3oZ{HpGigvSpEIe0<-~lx6_!zG z!oyL03lxyrGlpj`B}kRp;gKnXi>=K1gB_7OAgvN6z1l%VnQO!MdDNi0zsKtdXZTgwx{&F{M!A+@>xq78uzdC0W9Js!NI5@g|o`<^7 z7?s6MwausPLj5LmckAIt?TjSQP)<05yiloyMBSv~t{l;-fj;u5n&rlbJ6Ge%RIx*+ z0*|=GbIi-lowrCwf%7n3^K|hxemS{Kzj!m)PvlK$U~T8C*h~n{Fw^|C8XRgB%@5nC z>r`UrcX9A9KNC$hgOWt(ltimwv4#!S5~lE?5qhvbc{P>su+3wp^z_B2F7bw;7vJSa z8Im-TL!N*Y&(m+VSdFyUTnY_$qBe>fP3`oWb$+g!bkuEH7-#1A{@~{QF%A3km^9RG zw(bw~x2hMvNQfQW5aFbc6p;VpI-GUa95K4$`=NNlyG)Sz|$Ycj_8F3^>Gi&R1L z5Eb7`s@^^SkifV3w?1R?bYzCFUkf~szXP0qZegSbTb00z+1zsdl|vTt6nKBahxZtj!*q?0$6&?pKMD0~!P@Z0e1g%94OTYQZaj$k z>B6V$m%BQCw+j?XLt(uhj~8dc2#jZ3{g*}-_|3?@nBGjDx?ZAV;SHeLKKpo^_8}_r zQXF3AfpAp-N;?XT?|Lz{$6MI3J-$O6_H~-&m1_gzi-;ZKr>os3I#}-p#_JhOd9?wH znhR(u<03gLW6p4}@vi~h>fGfn&o2*bBC_eK#3O3mqi6U<+gFDpiSAs5+Z*rM-_JDM zj^4x<__JT*x9%Fp9=d-xubdT}AW&J5PoCUg0it4bXNJM@;vM9i;mQ214+e;g_0)RZah}R*^f8&j_@j=g3pEW8mFjdmZY$m04E2*gR*SQxFNKX?*U| z7stbH85u)5W@^0c{W$|#4+(a6@`dey`iqAE6dP^mQ(@#=U-8X=2LA*1I~zfi#!2hu zFhS4lRFEQC+b@ymXL>@Uw1>Snh<@Ijn2jMxoDfMB%oLqJ$ze0~nHqNKT!kXAG2+`% z@%1{8PIHaTI2s;!gp_c4@&FdhAT%p3>4Rzsd3Jy?M*;=>uv9;U!`j8l>B$^keEp-3 z{&TdhTd6@)&m&PD@F4^DDWjN!yylK&9}6;wfuCKjmGj0^yGJ`vX{S ze2^U&KW1O;O!8+egZz<9YjvNw5BkLrp6VM4(2DCB``e~teCvc08qlv9H&zC&GN^8$ z0Dz40LAXTjy&^2Xkg{D&x{A!BeDXZn!ex?y%Yh)G`~d#fc-dd^XV`@2JUQU?QVoYf zMn{Q!0;UCcuT4-fNMB7|`os*Qi*GbfrNhMwje5Rek=l*Md2e-YlLm+pde4{YRo^T~ zrx`)`lXvhtbggJXh_Oq6f(tDtmIpw5Y=#dqS-olx0xMyps#7WDQk@6NyBGA5(RinEjO6=$_sluf2o( zZkbIWNovxehwc$+wqMl!(~QATMIpIJS!+k)tWdRJZDGe1tKP^C!m&U^!# zqn;MjB_7wDMV(7)z3-JB0aPBZCn6WLzZr51n$_4-f}Xelx-_P{6ZvSk7sbQtkH$q_ zOV==3w>5C4Ve7KplV>dYF@1^10m}W;e^hXVwDK=@QY}AMeQ_kQ0DrTvJCR?1 z^m)6(+k0!B=q(PINGt{qz;m#_AMQq^`jjH>65MUhR!f&-lb~Mq-nj-+x%(*OQbKy% zRyf~eY)K~YEj3R9q&Jf8ECfG&^m9X}UoC>fOg+y8|3168*`uVbnv{2tyurF`*8Fk_ zalN1E69yh`rlMUU_nsbps)^w{(Nb4;$!pyRCb8;lm+Eii9}bq_?YH&T`T5z5Gy9$Z z#{0ew_lVGj$yr@W=(n+xv^0si-VN-Nww>v62saLzngAXlz4h=IZ@0vq&2Z^!!L9cM z#Gn9g`-n%TFr?(az(keC+9uZGg01eOQ;!Up6AF!txCn=(76j*$6!gjTr20@?Qr3nr2ueJwi3pavA8_DWV` z7-2y z{7X>q3M46uBRb0c|Kh;KM(NkiMCsVXderbZ?@*t*5ok~DQR>TN6SAbo4XolCVz?sc z6hhjsVMg;8T09v}kgOo%@kUBSUW45~7;`R%`+PuYSUC>m*%vn~R8h?r*?Ir7az73^ z*mr^bR7cxw)R!63ELf~Jt%A-&kpin@mw#LWZ^Iq?7>(b)au;vX^;J{*eroZ2I#fhz zEn_TrwYgZ}iQ;#6ClR*gU|U=?i3JC~OkG$l=f#yFOcD>)XJy2%S7?J(>4dns$Qy{dVb+ zy_hP%_O<7)AD9*~M#1w9awG%UWqb1vJkKE8&gCN(fbDwwZ->JPNf0l-_RGP@#`pa7 z1XI+Bbd9szF1M7FNqZU~klj#5sXf&xGKXIQucL9AgblZ}gJ6kVHw#G(7l z`z~Ex)>d8X^#o#P83;a1J3e|e@YwfWhI83#%KI3$C5;SMs2rjoGJ2#);C$_ZY(Fc< z#cPtm=h0+^LQ_Z-B8g{}$4cfZcc@6+i&6nUqd@nQEl*Gvw@X@(jO2A$+fn6B;9Um7p8RF>pAY>c zvS-BeIWCUckDK;CSLexsLn-$uSXxS{4!?Zv&O#De7iqNno#l)XI7=`SI7s{b{&#hIZ}6zFXHJ0T}9X>p+tpDud9`G z{3agVcvXg{eH7WEXg|)%Q}8ivJPH|R6;ih98Ngqkvp^3A*o>58ZUf%FSFLNdj=f7& z*6(%0S7u|uebJ?o^xQxy^AmNo_ecGt)R1y3Yr6*bFUW}Q=P4zsCv~MCOzsf-7|)MP zxMvgRiM{)j{b6n7y#O7VMc*xz=q126HE#oZUuZFyor$n8B4}a%^@2th6?r=xVbldY z?Ty_$C-5|Cm)yIdc~Pl)_Nw*0^Ml%-QoJVAm}a4xEsBq=>HBdfc)a6zE7gp73=+D$ zi|n2WWSUugmm~7&-lgsqRlHzw0EZ(<@`fb2=5L|T&eERJs#07gBKW4CNB)*-bq#KE zr`41HtqDDzROp(>)mpYlnIp5zk;0`SD;s{?hS;xt{nQL8^<6%r3n7R|uYTJs&wxuo zomB?wEB5BM_-k+5Bm&CT$R+)6t^&=mw#`3(OfJoqd$R3U%iq0s@7{J5_Eu7GXTzC< zQiDLL)!(7|;tmlJQPPB_O4*NEcnU5Jl{3(C26WOmvVu9UU099~jl z^Y}s8I5_>ji`nee%Z>4&cP*mw?gw|anKM$`z|yIYK*8Qv&CeNwxlQ=d1>^C6hjc2z z@r+AZ-mY=xbF^VWgFKT|v-%vBO1X&0FNCZ343y)u7WB3g;0Q94!11Fa#o;=eH{Qc4 zdofeS(Zzc}xF91(pXmEjPpANLkNQ5q1Fi<>j4c2FLe}?L)?Z2niV0q1%=uJiAZ|nw0a;6#C zV(D60oBHHF++zY5z9Wk<8&6>Yv9CbjTnsXw9t*vcn!6Q7S{yTaj!SRnw&0Ja=6!bO zd{cux6$d>D6{3?|mbo;>BE-9#PAS?-J0q&LdgRaXW%UyUzsA?2rG*Da9ROB-E#7KF zco)tjI)V9wVR``)lDB`UFLud}^2l^c&$OU<$AVBduevXs6lJVk%pEy_AFb~Qsdm#8 zQFhUudMnw7(<~`0vfpDM5PH$;p<_&s5`KEI7GM)_vPC~vp^|yRH*&}syIq{PTwP!I zC7eU~D5~YBXB*^G<{hL8M((cL-XmXLke}d#BP4zn7kzHftD0Aj8((BrEDsw;gD=u} z3F8R6XSTIImHW{mP@mWb4KwbZq!|S;Ow@V2mFpS!q5IkSZm!_Wsy!0yd$ilQv5r|= zIizQ(GTkR-6(dc`m=ro`GExAlk-?)TvEml4tIY))K3BxTT0T8<`G}q^hFH6%&krnT zp^v-8?}UvX^YK>}U9uAI@CSB+~t&YnHwHgW~`d8a-D*nwy8 z%=&ttG z^d*`OzrLW2Q;>|hcqR)>~Mgk@Sk0g z7&&KG`ZU-G$z-^IMxtHPqfvf_?I8Kd^4byJOM{V&^u*cJo~M*9dMcI-ZAG^)l_)bN zw(D9HQc7l*0`BdKK=_w37p6~-P&P%ER`1y5^27IN_O+gNA!~4iyn1j=0pCN+3p{}cDzF@1P>rH4j z|C6(khI?W362clN_2-p`jr>oh5H&E1mXixc8Et}iP<}4GS8qY%h(91^V46QG@A-bq zMC<(Jd#5ZuZcI`%d3MEb|6z$?FA-YMmrzDq%<3CCqb`rddBk7$`gV1taDEN@jh+*~ z7&-6xFe_qKUcbkdyV@G}!qo#&Eo+%NB1qdyoXxM%8l)~Al2Iu=p530C0e$T)BUl|y z<*9mIhdUpkH9oEwaMkJ}&nUIy>utMonMOp;#o|Hh1=8bp!1@rcs};2Qo&_}HMhd1=+A<18&G z>yPW}pYwIsbG)0*@C3vqH)cFWBf#=!4=sP`HyCQu4OgbW*yc*}XF7q?$+K`{(pC-Q zx|!EajaTA%Zmk!S4f8gvm1lYu?MB6Ty#-3v@E-C$zqY-8+HpR=b7 zUYdIZOL2tH;tsYJIGrM^E(eoqGCTk-&)H91mEJXqzz|MJgWQ#sqk4~$b@F_f+rJp` z3)H{A@v7VNT~0V!v~+44kuE89+fWNYGF{8JLv&d{sJ(V$358EF4qgmxG$Ym z$1Vyh4f1^Jb`~_Fku$|PW8Wk1AZ_JGs+HtOGEQpT>*Zxn8t4=wbBD~SPmAFQhtZ)1 zG9jcshl`6=CGLFoZlLS%$gi(vw*|YkrwakQpKvK+!Uxi$Jz6kySAMx*db&$ZYqk-e zh`;)GHy7NYV1Ac=f?;rl`)$AIu`O;e)aNxHJx;>4uj^9&}G&uVGqGa{J{J_kKW1&R$SH{tz z)ilF&$~tU^TdS`iO1?URZ89A=0zRy*U=KNu6?>kgUlbXsz3>K6-pt+{^h1spnwOQn zICs!qC;t6lGsQ54a;3luW{xSI{U5ZwWmuD8A2&QeKtVu6x|D8^ZV;tQYOs+Cqf5F& z1eES(f`r0GIJ!YZVuaM_?(Q1(&ilUK_jtcNAD{Efw&OT=ZRdIII(Gg4zY;I~q4X<# z#hFf-`|cw%$EJR9^4cby*jV7-xn4=WtA*jI!<*W!3;A&hJ(6Qpuz6&QQ!4Xt3hs&+RF=NVDk|fEoBU&__i6)ptk9tW!#wU zF1Z=5U8JzEXFRS`YA=Ssv{_8`QD*?nBKgmmD1_R;{-E3@#5C(vVmZZgtYVl@S za&@E5MzwqW_9GVR-1hW0A(9U6Yiz&7sP<`~PIczxb!hy}I^F(gHLMC}eA!tuU_lPQ za(@ji+Zo6DVn=rMhR5dnQm`cZHH#($IZB(HNN!-nM~g+H&iw+N{T((IV23GiJnTj; zFfI^@FJhsaH81~%u1X3Ycmoy>3~6PLmfi6G2d~ODSSEW}ehekKU1ta=ursu=xl$Yw zJGxzzZqG{J7NZZ6w(OZ^5y7>HZVP^*!<+M(SmVKbY9T&@zKBt>auS^EJ>-_R#Ue4q ziM!uhl=5_mUUejMvzt|wfjh66CN-fjU!Gr87dmiQqL_5-h$*(05kHgN+4mZb@{;t* zk3q?Wa8Ea_f>#M`_Y_Ro5_NMd7AY*`yrndUhR&)8t*F$q@2B4sb^jnn0t{ffin9l&?nUzHK)$1*4<4cngWDbeU7%~?PaX1YBpF^m8PyHVt?jOhR87j>=225XJ!8!QADEU57JC1KIkVw? zcWOhs|Ajg6%~KM0%6&j-hO!mxEnj552B7YAlu-q*7**(&|n(>wS7!kouF|6op1t3amMHXt)^;kS2(*<;B;x_}!tgB9g1<4JBJ3_qP(fyaK3 zw4{^N3w@2>jjFOndp+thdo`ZdjQASnHS&lCdGg@0hqd~<32`Uo$jB)NcX^A$8`h0B z8~!_5sb=MzvQG%vlyc377W{e3OO*;3q|#27C*A(AMVRa^`3@sLkPv6CSuWf9kYsRiXrb0g2%j=t|u3g{wmf|FB$6G|0)$8m28Z&&qOCgXo_ zy2#13QLfluZ)pM@ncjaD+R<}SHId4OF_^BZ@a*VqP3Q1872W}S4RWXZ9PkNB@m?2B zU_epnF<~>5{__eNvP5Ow@YY~cSVuRU>RQ2f>m5f~cj&!pWKS{Pk)fV1X0F+1*d7K@ zms4rUNuA2e8>KBA2BaE%iqvPwD=!_Ea^6A3rHhbpe8a0mq_C7|>cinU?(|){yB^ zQsa&k{r>@hoi41bs(S^NkUf6>68)g>@b4inM)CzX_jOKnKR{yp9wOpErxp*|X-5#Z zy4vkaW%gtG3{r2wlrsJ7sJ8J$MBMUeqgz>aBwt6^!=NH zkNg{Prb+PPe+fu z1_QW@?E90ge}UxM7+n(1+y}B~EvioOx{H3k)sS@+neaqxb6iu~vbbC$-K6k#Zf7e=WuJ?2?3+^^7lG%`1*%T4^&)^5vM{A)I>6PHN%90f(V@x zYVxcHo=-j}TCQ9rQ9ONSa-_x+oz?e&n87fpJK9YS;Ba$uH}C}EwUH223$sgI|Mj3` z_D~jjJlkze2|2h`(kh{Or?_^8`b0CrFV-{7W^i@>y>;Z7mO4orBl<^k`V9t4Q)6*J z==<)vk311IGFUIbl#hXImOZ7M4MuBgLVjIcw8m=Qko2e;%<vXsw(QO9QX}(^D@K4oz@E+=3s}y+9 zY@fn~!eDboZ~lwTiMKcGWxXNQ)79;HtXg_g-(10xLO!aSG{*6atw~*Xus!C3H-34* zc!N%Ko#oeM<6)R$_PfEa_W?z7nB2&Nh2Ygk}4w^*YwP)YR?+#^r&*!y6M5WFTZHu`HXGI;yb75SXni^#G2#A?_bJy zc3dI48%n(M^9KPx;fB$*ow;)NM`}&p3Te~pIF#MjwcB;17#%+SB1RFxnDNf|F`RC; zL~}vnWqU8bGBMFc9Rs~wV{@Zp#Faj8{g z&9YYmn^vtGZrOW4OE64vz%{UBtz?%xEL3Cp%{~P@oGx9apBW-pk5*aZoIAFDi>zAcP)t&NUa3 zi2JT8K_D@^q&zyH926416fn$>7Lcr1Cc_BJR}P)pnOIw)Xt0# zC_KRMr9yn$wKZ17+27(fymgUJd%*imIC-zX+295e4BfR`dhE=Sv2W}=Jng5zh;Qh5 zmZ$Rdpjlu%8EIxfM14+ty1}Rz)2_}0qq3t&5wzvJ&oUp0b?gj;GslTh>S`5)i>w6Pl#=awySQiR<_ARZxo#cr~B=R zeuW>g-|+FUhx!GyJS;T#5hE$a0xf7_1JGtieZ3YicoxZGM`4DEBXCsFC_N^Ek<`v`?T-9 z-Un6Zf_fd`Onszr?O}Kl^--H0M6izMn!TZL`s=3m<@A)*u5mv`A@v@)QSTE*9G-+ufRmU& z^!}L3jCE_6hJQn|m`ry)aa{fN#O>UIAl-AsG7D3brlzWInfrx?N1fkf8tuGWt4+5C zGk5ji`H8{#_rf3MfdK)!0uigHgH~Q3LqA_NUnTKt^>TaLuM2{T*Mar%LuGt$?d!`+ z9wZB5-&9afhCAS*)gKMP)QU_yX5APbIL7edRI2SAIvuhurXh>^d(L$5Rul8X|~KPdHx6N|O_~WqVc?t<0KFYpo;iUUccNB(G3k z(iUdI<4yatCp)P^^D$f8>zj`i}_QDwMkV%Q#YHxJ0+U|4QA*$Xi2$?o=%!2Dy-p;(kuyxCn`Kuh6Cptx@gCB zl1j;1y|j1QXkmKKDhs|`2GKzpBy~9%M9kdKqRFervK6g0?0Y5mu2xaP;I3yqVAkGc zw0gu0de2}f*Fpf1q`vB<(nO<0U^U!sfMOIHi;xrg@B}`qQmkp;LjZrv#YSZKk`;mK zJ$AyI-hSt9+_1K}|6FKeA3}dn48(63LfD@9;!STRyEoQmc#_A!vwzFZ!!&3nTy$Bu z_m5JGWJa#KuG5BzQ899g#Y1y_lEzk6;#=}Sle9L5-h zJtv4XxLU_c3uu`T$H>o8&kT+iqWctz6UgsYa6d}j(y6BP<|3m4kX;)9jEbJCAhZ(0LnJ2J@~ou z$&eP!590S`+j&aYYBOO`$ItC&f%Xwz8mZHShxLAIUls81eGIVx8(ULYNwilM4 zFk<^Rr&S(=A_!+QkT{9R*UIn&PxosCu}qr#xr}L747vN@v8Su zn|v#knd>EZi{3o0`x*Q~M)YMbsf+IOozV6(z`0)o)*W$wZjseM@BE>#5^&_oWKD_P z5KA)@E(mDC0dv=#$Prk*0G)TF-Or^na5;BkgI;=kuA-dG{DQNvfOSLR`fQT&Nwss{ zUES)PbEmtx=;NhT^rwj-MxvY5CxC9Y$6v{Ze-Jzo3cjRd*Tyr$9SnHsx=Xwf>Z~#J zQfdW4o{sniH``yB`)c3;Gb)QdR&l}McH8OL1nR~mIYt3WG%<{2tHiv~kx|q(g zZ+BRC-|wk@Jb%vPZTXNh>xA8qv76UQVmctri@VnIuPBpj+tCYTEtEa9GtH3tyF|NA zJXem#7U1ujCaS_%c_U~9e9YA%?0yZ-?O+-zdZCv_8PX=I%WRA>TCuPg|JLpn-F3am z>u_l2#p{0=#>JI+N(PWn{|Ch3t-HwEgkz8W_%)P;eVFfc^m(Hq6f?%7UGo!a)+Vki z#uuxg@CCQ(RzEoid35rb{BHa6DJ*5yvu2lx2t!)%No%d-ma|3vht72UA}t0$mf?~y z?}LnT(`T`!yN~?+SChB~`w@sVAFqb5IAl!*0RmdUOzfX}l33YXVb15}u= zyC=$-su({jHL-%;UuJw8JT%gmNDk5Ty=;x&T@`f|Z3oU!FJ7d7^|Ncmxvr~)wKL4< zURqv$GBypJy=@6Yj%GLNYael*L~ejFs;&_7p&B}`^E$?m&$tJHzBe+tctOEyCFwyP z7$=LM{a#8oW!4`ZL-(T5Jy~}2?)UMai?(3YS5rho-R+WdwPdQH_Qs|SyvEF4Gsjw;_14DmaO~U^qJ@?iSC|H@H0vG| zM%Fkylgp1=vzQJB`^eK?><*QSag4l0#W`+SK1}UBss|I)fKXpsr0ta`A2ag=wMUERRKrVE2_Dr>N$Y*?e?u0)wKj9p4=are@4CMa@dnnnHQ&qiKg7t*2Slnro z{^@8*@T{5D+u9_lufmYeO*M*^`GmRY@7L&H=12!8YhvJq1`2kiU0iTx%%WPpr&m&K zT4YK-p(R$AQhLew5_>-r4%sST<>u3AU;(XTwz|!yp|^#^b=kq_r7djg_Cgm`#a~=O zzPrT-(N1L;B@T`@&^=E$$ZA-XwDP)bNn2B~5u&+&(cq~a`>|Zs(zDhAgB4xL_U~T# z^XynP8o%;?Wv9rM;$8@Lgu#%^8~x608;D_%o}WR_-l0$W0AT0f+9j7s>uU?mx6j| z<|R@I574&qe!b64rRajwchn*hM4l7XzMi{giXOul96MxtRAp0O+-MVDvVk$4mSGBk z2Vw9ZW5M+TxqW@ldxa6Lr8NH`O!0FU?>b<(u~tQ)@CLr?e|#(=tMNS*U=lLRuh_g05CeeNv2!*C4CgNCsMuXV15zfwGn)>S5w*8yB>PB@3U#K}%%Db zru&cc-|qiFE6vKk2 zNvrs$DJ*NmAGLXsQv5Bg6VISr?H*^`g+SN5jOA9-vE(Fub(-JAcEWE#B9;)-ADpUw zlH0Alk`ACG6rx!5kisvq0WR{n*2AeYYKjZf;Yg{K(_u-|U60 z;sl$$HNmLlwZH2mMK+F0?nO2mL96)`Z_WHJPlAAJ9s=5>r%QyoIHF*KDbZ>5q5@32 zBP1YA==ATbf;}!#;YOS9+&4j;0G8Cvh54jHczO|ZbeaV&GwT47Gk!e+Ms<(W+{eBrThKWCJ0$5vcGw*85uYd;sY{${`=cHT_RMXUn}^2ancW^vA~x<$^C;S#qiwy zua@L1OJ68}5%k)>=j>qsC$FXws#RMs0}ceMu+sAt0v*$!IC$zOzA6%NVdlDs@8SF!{_mQ@El-J7GRP_^0yVqd0;ePtQ_jh@a_*%qRI>U0p%+R(wFBd3iKXQVn$* z#O~^WVez1swZ^_BTq%a8?b_4v>DygFZ!~?cF=863iwnO3_}Utu4G+jtRqg4ixw-2<7V6^WA_79{if(muXXWc4D}6N37@kPxF#0NB z>HO0xU6nE)WiSa{qM0Pq87?0?tAk=da2wf$I&dyXm7l0#ZSfbS!qe7n%r*Knu^PTY zAwz|A7*WkL+1vkrXoR;#>L&_w0)bM#TGDD|zY<aTZnHzV#qVB_aMdWP@RTs4l5p zd@3&_%^ROxmWZDXSw4R+W>P)7#x3(?3lhLDRlxvwdPGioKxp8|^Sk5-tgdCNvebD` z=TDp0tlR>b)Rpi$I&+DNRegyg!i_I#wN}$rV3j%McMXwIZzLJ?@19)hY^!VAVZl>M zUuvU?2hmrzK4I{$4yajxaiN$vUu^jM<4{;bd9s1>)S&!p-x>3?nU_X88+Cg?gqldx z6lK7A=xW-Hc3tCZVQ>;oE%DKw;69y48rOU{k3B~`N>8P1@ z7;!C^778Nf`9Su}pM9<-rC1)N7utk1xaLUo=7E+b5arBYVmqS@ACb=BNTt1t_%40% zb4qc_{9upOfEoV7?Q!ng*Jw=ZWTud7l;Sk8b*WlE&~dZi*sQtvi#}%}PqzM5%La?w ze_G)e$H?pc+msrCC#_x|%1q=*=&bxL%9@hKjr%B6U(|dAjwD-7tBvo)B&!kabJ#tJ z`=;A1cHE6zW-S>_H!U)w)+S~VDlT%q@V8~^MCd($IL3BnX)e32BT@oldb?cn?jBINHz!+g z?+W-7aNc*nK4S?9zZH<9QD~*P9Hg)r-_5RL&voxZyN^XSDd1>hzgr-pJ7NcP9JI-< z4vB*nY#HPs7d=f0p1-XL*JB2130kXoahSe)Ih^nQ<_4T^4ar`|-a5=kE*`wNEmpjL zpH0H;q=$9=&CuBME3Nze^kNIU6!&5Uvg%XKU-Nb;iZxw+2bW15eX}%zy<@C9K4(_b zmi1Q}?&@8~L7tRYgDbaa0 zBn`-yEk9X2xZ%ng9ho6+isou^_=$>YpEq@eqB^o3`K!U+n$U0?h7rTZ^Fs{Ig4Tsyw)Y9Q{zmxFb(YK?i#An67| zbA65f^kMVaya8pWXTF7( z#2(*kcO1y8nZa6AKC=#$#o^mP;@1$GV2Rp3D5$}{Jc2#u#A?5_`-bP!SEj~| zdzkcb#fr0=IZZA^?Sr`-qV62$vEFa5*aw$}LU!bJKI|I{9M8#p#n`k^Hv#1p65YFM zo*;G{?Dxw2F^?ZkreOdZ!6%FSXWnrEPiUM1=@v`F|Wyo?X3_;T+O&t5v}+; z+H0K>J_ox1_WS$rD{g#}?jZSR(@5N=*BFjss;b7v^yDS0fS z8^FjN!igpjUqQ2Bt|&%Zf#z>cAuB|3#oNs7Iy$6@Qw%shdP3B&E5Lz=H~eDmkx6yy zc1q1a4dh1odnk_s)<$l&Gs=S}dJs*jC6jdh_$%ydSk?G=cJdUGVKJ%VxWhFNU#Z~%kAnlalXq&wE{YF!D& z0}4gI&gC1F?i80W2Y^)b?CmHvYG~0a=z`4@xz#CEZWE4{UEIFrUGkj=9D2v+0&OB) zYHwO$7JmvP2)w2r1x#)@)-4-8?Lq&r9>8}_7%#FO$nj?5<{45@lhJ|q6fii;b#SHU zXv;0XMCl&er1@C+!!!K4Yg6tZB@A7WX$k{|NhJai}6uAr&%9ib*EcKYeu>BbA^cfefhE#+_( z5XZ*4MeIcFeBPWg+4IY+^PeoDM65 zZ{k$;smtnHiGPl#WuJPgr~##5qpy!J7~1WSL|8BP^lRF^7c$80Z8WVT=M7bF511-r zam^Gn(+t(U)X>-w56;wr4A(oj9bEn$DA3p1828jgETe6(#16iurohqjgOnl!7_+1? z5xhl9-tU@TBQ@}rqBHrIsdL#mJiUpB*s6BVqst{aH}yH#TKdpD-iS^Px}ULGl|l{S zocnrNh44s1adYW3xqP^vFV>|Bw{0?aAd(5aLGtI04%+`~Y)8*e2qd@j{V$zMf*FL~ zK2Vxyki0Hwd~*DX-8BEb0?3KS5X)>7im_HDO4bD>&YQMQlzMP;Yr3g}WY%u72NziK z;qhm@PYmj-R~UL4FZ;?L^F>p-OPl_7wqi{Z(F_Gdim>IwPr?cK8a-BVHCL<=|8OoE z#Cs?>eHg-@RIo_!?j6)oI9;fD#fA)meFG~UjxO_sV*B$JarB{m`c zAg`ffB$+Bw+7n1+>>D}5_+>13Z{6SEk6V- zQ~`EFA9cS+Z|%@7oCFO?W1Pxi9TNc}p3$wR3qo~ihGI*2r+5@uh4b}054z&J$y*I0nKoQ>DzN}=tnI+j?0;0C2U)JWyYa^VJ zxdSd4M}rZWHbI>|3VQqHm7Dm>wGn3!smi5D{XRp#i*lr^l`u$-&Fr4?SW`V#vthrL zuBs8$f*kn+wtRP{Hs?!7C(*{{z<#4hc^`9Dm#6~F^O`rci#E-z#49?tNBIdkbV`D` z527B|_IA})*2_N|xtX%T0J`Su-FR-k}{Yd+<0nLWO(+%fY362|_% zN=N5bLyYb5;6{e4P1mPDa^pvv?`i^|c(8^^x0eH=9lzqX{3*j_T>=+g;iY$#O5My% zCWzH?UouQ^+WpL8kN7?ZUv4APn(()E8E-}kE{BTNfYnyI+2=p0+FR>DX*#U%Ko_g(A<6)*I zFlq-2|6nbO_+FSW_ffp;l7wtQa3l+4?}nMR%^n#{om|~Bbj5Y6UYv4vjKkTWPWiKs zN+sGPaNyXDBU2C6@KI)TE-J#~j^CkuTK~lby#G)psoZ9us<0_LZ3)KE$^M>i1`P!T+ye`;bBM%jb}Y{? zTYA5Er9m^x(*F)@gXF+2=Tuso^z$!c2^z*95)Qw3@o0gJ^y$<0_`m^;<=n(N4sJFU zjJ}V_lKr`c4L-cN$nYtH+vqx;vWn4qV%Ie)yqABsDEo{dtB}5z2UM`$W%J18G ztR-fd(u&A8h`ttRH9TR2Suh&DDroON-G^qVKkaUs&w9|_Es5vY^s^_WC+o4v)lR2^ zA4!PaBglt^gPZaIj%$vMDalZHA%-j}I6U=4{AjF$<*x3}(@oavkFvMHUVt2Q^TAA2 zC&rTb!tXYk`vZbvlK9nWd{_iu99HGM^l@0O7U=piTA{lzl55r37DZKj$h@lB8 z6l==nw$BfkO1&O@o9^pEPU{IRMmqMDpvaA1>r?uYPlOpSdf3Hpvl?6}%)aCn_fbM8 zL+aMJ0-c(Vhqr%xgxXWB!)*-g4^xg+zhd7O&iY~8oKowYoM3hEs*by)S?RJc5uC^h zE`5&^aC)NEheq87OlSnxHHqQOraFh=xp@Dmz-ImsJrBamCduot7EgRpHU8?!;xrx} zEZDaHu^8`r*vv}z<@cP%SNyTwm6G16RE=xW*bCdyMxgfQ--v0??agCeo|6E&`3YqD z-bs}M-Zf%eyghh)jAnjhldAR^55p_Dx|YM_`Pfw4o2-4SvXUiPo{qD5Tk@;uo^%mu zU>XgBjXzX@xYh7v6wXD(uf8{UoNgmRRdJX95^G8%tO!kTF85Z>a=c2*kk?nh4it?2 zU524^o*VGcfJ9+-2T)x$$g+1hYGF|pfh}z#L7!IDh~uLe?PK5{x-b@-Tuz^p?aAxe z%>QwvfV{U)vT=BXEU)7H_cJbf+{Yu2@$8tN6rxN9hp8++rv05g5f*r-gHH9#OU8tR zG1wG{P(DN&G6B+8A>0%)w!pH#w)YXYg;**h&F%%=faV^{1IcyvpoVLF<0p-7N(jzd% zWOAA(BkaIX$v3TSztO*%7`uAZ;5%FwfaA&WrvO*pwHe+BFeU$BA_EATcwFo2Co8tY zVkH2|xf1NPOmmkV2bLe2z5=Ov>5*7>?J2XVYTIU1B`x;z6Xt3G?MM~30dEUw$yO-h zyN?VsIsNOBd;6k-IQIvH?#GNm_Od!&lLLNCW`RCLEB`w6OS1XiDVAI#d2w5MtTaQg zcGlMV2vYqE%WpGA4@eYpzx7wA1^WpNKMc3sJ?hf16U)mM4<6K~nbsXNWQzzn40cXg zqriC!6QG6H1QRjq%Vbo<0~B6o6n+>MnXUcxmiN(r#ut<2vBLriol++G6lJmEZTF33 z*O1NyVS^?sNVu`c=+*k?wp=pH+hXc54(!FAg)Cf}6%#r<*8nAMk;Y6o;e|ubr}k3k zFYCTvl0Bd1#TuvRp_qRnv`S^#JA6Re-F?&FTR(L3EcT5+B@p3O(x5vD3iej301r?8 zGQ2^b>W#oM>{=K;{^?`+fl`eTVo@W;SZk;(UdHU!F{!#QIE@sc1V7CQQPV5tpij+M|Zy0jgA_;~4jH?t)Qyq5=zgt`>O)&nw=p zm-}+hSauM47!aRmlveO|NyXw15#k$S0uNI_9tL-UI&P#^dZU7-SdVN&I=xPHb5DLZ z!_BXB&3~1!p9%`Qo8m2CPULug-5*l$WoTffXcLKW9ZzVDMD^!}ZK^-}g4=lU*2eh$ zQK?_@1J>MKTQ`$?k@jnAE%dePWsipJDIAe7Y9754BwcAKYH{W1!$>40agkSVvTMT) z^OhDWO*E|=$lhR8-N)E@lxA(h9{tV_KQ5)=6$QV8^v!BMbxd{n12Y7)-v?pQTw`1r?p+V6 zS{&&fy2vEMk#c&`jyD4qU%v22*X9T0Fy5UKau3cBV@b84p9@S20v`HSct(21;lTH2 z5POTJ|CQ^pC5g@6{6`hlt9T;7Xpy@|@Fr-!c%?3`BJ5#ZS;Uq) zR#06RAUNQ3YH$sWs}-|QqyEkwUq=w_5E~e;{1GHWhPiPS39?S>1B>xj)r#FC8iWq z&Yo-~ej>^JCHialHl?D9!$nJPPt@MdlNnLUiHW7Iq5VBCv$;XA(mZ)3$IfY z1WCLca|NvJ-3%4C^0e)K$3()*MhcVtPnqsXD2gUFdYtQmQK4B4l)rCpW@OrW0(VFS z!Er>$uX{BuRxPt$Lv4r9EZ+r`m;L$Ub6d1|?kqv9Bo2nxggE88aJcC|>r*M%SAHN} zoK-7;QEcVh&Fv56Ii~YO>*c4MTv(Z2ko0a0JK=ytG2M&j!6kAl+ea7G3O9?9`Pnwa zw=3m+;@Jn0!sbIg07ms;Z>>;_pMu0VYI;GU{^I2JV7%09>T)mPW{MZF$%jvSBU>hA zy?4TGd-muv9)=bz$UGF*0U*3@RCbU=GQlFW+6Eb-7&{V4g7vmKUs0M$;Sv_8BnXipTi+`X=Y;Uy+`9`e}8l;jhiCf z0+vj78MM^j&Pd!hTqj8JGZAHXmJiykZQr8tUUM15fpUU+k5}0&hFdaVHn~-4klXz@ zb#Wcfz-c)6w{~t-Q*8#NvwTq+kI{7oL@J#?yNQx}rcX2JbSPI0@3B@ihLtyddu+TD zF9#7WD&;Z4l{y#@4rnS%N@-O}t*+QNR7t&wr&TNAC-h;vHm2O~nm+eP)|aGrYNanw zD;fFG@<#{UOK{I&Zg%hQn&~5%ZFQ!}KjXYJQ)Cdd$`{5#1nKCiYdOBFWE+smk9AAD zZt_0OmKrK@o9nCi7Y3J8Hz4D0eJ0+=p6zxWLm(W82|#wsA- zY8?!-Z%=G*SMTjrvj*uRzWv#7O)e}83GA+`klSWEu?#KR4!NhLvDUeUj6`?_$4yBC1Q!=c-pIn0FIjS%0RpGf^NlWW6~t3BKD2UuL} z#5Ufw{uw$r_C#PKCFK61r@rZw9ru;#4pB?>@2zRDaHB^jU-x`HVwa{wo@`+57F#Q~ zF}Y>I>5Zzo=fN*HFzVfcBb!ch%k;I$N3;laL~V)|TT)G3`1`MO-ihj|>21?bAzp#j zQr2Cjt8)86Wnv#4^39#aY0;OLV?!m)u!>&9Y1Uh9nok8uV;-)GtSOMdKn2rY5@krq zUS4F15h^v&+4+bOxD_?ElLJu5 zkc;!x*c6;foA5&4qU6jsYFIY)ljmk1_KzW$q)kj%du6n9i31m|tcN4=C+MqgZf}*V zlhl-cBeRHCfqHBz>LH2pLb_eT&uAS`(^5lBom^`&$^{7T+gq}v>fAEOx{>OptsG?G1_Ua!KiKj@|* zSO(u2ZPC(2M5qd3G*mxN2@kz=5p3%2p$rofi@AHmO~H3RY;HfgWVS$4t$6p+KEZRl ziU!1b(65ifVMWPFH3p^heS8%xkSRi_dFK=a1&y}feJ1!1 zF^6Nv-mF1(=AP6p;h+0cG0z03Js8TlvoY}X#-}w>Uz%AChcELF?kX=aVT>pzB>H)GG$zpV+Qf(wlMq&bZ6Z3 z2x0HP`)Sk{8W(t#==Z;a3rHIp8vQU|f$SNuhw$fW?ol?D9{)?H#ZY`aP6!GdLN|Rt zgr^^F+?vWl&d!j<;~hPXWFYm^*G7Y8zmaIIqp*0WV`+q#uh#UxRj_djx^!QmZ?oWQ zG&qMlkAFG%Xthp_ivRnzLZIfP7D#>uCQ-rKpXwp+Y|N0zb@3VvW|^9hvV{3|&Yi%6 zEdhNQuRTZDlyu@APB3QNYyUe#T2_jbe@l%(%#g56Gm2g$<8il4o;_y!>o~c&Sm^5M zmP0ZU1RfH80f?WTlt#;|IBLMc{%z1cALgU~3MuIiCMaO!Y$mzGJ@1&biesy=c+U<}CnDJLlg| z)BetwQZnd$`>3Bc^$s@OE9UV*ia_DThpwwFl;|5Pe2SZ0BVpS^!H~}-IDXoXL@@gQ z--xX7KF~;)CDva}Tn9fG+2nX#95kR`uc7I$;!DO=UD1O>MiEL1n-lm^9<&)d9_3ty zJtF1W+U$Qel6Gtp3t;9>%HMQI#fB=rjswK%;xq-1Kj}V z>{%TDI3a_<9P=(;f>hQERBmNIx|S5-aRTq`a@DlL4I#ZcYVgV6d zVUV}5%x!&73!3QNe(b8s?{>y&=!*(a3$~abR7IA8pN4qXWGQn4V*c`F9DP>Y-@Bm2 zQUWr8oUqO}HSsnF=!uJGHGpN(fKwC$Lm-D4MNZCbOi|o_KHQ+6SOcZOMd=RRa%S2g z->9EkBbX%X@jD_6iF5XP-QJ_-;dCjx%b9lZ^vsd*o?@bLb6 zR~nHl4=lis`!&v$8uM@a;A-K0<7Xb`4+(jF!TRTZ+G}^8Q)^V96__*d$=5hqVx~%| zME)czja`AP54;0y^6geANB;M%KzzJf#%dA^&mm&VIKkUkQ{ovlK#-5WY}*uFWnh2P zFuvn;=z_V^`E1>A`^N(;(J8)m)P0SUjB6E^g_S8ga#!MiKfh!ie(oO3D{f7z%=y0m zT`d37Z~hyR5YeNWetDIzDTm9cd-<4tJCKnRf&x1%fpF|*Q_3`YNqb2^kx`C@*;Z0$^{ZcP?mm?@=?hF++aJN35;VKQ>(W?4Yk|9y&N;WW;e z@>T+Ju}fC0q6T&)F=@*u5!qG@v6u=%4pt<+{s$9xkNzN?C)N)Y!i2{Lft}8CF!8Tb z;MsTQCcrduvRq(hnl)?o2L{29Jcf|wY9SWEk8rFLAuxj&LhutDMD6VE@&Ak6`QN(# z^ROQZ&q(;sxUciBSkf`Dq3u5CO?*(Or1ez(36G5fA@r3*xF~KJlO`_LNAd!4dp%s` zmmi;xJSTnivi{Y>Z$CotY{>}<%9K9WFyUhv|NZ=#N}lRr=%<9IZ}jfOV~j}&c;+%$N#a)c7{kx9Ql*b- ztKd;Jye_rXOg+XyoU3SO!5f(z>bpcQHltnlt?d1FM=RSM*{>xR}vsp zrH77!1w>*(lWGAe3B{2jgc3xAfPh4rB?!_%IuVFLcGP#j-TkriCpUBE%$eKHJoo12 z9^L5toYCObm#(8LVS!P5SvFo<8PE#%vd=D!%U96`juQd(KUv4YZ9>-C3-COwRj*3K zVqdLT_8-k9V)1Hrjz@&JqZT_|q~_J#vn5@GT~vf!O9Jwx(EGRJWK8w?M`--OSjMuK zmxL)SCB`ocI%pHrBq^l=Ti1h{YCEV4dyZsDx-j{}NLA+TYk=X-=ql5`Og<$_ zs?6b(_S*wy*Jk5GbHZx>aOLyRDWMbMX>vY=EaaTQUv|ZLG=#C-N;M6Gq5di zU#Q_->+2g2Frr?ZkVVh?{oeu~Oy1e;E4r8TQ^ZT>I&$WHVVbk^zNRJl{Vmt?obmj! zKugpa+A^)$IVX5I>?3l>7=eb}Nq<7;k0K)<5p&*AbR?w8rX4LZDdiud@$~PUEBqdS z#c0M%#5|{cB{Bv%eY0lg_EnhqBul}Jm&CNoi}xb z_!YiqUlOKy6|fM{4~YCHx$#2!p2%qxY8)8hPq22?a4oGptYElKNzi>yKZe|vjervB zy0ulIEx(G-TJ7T`b*c+9IabF@125L1)%=uJ2gm}cm2OG;PtXppbt9tO3EPVWTWvsM zvkggoCBOGYBIi>+0oLoz;VkzDW+5V$jmom~QmdOsXjehi!+DDy!>WLDqlOD?B}n70Anvj z^s>cOjeb+O+Pd0qexN#h=ZAaN2ap%d7hn)hR0`;#14+ zs(KgfrPHqbfR*$OZe>I*3|@fSS5^6w1yukaXhk&0^PC$}f~y>3L40`a!AOlQbJ8&x zd0UQZ%M}Licw&jg;n{qV&&WK$v^G|wNNdF`a*eG1e0nKXX!k=pJ1Zpa!c$6l24Hnz zbN}daw2sLomgS+6HX!~rF>^NKjI?kIM{6Bq&NY>2{|$hIEsUH8V00F8up=BX{GqXo zYV5anHmf$O^lqz5@I)04gsqW=9O}ai(;r49I9*FA7}gy}M(0vbB1C_(+`qxEEG;}X za>T&ZLJc@4uQRJ-k)Jle;G(2Qq<%XejonFwQkhIn+bv58IafVKxZMV1Ci$8n>4M4oJhxQ0+9T`rTYLr~fJbGV1+_sz3I*u? ztqlq~kcj`eZQrLBrIACvKwlEkEeUbe^0(SvArj@Mtvwr_7ITPv>Bvft=E0plCd{2q z-3=y@@Xu-=D9|as* z6S;QlvX!(8!sCtB=)u+@0jrNvZ$lTOx)!Eb9o>>!QJsA5#b?|z2rKP5z8ms*dJK}r z@_y-BnDoHpT12J{+bgHaE3yN}49ml-6Csm~lg$(3ZpD4>KOJuD+EsSo>0xODhC7y)%n%)hzth z6FsN*w-~-F3je1z{wDsE``-$H_Q39-rjzXDccL@w`jJEKj6^o~22V%21)SUS{7x3Q zU)!J4>(9E~leS)bdqfoORTA$;DpdbT!m=b~)~oqJMepPVbJ!atVi=Tg`>ZYu=UX<^ zha9>NH-}j~`I9nJ#ERc>nzw+bdW96#d@}O_d~_F!Hq413P{P~Ez$XJ>qY33%$Bxx^ zKOzQW6$E!GfQTG}+N3H~G@ChXnHtJ#&PqKLPNIoH3R9QI zDmyPIttLk?Chjo4s4gZ_4rw8*O?+9OT@BoQMUdPTW*SMA`G$0ON8~q#W)^*J(|5o{ z=2em8kj-+m{=SR2A_0tF)`^OesR&cEYeiF07PydVKhW;O<&YG@ z{|Fz$Rh*k{-vzsaNW0jR(a%?g?(Q;SqrgfKWmkguS96Z8oH+c%28a$fJ1g|$ZduQA zW`QT^XeD&gP55N*x-x9t-vrODW2^;Z@BGHEqIlNSEb$bKuz6b8OveJ=)jKZnp$t3S zoispYkA;p1WFME8^Ar3c;eYM1co_M65Mt!?W^&f+@Kpoq{%vRB&akqz(?fmZXN%Ef zZmRw<@oDlTH5V$Eh?FWSE>1-&plrK*xZmLbbWu;jK6@;&j?9JDTMA*rx+$9Dj_&G? zyQuyU6g2Vps7SNleKFxeRdj!-T6ik?M>`+7jB=uT4+F2d2pdJ z$~KPH_xV`jYz>Bm>VMwrz!<<|WSBYeMD$RP8SYU05Y>k3ib(Z4WYS~eQxg|w1l%a} zK=Nw>Zpe8lq3YHQ0&=vV>POtUe~Er8v$uT4GzA<2cFX3-(=H?6ZVl?^ZwZ9}#uuFa0HY9g|z9ou#u72^g&Fl~|HS>1LkS|Dk=!_Td$94ot zqP-<{-?=IJrV^OZ0>xQ*+J&_hn2-42W~kgzlxPr$^m!UV$q?5Ncw1p!FLP!>fV2@8ayRh_3}SN@AbTL{5R3FZa}RS zzszcWPGEZ7=6*Gq!8h*jb!s(faw>;-`5~Q#pATEPJx$FEt|dVU0o`W!PbFMsRimun zn`Nkoh<_$7RW|fFTOM!sGF)!9qRRmr!y|Z61rcgDjOJ)zdE2^h&u+-w!rc7l_ph5Z zvI+`d##@NQ`Jvh-hDAqDuYTG9#z<~?Cdx|Kp00k}RzE<^Aok^~Tz!k^7KbuaaY6id zUuHM9AE8#6?hr(|7rt(6_YXZ`M)_h}9%~~KTukhyLnEzz3_&5WXprq8;R`Y2vAsM^ z*7r9H?S`L6DcVUP5cQos$fTP@qi2(gp_m0p><&Ki#;rOC?}JZ2b5k6-Xq-$B_vkP5 zzzJ(_;Y?*-T~;Uc6|K$#J$yf`=BY$`^-ey;yJil~!*;o1EPez2|Bdqw!7;mx7+nL; z*ALU-V(b*hvF0d}8E|50tQl;>EC@CiiGafQHd}_}k=CIaX&5m^r&~zJT)_g1lLyHX zQ)Dyh7#EbgI;G(-zrdWEIuUu08fw==isc_TS*6@y+&op^%@E#@Cr8qphJ%9$qf;XY zk+Auxd!p?EhUEbd=3cFB4k#0n2e3!+{QH2nzKYvXodSV8BnofTH|?GIDiSmO$B^rZ z_vbaore?_Anx$7*wiW01%`XJ)|5f^KeBht>$r#xJTK=dAkH&Y${hc}*x)$Z1;%?07 zg2hE7s#`{rwZq3Wry*Y|tmms($+j_XgnDHYo?T)E&N09-@+c3L&bRvFd7g}~Q73TR zqz9!is2lH9bKkZ(fl@99Jb*Jj)l{1HyFVX$x`%zmdcbDA6XJrZvNxD80uO5MR1v8RH#t=okWHcBrXw`9M0x(Rj~u8t96fD52Y} zG{h5Ysz-mdMm7B5Y*}RU5wS->*bhkBGd+l}YQF8#8ARyC;d^%rAfNMZ-Ad99;uAOPO_N=Z2n>>UEVgU;i~ zjTG}+$j}8&<+pa^$KDZcHt|i_3rM<8G(((vuea^lSJutsbKby0CNbPOtEwl)qIJR~ zdQyXDsnK$S1E;3wH}eG7s=mn5UW?pHn6oMr`4v1n`aJ3uigxtHH($t)JE5T@8aPKZ zbQ1B4@U-A7Y3~ZH|08nx4b2BIv=Dg2T~s~6R^56hFg&#d?(=zHpwo!8w9L9XZY7Ch4U*K*5%PljE<=k5G5JB zFVCJjHE}C$yZ<5+3560Rl`xHcUy`zAXYAP}Lv{us%kU#hCCQ9^jAY9&wvb&(YQk8@ zHntes$eOWpb$|c=z4v#Y=RVIp&wI|hocBHFJkNQ~=e*~{Lk%>U&v2gs0DxIrOWha% zXvimWAHzul7@5TKp0I0=w2buu;HdxrL_`C?-iaw<0RRHT0bt1i02I>zfYT?d-bm@B zgVs@3Qyn;tP3&wr(HMQSEd5USiPuS8b5|vt2&etE^)*f}GSF}WY?vs9O#nFeQ(Ild zG-&kKc(9Er`KWb0)~r22MXPQ_HzVx~L{5J|npFOcU$;%CEGGT|u}R}rbP#zsk8&6vA$GV@u}yN@Q;vZPqc z(v7qVFmDR)A^D9`)Gdpr@g{>|!m6gL=nbM}6^jMcC2XZ|!`P@P5O0Dy!wEAwvL@&6 zW$uD6;9BH%_IFxW2vfh=7T zZ=yz79Bu6*2Q{bs_afif9QYra4JU7tWBQOXf{}Se@^{7tYJI3xK~3Pv?=KG>H35M@ z=-|EGP3lUS43o=f1l65IHuK2%1=Hg_gQrf=vh^*NbEHLdhvs_yQ;n8_LCPHr@x=yO z2yt=CCIDI^cB4XNDf!W3=cAf($*@mykz`gR<9P&*wkdLacF5- zpRTrL{v=isYhT!<;qAw0uJlYG!4~~V&_~zMVWHS2|F#DvD=RBh1DRkxLS>_jN_mPZ z)dA>`1%0wT1v97GHh?YuozZq3&a1$W4=g+Y2tN#X#xs^3M{VQ>3oywc)A}%T>8k@y z%6n8+O0Be?H#c}_I55~QJOZCRWMCsj65Boc$;&!-rHf9-@9orsf<`-8XPD)Fy%^q` zMVhpmHxp%{TyKSh@mQa16dTOw;y)A$1YpYtW#}YnJq3n6(cOl(eVKX%{ zdkD>|Ra*JFhQ7AE6hx(p@r~B$LqL#iS_Q z^E0F=7R84Q$jZN43Olu%BDHF+V8s&|HYZ!*WA}^^ih`tmX{?=MXpQ7r{YATxa++6b zoVGRKGU7oWyw4}jqBgBU>Jgsk5v9f}tVFkVc+e{LJYpI3;uRONsxvjB1zRXrVp4dj z2!eWnw$@wwU;|oJZpTeY#_URpd^WmZTz9v<*Eg>1QOx&=-C&=s#K(*#JX^G;D17`Jr-PNZu`r*5cAI2Njlyh= zB$$A;hBtY6ydv9ynCsi+8wQJRJjv{6i>=1>cnTcky_y+RNcF!Rwb7E91Z!T+QcT#C zUw=Q^Lbj1)!}!n5Skb@cwHVk4bBIwR&xj4OuAZb)P7ux z6-PlGtJ%`rpcBP?#F3Of(dFWH$ohL@GA-sl(O^7l-JAa2`*tgb_bQeywL7KC=LTkq zbei+wEI}z{My0dZZs&wMrk};!1kX7_yKAlXH^6C26>9k}3{jB%u?`g5ROes!?}ghK zMS&e^RML|OPtuuAchOy5^wJCtZP`eWc5l%W#YWp)U}`j5ztQRWd8I<8$FVG=jiN$jS!NLJA+@%l5fnVj)%-I%T;ceTxpx%W`(*pCIRiksVelC z=c_eWmnN>6mcrAvmg6SImuuH;W^!sN!p^ZW z>>r4d+_E@YRC9c~9SCGa`6crCtnKxWO4OGDiFzeF!X(F*OA(&o3ZG4wiOqAhN%MwC zeiP_~gj)99ev1?EV}oh`je;k~Mx=argU+nemS`1DkRbseT28p#I0 z*mu6U8{w|w&m)?GSv4=hX3UBCag~cg^WY}%K}onlG-NAcfQ zs%x?iQO6za>hEFYfh)x~jAX6Igp35c0SUl9HQc&J#(u#Y!9CXDvJJn=?XWGPe0bx2 zJxT=1yY?{kSsQPEd}S|0PGrXE7u#fE&KQVchqK1Msu!0Q|LIoE?!wjaH}8Ir?xptD zPTs=b$0}X=9T2{be$lPS?zra=@(PW9s5AAoxizilO1@zcd1d9ONAk}Cj(o;{%1t`_ z-s*E$WbhzVRWa!?k|yXbl?!MtCMxF*y7g~Du#LG5Z{In^wn*I7Mq+L1Rt$w#V!Xwb zHf1$%STBf+JDbAD8s5yzbik3xmBj^u%{s@%<~YyI#`Y$>&N4Y^yptFQHxt@kAQc}n z((+A*5{R$Zb$y$^(6BNvklj08=?~?s&bNM|@??~J#JVrbCIb^50T&L_V*PV%+VO-1 zKd)wK(Zd;)Fk(W(NIr><+)+q_!M@0~8N_cp&yJZi=g}|SclFLGBb*hvA)O!cuJLU^W>Z|Hwh+bc{RLBSSt|CRF|_P{ zbJOO=jfR;@$i@_kJ)}&EblG=0Z~Kl#cuk^3}L0@ja(ik^RR#G34 zM$zct%p9GJWF^Tu3Bq_*bIS?At?ln2SKa@rWW0&*J2XUU)IAMw6l-0#1twQz`ve<> zMLDnWiC+H5mdfbV4tu_laM0x!Nrce*EW+~eLjyDs{Y!if1sQb0rt{&)af@YxkICb1 zx3kQ*QdF|@TYQE3mo;)U@5nirQYLu_w$?6h7KJ!FPEZ~EpQsMn$o-4mq?XoaTmblN zrGJIS*UhlH=72h%{3*v)c^+$g4N;9)u#DKDr&KZH#j0cUHWgCpdYstqnS)vl@{uQl z-L6fWay?JKDT1MX2*NB=^<$4`;%msKPqP4t-+P&L(_wy1RzeMprt`d69;Dv>=?biq zbU?;V9+@T0vA0auJ8GkFiOt=UX~}!4WKXPhRs!I+Dy>=+;w}FOG<#)L#YLCLz`Sd_ zt{0+I6!N@Wle$2q?aCk)+{9+3?jhflWG+%ypLVy===@nO!#B@0Rz0E3Ik~~E9NEd1 zo+-X2Tnk2YM=5^awXRHk#*aSxgYiGo%}HcAiAHYP&-) zzTciSBb_0qNaMnc_YbmN{g@SJ-q?rAD6pC8|2FzQlS%4FnU;<)k!Bi1megNB?0ps< z;#<;^v0#a|CF`1?zg;qEd9qD?$92Tv)1*35*x?(H?YH!@KXVG#J5Wh{d}OJHbQ$eP z{ki%%H00V{T+tEy+P9p}xL$PB^hfdTC++hWhr}^ca z?itzU>)Vzc_u@8G4T%Zo_7Ag>&!MUnHQ|PQdbWz7cB4JZ49Jm@R|R`QoJ0)WUik5+ zMP~5hI~hLM&2v_7vpl`_LE?i%NXyr*;kU%9RO-rHc|vbO)C7(l|AgW@ttN8;f4BJS z(N>W zaAHrLKMPC^-%Q;7bOC>=1T;(utjG|G4P!cY7kBLL3cV!l^z+le`sHsS^WU{ykLH;Z zg9v zD~W1WBj^9Y`UV8XPZXak{J)7+FIXc`O>-AE(U8;aL55$nj2{9V@W#j=W1V;}_Jh@rcH8LN; zRXC5=6)Lw?jL-1jrZKQ1H9(|X4q14dG|LDxu?BusjW+_zl$ox?y*z!U#zRNi3I;y% zM0Y)HYZivl9wt=o0;_i8w|JDk4N2@-V3nHM9#HFPgX&&#?`f@mFQwb5sNPOX;k@f9 zIh!~1j?cbgH;K#EMQKhW`$y|K`pB-%YIcniPeCrB%}kIj0v*DXWSX@E_B%D0rBS5i zDos$ZkwY`;v0RMEGX#Cl{^dryh=z)VYvj;3CJ8Bi)9y9cG-@Y@Zz*(X4x_??ys2v3%tOlEB4oYh@dP4HfcD0s>)+gjk(@0ft4 zn!*wHUi!;JZLGZEKC=ZI&Zx6=1$RDhhyKDxeC7Z=P;BJ=5wA@PH^Q7P_N|wB(m5{$ zxN84O{=v@$`7hHIa99@Rw8e2;iuj-rVoWeVW(1$f@Pme~#DxTHcH_cVTUmD`(5EH= zz88Vn$8_qu%@C}S#?Md;A#oK41qdE3hy}(jEH45b& z#Vg!SUvZ9P_ipq>dYPvvy|L>;VYY6DB5~EB54Bq^<0o$kMjSz?ET|tKISgl?W74TkRhAe*ZpU+Xmm%nxddhQ&Y*nfBek5;K1V za)Ft`@Q+j^qP^vol_`M?4kP<9U4hK|f%Nf}H}P-4Cr`=v%D-Dv{e9kak+&3)m^JB> z4-kHTjR*ct5B*_^kKnKq21tra%7}=|i-=2@N+>8w%PLAM2#Jd;ii?|@G@Je-z|-5w z)j9b84iIDko&3xI;QyMz-_;Z5=l{^t=YPgXN}Y_YC?Ws%B(MO-$&(mBTf;ydr|R(6 Fe*oY$?TP>Z literal 0 HcmV?d00001 diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index 5854c4cb22ff4..e05c2a07821b0 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -188,7 +188,7 @@ g() # recompile g ``` After compiling the two versions of `g()`, the global cache looks like this: -![Global cache state after invalidation](./img/invalidation-example.svg) +![Global cache state after invalidation](./img/invalidation-example.png) The maximum world age, `jl_world_counter`, is protected by the `world_counter_lock`. Julia uses a form of optimistic concurrency control to @@ -219,7 +219,7 @@ Type inference proceeds like so: on the backedges for invalidation. - Release `world_counter_lock`. -![Two threads doing type inference while another adds a method](./img/typeinf-promotion.svg) +![Two threads doing type inference while another adds a method](./img/typeinf-promotion.png) In the above diagram, threads 1 and 2 are doing type inference (the dotted line), while thread 3 is activating a new method. The solid boxes represent From 980881601e488d045acce570f67f84464eaa80d7 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 2 Aug 2025 07:53:31 -0400 Subject: [PATCH 619/662] Precompile out of env deps in serial within precompilepkgs (#59122) --- base/loading.jl | 3 +- base/precompilation.jl | 53 ++++++++++++++++++++++++++------ doc/src/manual/code-loading.md | 3 ++ test/embedding/embedding-test.jl | 2 +- test/loading.jl | 2 +- test/precompile.jl | 30 +++++++++--------- 6 files changed, 64 insertions(+), 29 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 7d2f33ede633e..33799805e0601 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2625,8 +2625,7 @@ function __require_prelocked(pkg::PkgId, env) if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) project = active_project() - if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing && - isfile(project) && project_file_manifest_path(project) !== nothing + if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) parallel_precompile_attempted = true unlock(require_lock) try diff --git a/base/precompilation.jl b/base/precompilation.jl index e737faa210c79..1b609f3e2015a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -26,9 +26,24 @@ struct ExplicitEnv #local_prefs::Union{Nothing, Dict{String, Any}} end -function ExplicitEnv(envpath::String=Base.active_project()) +ExplicitEnv() = ExplicitEnv(Base.active_project()) +function ExplicitEnv(::Nothing, envpath::String="") + ExplicitEnv(envpath, + Dict{String, UUID}(), # project_deps + Dict{String, UUID}(), # project_weakdeps + Dict{String, UUID}(), # project_extras + Dict{String, Vector{UUID}}(), # project_extensions + Dict{UUID, Vector{UUID}}(), # deps + Dict{UUID, Vector{UUID}}(), # weakdeps + Dict{UUID, Dict{String, Vector{UUID}}}(), # extensions + Dict{UUID, String}(), # names + Dict{UUID, Union{SHA1, String, Nothing, Missing}}()) +end +function ExplicitEnv(envpath::String) + # Handle missing project file by creating an empty environment if !isfile(envpath) - error("expected a project file at $(repr(envpath))") + envpath = abspath(envpath) + return ExplicitEnv(nothing, envpath) end envpath = abspath(envpath) project_d = parsed_toml(envpath) @@ -468,6 +483,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; fancyprint::Bool = can_fancyprint(io) && !timing, manifest::Bool=false, ignore_loaded::Bool=true) + @debug "precompilepkgs called with" pkgs internal_call strict warn_loaded timing _from_loading configs fancyprint manifest ignore_loaded # monomorphize this to avoid latency problems _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, configs isa Vector{Config} ? configs : [configs], @@ -518,9 +534,12 @@ function _precompilepkgs(pkgs::Vector{String}, # inverse map of `parent_to_ext` above (ext → parent) ext_to_parent = Dict{Base.PkgId, Base.PkgId}() - function describe_pkg(pkg::PkgId, is_project_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) + function describe_pkg(pkg::PkgId, is_project_dep::Bool, is_serial_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) name = full_name(ext_to_parent, pkg) name = is_project_dep ? name : color_string(name, :light_black) + if is_serial_dep + name *= color_string(" (serial)", :light_black) + end if nconfigs > 1 && !isempty(flags) config_str = join(flags, " ") name *= color_string(" `$config_str`", :light_black) @@ -630,15 +649,28 @@ function _precompilepkgs(pkgs::Vector{String}, end @debug "precompile: extensions collected" + serial_deps = Base.PkgId[] # packages that are being precompiled in serial + + if _from_loading + # if called from loading precompilation it may be a package from another environment stack + # where we don't have access to the dep graph, so just add as a single package and do serial + # precompilation of its deps within the job. + for pkg in requested_pkgs # In case loading asks for multiple packages + pkgid = Base.identify_package(pkg) + pkgid === nothing && continue + if !haskey(direct_deps, pkgid) + @debug "precompile: package `$(pkgid)` is outside of the environment, so adding as single package serial job" + direct_deps[pkgid] = Base.PkgId[] # no deps, do them in serial in the job + push!(project_deps, pkgid) # add to project_deps so it doesn't show up in gray + push!(serial_deps, pkgid) + end + end + end + # return early if no deps if isempty(direct_deps) if isempty(pkgs) return - elseif _from_loading - # if called from loading precompilation it may be a package from another environment stack so - # don't error and allow serial precompilation to try - # TODO: actually handle packages from other envs in the stack - return else error("No direct dependencies outside of the sysimage found matching $(pkgs)") end @@ -846,7 +878,7 @@ function _precompilepkgs(pkgs::Vector{String}, dep, config = pkg_config loaded = warn_loaded && haskey(Base.loaded_modules, dep) flags, cacheflags = config - name = describe_pkg(dep, dep in project_deps, flags, cacheflags) + name = describe_pkg(dep, dep in project_deps, dep in serial_deps, flags, cacheflags) line = if pkg_config in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) elseif haskey(failed_deps, pkg_config) @@ -929,12 +961,13 @@ function _precompilepkgs(pkgs::Vector{String}, if !circular && is_stale Base.acquire(parallel_limiter) is_project_dep = pkg in project_deps + is_serial_dep = pkg in serial_deps # std monitoring std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) - name = describe_pkg(pkg, is_project_dep, flags, cacheflags) + name = describe_pkg(pkg, is_project_dep, is_serial_dep, flags, cacheflags) @lock print_lock begin if !fancyprint && isempty(pkg_queue) printpkgstyle(io, :Precompiling, something(target[], "packages...")) diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 567b4699c3cf6..301f5e5def618 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -42,6 +42,9 @@ These environments each serve a different purpose: * Package directories provide **convenience** when a full carefully-tracked project environment is unnecessary. They are useful when you want to put a set of packages somewhere and be able to directly use them, without needing to create a project environment for them. * Stacked environments allow for **adding** tools to the primary environment. You can push an environment of development tools onto the end of the stack to make them available from the REPL and scripts, but not from inside packages. +!!! note + When loading a package from another environment in the stack other than the active environment the package is loaded in the context of the active environment. This means that the package will be loaded as if it were imported in the active environment, which may affect how its dependencies versions are resolved. When such a package is precompiling it will be marked as a `(serial)` precompile job, which means that its dependencies will be precompiled in series within the same job, which will likely be slower. + At a high-level, each environment conceptually defines three maps: roots, graph and paths. When resolving the meaning of `import X`, the roots and graph maps are used to determine the identity of `X`, while the paths map is used to locate the source code of `X`. The specific roles of the three maps are: - **roots:** `name::Symbol` ⟶ `uuid::UUID` diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 34ef9a796ba56..0744cac679698 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -32,5 +32,5 @@ end @test lines[9] == "called bar" @test lines[10] == "calling new bar" @test lines[11] == " From worker 2:\tTaking over the world..." - @test readline(err) == "exception caught from C" + @test "exception caught from C" in readlines(err) end diff --git a/test/loading.jl b/test/loading.jl index e95138e27f4dc..3a085277d96b8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1495,7 +1495,7 @@ end # helper function to load a package and return the output function load_package(name, args=``) - code = "using $name" + code = "Base.disable_parallel_precompile = true; using $name" cmd = addenv(`$(Base.julia_cmd()) -e $code $args`, "JULIA_LOAD_PATH" => dir, "JULIA_DEPOT_PATH" => depot_path, diff --git a/test/precompile.jl b/test/precompile.jl index d1ad0c459932d..8e091692b68fd 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -686,16 +686,15 @@ precompile_test_harness(false) do dir error("break me") end """) - @test_warn r"LoadError: break me\nStacktrace:\n[ ]*\[1\] [\e01m\[]*error" try - Base.require(Main, :FooBar2) - error("the \"break me\" test failed") - catch exc - isa(exc, ErrorException) || rethrow() - # The LoadError shouldn't be surfaced but is printed to stderr, hence the `@test_warn` capture tests - occursin("LoadError: break me", exc.msg) && rethrow() - # The actual error that is thrown - occursin("Failed to precompile FooBar2", exc.msg) || rethrow() - end + try + Base.require(Main, :FooBar2) + error("the \"break me\" test failed") + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Failed to precompile FooBar2", exc.msg) || rethrow() + # The LoadError is printed to stderr in the precompilepkgs worker and captured in the PkgPrecompileError msg + occursin("LoadError: break me", exc.msg) || rethrow() + end # Test that trying to eval into closed modules during precompilation is an error FooBar3_file = joinpath(dir, "FooBar3.jl") @@ -707,11 +706,12 @@ precompile_test_harness(false) do dir $code end """) - @test_warn "Evaluation into the closed module `Base` breaks incremental compilation" try - Base.require(Main, :FooBar3) - catch exc - isa(exc, ErrorException) || rethrow() - end + try + Base.require(Main, :FooBar3) + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Evaluation into the closed module `Base` breaks incremental compilation", exc.msg) || rethrow() + end end # Test transitive dependency for #21266 From 94cc9968af3046331da25cf6eac50c524d132f88 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 2 Aug 2025 22:01:42 +0900 Subject: [PATCH 620/662] inference: propagate more `LimitedAccuracy` information (#59187) According to Jameson's comment, we should propagate `LimitedAccuracy` information not only when we actually use the type information of variables that are `LimitedAccuracy`, but also when we perform computations involving variables that are `LimitedAccuracy`. This commit implements that propagation in relation to JuliaLang/julia#59182. --- Compiler/src/typelattice.jl | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Compiler/src/typelattice.jl b/Compiler/src/typelattice.jl index aae0241d69aa1..f4c3b051d3e3f 100644 --- a/Compiler/src/typelattice.jl +++ b/Compiler/src/typelattice.jl @@ -8,6 +8,19 @@ # inside the global code cache. import Core: Const, InterConditional, PartialStruct +function may_form_limited_typ(@nospecialize(aty), @nospecialize(bty), @nospecialize(xty)) + if aty isa LimitedAccuracy + if bty isa LimitedAccuracy + return LimitedAccuracy(xty, union!(copy(aty.causes), bty.causes)) + else + return LimitedAccuracy(xty, copy(aty.causes)) + end + elseif bty isa LimitedAccuracy + return LimitedAccuracy(xty, copy(bty.causes)) + end + return nothing +end + """ cnd::Conditional @@ -40,9 +53,8 @@ struct Conditional isdefined::Bool=false) assert_nested_slotwrapper(thentype) assert_nested_slotwrapper(elsetype) - if thentype isa LimitedAccuracy || elsetype isa LimitedAccuracy - return Bool - end + limited = may_form_limited_typ(thentype, elsetype, Bool) + limited !== nothing && return limited return new(slot, thentype, elsetype, isdefined) end end @@ -86,9 +98,8 @@ struct MustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" - if vartyp isa LimitedAccuracy || fldtyp isa LimitedAccuracy - return fldtyp - end + limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) + limited !== nothing && return limited return new(slot, vartyp, fldidx, fldtyp) end end @@ -110,9 +121,8 @@ struct InterMustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" - if vartyp isa LimitedAccuracy || fldtyp isa LimitedAccuracy - return fldtyp - end + limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) + limited !== nothing && return limited return new(slot, vartyp, fldidx, fldtyp) end end From 13f6b3b5c0361c865ba6f3e6f09fb7d3ae6a6735 Mon Sep 17 00:00:00 2001 From: Thomas van Doornmalen <60822431+Taaitaaiger@users.noreply.github.com> Date: Sat, 2 Aug 2025 23:26:28 +0200 Subject: [PATCH 621/662] Export jl_declare_constant_val (#59192) Closes #59184 --- src/jl_exported_funcs.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 34978e97abd54..64ba74722bdc9 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -178,6 +178,7 @@ XX(jl_gdblookup) \ XX(jl_generating_output) \ XX(jl_declare_const_gf) \ + XX(jl_declare_constant_val) \ XX(jl_gensym) \ XX(jl_getaffinity) \ XX(jl_getallocationgranularity) \ From 905a847d875bcbcfb88b82206bb6543d2a3444e7 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 3 Aug 2025 20:49:25 -0400 Subject: [PATCH 622/662] Align build flags between source builds and jlls (#59195) Should fix #59171 --- AGENTS.md | 7 +++ deps/checksums/curl | 72 +++++++++++++-------------- deps/checksums/unwind | 56 ++++++++++----------- deps/curl.mk | 2 +- deps/unwind.mk | 2 +- stdlib/LibCURL_jll/Project.toml | 3 +- stdlib/LibCURL_jll/src/LibCURL_jll.jl | 9 ++-- stdlib/LibUnwind_jll/Project.toml | 2 +- test/stdlib_dependencies.jl | 8 +++ 9 files changed, 89 insertions(+), 72 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fc282bfdfbf3c..4dcaa2069fa1f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,13 @@ When modifying external dependencies (patches in `deps/patches/` or version upda - Prefer using the full upstream commit in `git am` format (e.g., `git format-patch`) which includes proper commit metadata 3. When updating dependency versions, ensure all associated patches still apply +### External JLLs + +To update a JLL to the latest version: +- Update the version number in the appropriate jll folder +- If the dependencies in the upstream jll changed, update the Project.toml +- Run `make -f contrib/refresh_checksums.mk ` to update the checksums. This may take a few minutes. + ### Writing code After writing code, look up the docstring for each function you used. If there are recommendations or additional considerations that apply to these functions, diff --git a/deps/checksums/curl b/deps/checksums/curl index 3e947ff9d8070..c7f63bfb9dc2d 100644 --- a/deps/checksums/curl +++ b/deps/checksums/curl @@ -1,38 +1,38 @@ -LibCURL.v8.15.0+0.aarch64-apple-darwin.tar.gz/md5/47db9bf4549442b466ae465b48544f46 -LibCURL.v8.15.0+0.aarch64-apple-darwin.tar.gz/sha512/a4819a6a94958cfeb0b035b87cfe6cb40ac3c0f015b7b52b5314ec3f8a129a9aa96ccae04fe3774318bc1a9316299c3fbe1a47b2e2e5ce37a38d9a8e01d6f5e5 -LibCURL.v8.15.0+0.aarch64-linux-gnu.tar.gz/md5/7e04291557b99a53f8b8fca1235862b3 -LibCURL.v8.15.0+0.aarch64-linux-gnu.tar.gz/sha512/399b9a538dc54647b5ca1ad182e5c1c4da8ba0f3217fe8d01737bf0a689a67ed0be185ebbd98d60b2739900df664feae6cedb90643c2e53dc7c12ec5a5a8b57d -LibCURL.v8.15.0+0.aarch64-linux-musl.tar.gz/md5/63f3e0171c8f56592a3c810136154ccd -LibCURL.v8.15.0+0.aarch64-linux-musl.tar.gz/sha512/03b67d73bd41597c6080155e5a0eb58d8cb559d1325c2954aa92c54905dd6f425ba46dd16c5411b16ac683c041ed685893954c4ab431d794bd3f929c11de4ae7 -LibCURL.v8.15.0+0.aarch64-unknown-freebsd.tar.gz/md5/31f249d293bbea3a0669197e6559d467 -LibCURL.v8.15.0+0.aarch64-unknown-freebsd.tar.gz/sha512/bb047b609e69926d3837dfe39d68057c55c7380427a3807ccc71e631530d75f64e9edd0a4ef5ab30b44e3d763d2798e9f8c4727fb565674de228cf95ebccd922 -LibCURL.v8.15.0+0.armv6l-linux-gnueabihf.tar.gz/md5/462094d229be838090eb6d3353bb3907 -LibCURL.v8.15.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/3bac7d0b0a065e5793d466823eb2a62d8256d41291c9bf6926bafc434eaec51a72ca289b2803b0b6fc19c4d0e6776205e55b235701496e84d56abdd4da7be8a2 -LibCURL.v8.15.0+0.armv6l-linux-musleabihf.tar.gz/md5/3f95a180a1fe427b642f05814029f9d0 -LibCURL.v8.15.0+0.armv6l-linux-musleabihf.tar.gz/sha512/8a4483486dd355d61f0c6f60d0a1b38d9942dc126b36368adcb9499daf2a134087fb54ffbd2930adf3fe1cc39497f2f7e1f6304f48f7540cf05ebeb792d0ec1f -LibCURL.v8.15.0+0.armv7l-linux-gnueabihf.tar.gz/md5/d033e4eba1455ef17b5d86d9911d2d32 -LibCURL.v8.15.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/91e31ec413ee15574ee7e4258eebd6963d6679ea8b41c43e0f02f18521d52f5a9f258c46f8df27163019dd89a2598b7c3bcd3d2aa2bd3fd3a545de9060ef7dd1 -LibCURL.v8.15.0+0.armv7l-linux-musleabihf.tar.gz/md5/3ba22fe71ba61b92e297f64593d69387 -LibCURL.v8.15.0+0.armv7l-linux-musleabihf.tar.gz/sha512/a87bc28cac150d76c281b1baa6f79da8140bf849595dcca684fcd64299fc801c3c2dcd733b264c5a809fbba95dbe8dfac0c30ecd7c5d0bb77fee7d3da91d6816 -LibCURL.v8.15.0+0.i686-linux-gnu.tar.gz/md5/67612302faac6b9822271d01c1959d78 -LibCURL.v8.15.0+0.i686-linux-gnu.tar.gz/sha512/0f47c1f8307054eca308650f13dfee62a6eaae70f5385f460339a821cb73f97eb56ac5f685cb9e14d51e109a510508e6e0e7b6f31319e57413396473ac19a1c6 -LibCURL.v8.15.0+0.i686-linux-musl.tar.gz/md5/b14f7012d2aa9ed353387c3b357255c6 -LibCURL.v8.15.0+0.i686-linux-musl.tar.gz/sha512/177b4581c4717854420073731398dc8bb2a3370a572fd6b61e545a8e852dcb9ce76f4c26bf28e1250cece1dcbe34e53d0edd2c421c6251c117e2f3255b8945a6 -LibCURL.v8.15.0+0.i686-w64-mingw32.tar.gz/md5/2afb5832ab12120e755505916cad0e95 -LibCURL.v8.15.0+0.i686-w64-mingw32.tar.gz/sha512/4e78b5d0a8ec3e1bc7131c66fc5f041090f5fb64fe6b85224fea1e3ebcbdbf56604492a059ae311281052da7ff67c066910c6c74fb7d19b3e46b4d3cdb7163b7 -LibCURL.v8.15.0+0.powerpc64le-linux-gnu.tar.gz/md5/9d5bd94798a63a14ba945949b054195f -LibCURL.v8.15.0+0.powerpc64le-linux-gnu.tar.gz/sha512/eb4c6a555b1b6dc39a890e819803d4d5015e9ad8a841e21b484fa7442330ea0b3c2da07fcc1afb09faf115fc4818eaf2f328f92fc07400a523aca45828b71e4b -LibCURL.v8.15.0+0.riscv64-linux-gnu.tar.gz/md5/68b576c1f2086490a3d7997ce8627a32 -LibCURL.v8.15.0+0.riscv64-linux-gnu.tar.gz/sha512/e7dc57b985a6f6289d7a56a1cae49522cc623831167ec317bcdb66b578f1d967befb5d849d2c24dc49e183524a6f3b72b6c688e3500c5cb8a3f00ce748728fc6 -LibCURL.v8.15.0+0.x86_64-apple-darwin.tar.gz/md5/75320e06cd2370343684279f744b694a -LibCURL.v8.15.0+0.x86_64-apple-darwin.tar.gz/sha512/198f520a860337aaf367db970083db8c7a3742b3f37f5594e4dc86669b319cc4dc4bdfc9e875c3e4ca9f93aae24705b7c47711bac66daa629ed0d386d352b053 -LibCURL.v8.15.0+0.x86_64-linux-gnu.tar.gz/md5/3f159aada40e7f4489b179a570d36c9a -LibCURL.v8.15.0+0.x86_64-linux-gnu.tar.gz/sha512/42e646400aca6d0b233ab8feb3b32c3a9e89ca4b27fb0c82e0c52b4b008e9d9aad6640988f32397d705cc52c36ffa4379f6ba4c015ee24b6e2bd750235c8f10c -LibCURL.v8.15.0+0.x86_64-linux-musl.tar.gz/md5/d4e35eee8af7d9e640f9f2616abf1ebf -LibCURL.v8.15.0+0.x86_64-linux-musl.tar.gz/sha512/16a9ee5afe025db931e7209fbc711494ff7795a9463cf81a307e7ea74cf53b64a1c5413d388e328a372e4480255b4b1324448e17f51db141adcb2716aa00d698 -LibCURL.v8.15.0+0.x86_64-unknown-freebsd.tar.gz/md5/94e871b6200ab1768112aa3a7195f31e -LibCURL.v8.15.0+0.x86_64-unknown-freebsd.tar.gz/sha512/84474b36d21ab1d90331850b914908add5afd239f9111a48dec9b6e7ff770e9a730c7cad6d322f0cdd3567822fb157e1470716e181dd29b594a4e471b8fd6c54 -LibCURL.v8.15.0+0.x86_64-w64-mingw32.tar.gz/md5/99c54cbc43fc719281b48bb3137e9364 -LibCURL.v8.15.0+0.x86_64-w64-mingw32.tar.gz/sha512/d817951ec0ba4513e50c2bf2159ccefb6b38fa74478b84b8f006b6a47897c610fe6c9f1c9db0533595c243f8b84d40456aa9fd1aa3afb22ea272dced5b6e237c +LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/md5/6e0789ffb84bb7baae49814c36db354f +LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/sha512/9d64ef5d61db74f3cc67cdcaf9b41d3e7e6a0a3b51deea97dbb3da1928adc9d8aa9558de786af1497de42df43dd184af057465f57471fd01cc1a83020dddbd6f +LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/md5/54214152cfef71c9296a150594c8c405 +LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/sha512/9a1c96e943313af3151763104bb2bed465722d81a4a5c97f8eefbcb4c566718e2a426bb9e81fafb17129a9cc5de93fd8cba5f83f7314dfa4aca6ad817011b627 +LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/md5/be2288fc32572680434756ac1e9774e3 +LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/sha512/d15a5a7e0f08fe9a0ad1b95f86c53167e3f26356be8010fa332a1a7003d879b5506a8029b7ab36e221821033012cd3a5581c30023fc950c711b517b49d891a3f +LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/md5/a18e1e712923540b1e8ccf8c67e0e735 +LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/sha512/cb0b2f4041abdcd3b8f15f8fa1a894f8c4aacaaea9f5b4d5213cee53bee2dfe8884299df2cc1f51c31f058b19c57c6b9e3c3cfd63f84ce2db27af14ebc4c0c5f +LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/md5/389f85ab11b49689a941de10853072a3 +LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/2e1f70f804b540159baf720953716badea9854e7e92783b2f3a8805057c430b2701cd662c24f0cf4a8d11e79cd80a352fd59f23cbf705dd09ffd9ab7dcabef8e +LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/md5/4b071a0e0af2a64ad1cde498f54a7177 +LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/sha512/90eaf004296fd1ebb3dd02120e92ef574a8c37456c5c99477764e91cdcf0326d92de852d0c8261693403958a3d0e40c4b6b3cffda79f517ecb4f1f852a812c02 +LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/md5/b3da6e896d2fc03ff384abda6f27baf1 +LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/21c7bb94b94bc1840c5bcda0cd3fd01e4850e3d9e43804c7f2ba49f2f97793603191476c2f764bd5e23f4288c1b04d1645a1c55655cc09971ec15925aef7fd77 +LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/md5/37d6ff69f83338c222d65f5132025112 +LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/sha512/f483d2fce926272fa7cabbcec8a269f681150b4bdb1cd25e638d9cff358e770a5014ada6394a2b42a1fabe6f6830d131924442003eeb3841dafbc8f2b9288112 +LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/md5/24bf5ca0680cb664b2a9d11e5356c4b7 +LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/sha512/21b000f346458079d7091f3766a2ee06dda90b1c4e48d236e768181013088b9ee8daf51eaf966c6c88cf3d9380b204ea7f9f827336de841f4b5efb38644ea5c6 +LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/md5/c158455a2fb3e75d8ddc08e253bafb82 +LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/sha512/8b59acb7bc33cc04fa39107d85c5ef3296a901d138bf4951001b7cd7d5780d84e6728b68862173501f9f4720c30736611f24c6334a01d271649ab1a03ca7a05c +LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/md5/e681eea3a492df7cfb0c0e8d1152a56c +LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/sha512/41b9c183099b00eadefd7f121d17611972f201cf21ecdebec538c5ee9620646bd8fb7610888df4b9d1a1ab3055d9edb6d345e8f787d9df4c1b739cf72312d5dc +LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/md5/9c71b40f94b7ffd65c182dd96da76d04 +LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/sha512/763cf590b7fc8cc509b4762f8c3fcd1f0534f756407472bc86063b70a8a144fd651990d95f9300bc7eb30682875abafab399a19449af6a8a4d9d9fb548b2a5d8 +LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/md5/d84d6ecb606d7de45a7e0e91d29f8e06 +LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/sha512/5c786779c838797182824849f682d54cf57f248ea7b7f80c21d34540324df16996796647f82e3b3d1cb274278be6c935c080658edb780fd76dbfe1d04928c74b +LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/md5/2738d7814b7e75a292301c5bce181a0e +LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/sha512/7ffe3b616f6a6591608dcd3a17cf67420b830b17e4d5cd9bd0441d104910d1a5ce28484238fa7c32dee0306c8f2ac9ffaf673a28ceca646e6879e5a552e3065d +LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/md5/a945b3ed85f8fb6ec27c551fbfe2946b +LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/sha512/1646773b43433ea9f403c0242fe0efb0595f16507554cfe326d775ef3f0bcd400a04a7a7db5b184649f84531d47938ec03d79d335c53929647a50cef9011fbcc +LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/md5/6aa66d59e057fdb7da74a359648fa41d +LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/sha512/1a422ca310ffea3eb70d27472440631031a2f7c8f089e261cecbb3bcb1bc7161fa41fcad91d3eb01cd688af30c8b43ee75b565f09bdff8c2fb0ab64fab571eb9 +LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/md5/d95c02c6c541689250e0015eec5a8577 +LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/sha512/0b6e0790779370f721ba251f6cd5f3464b62361fdc23bc1ca9ae22c82c4efbf2c7fc8409277b90c1b2e73996eb4378c188924b083d4b9ba255eaaeeff339e84d +LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/md5/690e63d716b19e109b5487dfb4ef580d +LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/sha512/2e763b8acbfc622dac431fa742c9cfdb12cb95436b0eba2e409a7a1dede107e99952f394c92d8f3aa55f3febf039f5c16058ee122d88e0eac5a75ae98f2e9018 curl-8.15.0.tar.bz2/md5/8b475c3eec74c5e78a9fb45a902dae76 curl-8.15.0.tar.bz2/sha512/797fc9af599de88ceb36c8bc284d3f1a2a1ca0703c7bdd377d67ce6da4317ca673ba4a946c61f3bd5a66febee37b5aa88826f26fbbb398f5e20630769a0de033 diff --git a/deps/checksums/unwind b/deps/checksums/unwind index 65cd2a3ab0897..c7dbc28dd3976 100644 --- a/deps/checksums/unwind +++ b/deps/checksums/unwind @@ -1,30 +1,30 @@ -LibUnwind.v1.8.2+0.aarch64-linux-gnu.tar.gz/md5/f23ea74855dd2db466170f170a8eea7a -LibUnwind.v1.8.2+0.aarch64-linux-gnu.tar.gz/sha512/27b4c37d225c1632e1d8814989d768b0fda2675552c611cb9fad1eea158e4b3815e0a8127c2eea97bcbfd23685d970cf1c478d24f6b5a8abe8357e12d8762ce7 -LibUnwind.v1.8.2+0.aarch64-linux-musl.tar.gz/md5/5073e00eeacd4f5921164090966f5ae8 -LibUnwind.v1.8.2+0.aarch64-linux-musl.tar.gz/sha512/0499cdd22510745e3e1f7da5c4c937a79b2eb07e2bf97afa5db9ce7e91aa2bab85bef2dc3ad8b859ac147ca8da908047bf12daf6e3d3fcb2c7a865402dd90beb -LibUnwind.v1.8.2+0.aarch64-unknown-freebsd.tar.gz/md5/b4aa1f850db793ffe9c1e0bf0524da65 -LibUnwind.v1.8.2+0.aarch64-unknown-freebsd.tar.gz/sha512/92bbb42a742427d4a4bfb67530191093ee55a210a99abe0145bdf4ccdc3e583a9852a8045dfdd20002a8fd8eaf9feb6245f84a4995fee4a097a33a2b64998dcd -LibUnwind.v1.8.2+0.armv6l-linux-gnueabihf.tar.gz/md5/a6eff40a5bef97c13fed1546dd77a503 -LibUnwind.v1.8.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/b0e687036c080cf5a421a924e070e57d9f52cee1e67e6c9558d586edc58296d15c9e10e2e46c27e6d290c866b84edeee11023077fb2481dfc9f08fd08540326a -LibUnwind.v1.8.2+0.armv6l-linux-musleabihf.tar.gz/md5/19f8f17923dc1b87b3e89480b1eeaedb -LibUnwind.v1.8.2+0.armv6l-linux-musleabihf.tar.gz/sha512/23251a9fe459242589797a364ff69e38711ebf93d4f047acc7d302c8bc669d3b0546a318568787f0f9bf4d1e633670e0941887cbac1c34e34d86e217040533af -LibUnwind.v1.8.2+0.armv7l-linux-gnueabihf.tar.gz/md5/74bd5a25811e4977386ba3aefe98bd8c -LibUnwind.v1.8.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/2d41f653640ffaebb7295d49a533efebf2ffca7314a181e6e669c399e65b49d176915afeb60b4702632d65aa20aad772052373efbd3c6b9fdc0c9fd772986b2a -LibUnwind.v1.8.2+0.armv7l-linux-musleabihf.tar.gz/md5/7d882db1897c88c7e2eb7cf94e3b3f9e -LibUnwind.v1.8.2+0.armv7l-linux-musleabihf.tar.gz/sha512/dad5bdbccd630eb27af69a32f01245865fc9cf494f1e2ab4cf9e06d78985d997accbac7db6c947bb1964f0aa98f7c977cdb9288e4453fa22d47eb99d73144536 -LibUnwind.v1.8.2+0.i686-linux-gnu.tar.gz/md5/f75ff3a516d6849ed583b66a25ae3784 -LibUnwind.v1.8.2+0.i686-linux-gnu.tar.gz/sha512/8a9ab5729cfb52d3f8e0a1d7f6ef40566fd03fd1aa7f94269a3f6acd7c3cc969c008be97a91d7fa982ff5deb2db64d39bac304a948bc3bd07795a5811a41d0b1 -LibUnwind.v1.8.2+0.i686-linux-musl.tar.gz/md5/e9e479c47ea9f1ec297d040ef058dfc4 -LibUnwind.v1.8.2+0.i686-linux-musl.tar.gz/sha512/ac945a97288179d34f96b228ca631b78218b83e69320fd7c4516a5770b32078b184798e070ab87dabe89370fd7bd1657c43a20c543bc73a146e8003806f65852 -LibUnwind.v1.8.2+0.powerpc64le-linux-gnu.tar.gz/md5/268f72d03fd4cad65fe9913074d05226 -LibUnwind.v1.8.2+0.powerpc64le-linux-gnu.tar.gz/sha512/533d701bc78adc725626822fbe75002560198381a695373bd61805e6773de2a6ec4314f56d3d414cb9c1564da1ccb519d50eb3fa3aa99a9fd03306859ad8effe -LibUnwind.v1.8.2+0.riscv64-linux-gnu.tar.gz/md5/bee9019f3d9cc90ff2dfda358f7ffd92 -LibUnwind.v1.8.2+0.riscv64-linux-gnu.tar.gz/sha512/4b4b3a52af26e069184b3a273427da3e5625a2ea77a9baa6ea0efe17c0b32d6c4a49610ef6554ecc594c0c76572ed2da80860c7c2cafe2c0a2b9341d6455f409 -LibUnwind.v1.8.2+0.x86_64-linux-gnu.tar.gz/md5/2688969e445093554ff7545708f94b9d -LibUnwind.v1.8.2+0.x86_64-linux-gnu.tar.gz/sha512/64461eb5c0f00ba391933b16a0c5cbca9b59056982a04c26bb81fb60c63d728a909ab11190a1720de17bc8587e1a5c79edb100d6c5ff077aff1e35b53383970a -LibUnwind.v1.8.2+0.x86_64-linux-musl.tar.gz/md5/3c1d3abe07a178b0463c8f83c09ee190 -LibUnwind.v1.8.2+0.x86_64-linux-musl.tar.gz/sha512/32919df2e57562c9c85c22586c7d6f35227c360ef7cf0ec0535f876a36fcba862f9874d7fd627b1e6e25a85d04279676aa5192600aa92ec78c00d0b07304897f -LibUnwind.v1.8.2+0.x86_64-unknown-freebsd.tar.gz/md5/69612f0c88883aa9713fa8b90022654c -LibUnwind.v1.8.2+0.x86_64-unknown-freebsd.tar.gz/sha512/3a04384cfc3c38ebc3603c512228d5e50bb0313a2712721a2b60707b4b477c8959aa07aadb54f1b25c9002e09a29407c8bf8370aa4c5dbd7c9e71110c3bea14b +LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/md5/f23ea74855dd2db466170f170a8eea7a +LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/sha512/27b4c37d225c1632e1d8814989d768b0fda2675552c611cb9fad1eea158e4b3815e0a8127c2eea97bcbfd23685d970cf1c478d24f6b5a8abe8357e12d8762ce7 +LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/md5/5073e00eeacd4f5921164090966f5ae8 +LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/sha512/0499cdd22510745e3e1f7da5c4c937a79b2eb07e2bf97afa5db9ce7e91aa2bab85bef2dc3ad8b859ac147ca8da908047bf12daf6e3d3fcb2c7a865402dd90beb +LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/md5/b4aa1f850db793ffe9c1e0bf0524da65 +LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/sha512/92bbb42a742427d4a4bfb67530191093ee55a210a99abe0145bdf4ccdc3e583a9852a8045dfdd20002a8fd8eaf9feb6245f84a4995fee4a097a33a2b64998dcd +LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/md5/a6eff40a5bef97c13fed1546dd77a503 +LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/sha512/b0e687036c080cf5a421a924e070e57d9f52cee1e67e6c9558d586edc58296d15c9e10e2e46c27e6d290c866b84edeee11023077fb2481dfc9f08fd08540326a +LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/md5/19f8f17923dc1b87b3e89480b1eeaedb +LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/sha512/23251a9fe459242589797a364ff69e38711ebf93d4f047acc7d302c8bc669d3b0546a318568787f0f9bf4d1e633670e0941887cbac1c34e34d86e217040533af +LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/md5/74bd5a25811e4977386ba3aefe98bd8c +LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d41f653640ffaebb7295d49a533efebf2ffca7314a181e6e669c399e65b49d176915afeb60b4702632d65aa20aad772052373efbd3c6b9fdc0c9fd772986b2a +LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/md5/7d882db1897c88c7e2eb7cf94e3b3f9e +LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/sha512/dad5bdbccd630eb27af69a32f01245865fc9cf494f1e2ab4cf9e06d78985d997accbac7db6c947bb1964f0aa98f7c977cdb9288e4453fa22d47eb99d73144536 +LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/md5/74076851e0da8ebf94fefa35718b3ec8 +LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/sha512/4549cbe8da9b0ff5cb62c7cdc24534e4fd0f48354902082212d9c559efbdb02017884b41554432e4f16886bafe27b737f4264c05ddba4b14166212290d97f37c +LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/md5/9a91cf5da60391e2bbae8ec77ef158c6 +LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/sha512/5dd44a67c259cbd629d8b17375e00a1e18f9fcf2d9bb86f26eefc31c52cef0ae7ebfd279e37c6b2ef428df98af30348c65a0760e27409ba90614f3cbb80a8c08 +LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/md5/052e192212dbf6412a4ea12ca5a8b40f +LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/sha512/2d985c65453935393de449133dbbfabab68c832ecb6243100e1514ba7a613716dfa2bf690bf1df9a86d9c45eb683e7c61c04ff187347d61b17055964ee7944af +LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/md5/bfd355c19743b5b8121005f41a7a1eaa +LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/sha512/4659d1ddf80cf41c99ee374d98a56264706fd81836ce6cc5e71bfa547bf181cf7775d017d9c3b925a678e33446ca09bb8cda057ed48fdb702e338c9c1adc5aa4 +LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/md5/9c0d4c4c8d96a3b64d1edeba61df80d9 +LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/sha512/2290ca45c950dc6c0ef2f3f915386879e179f98e08b29a5f3e55e8c5fb646389e8dc09dbd1f8f91c382e623a167996322624dfe035142aea11f8e30fa643ec2e +LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/md5/19a6b5638f1c12732a4ea29dc2ba45ca +LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/sha512/834f2e5a2a98e11af20e8d0c3ec91a6e72a60573e8cd556d6c5d2321ecda94afe90911f1b9b6b313140f13433a5c3b1b30d4951ee40ffc862500eea88a44c8b8 +LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/md5/cc97cb986489fb1aaa157490986da567 +LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/sha512/7ca93ec2eaec8f1c22104459ad2db49d5f1303686e8f686312296287b37f51cbde3fb605be84297848fc0997836735e59914f2f8c86fe72bb9ec7c44c35ee8f9 libunwind-1.8.2.tar.gz/md5/0124a38fb752aa5492635f35d089f6b7 libunwind-1.8.2.tar.gz/sha512/f1ff26763c1b2e68948413c4aec22303b6c886425a8264eb65fbd58fc202f79c7b04bd4784bd8499850d08933f0e363cfa3a7d177efdadc223ed0254bc381345 diff --git a/deps/curl.mk b/deps/curl.mk index 5de8c1f766072..1f650dce9370e 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -49,7 +49,7 @@ CURL_CONFIGURE_FLAGS := $(CONFIGURE_COMMON) \ --without-brotli # A few things we actually enable CURL_CONFIGURE_FLAGS += \ - --with-libssh2=${build_prefix} --with-zlib=${build_prefix} --with-nghttp2=${build_prefix} \ + --with-libssh2=${build_prefix} --with-zlib=${build_prefix} --with-zstd=${build_prefix} --with-nghttp2=${build_prefix} \ --enable-versioned-symbols # We use different TLS libraries on different platforms. diff --git a/deps/unwind.mk b/deps/unwind.mk index 12c5cba543da4..2b49b3cf0973a 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -49,7 +49,7 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-appl $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ - $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache + $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-debug_frame --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache echo 1 > $@ $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-compiled: $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured diff --git a/stdlib/LibCURL_jll/Project.toml b/stdlib/LibCURL_jll/Project.toml index 3a2025ea10834..45cc2511c21f7 100644 --- a/stdlib/LibCURL_jll/Project.toml +++ b/stdlib/LibCURL_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibCURL_jll" uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.15.0+0" +version = "8.15.0+1" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -10,6 +10,7 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index fb091caee4d50..528ca92ffb9af 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -3,7 +3,7 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibCURL_jll.jl baremodule LibCURL_jll -using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll +using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll, Zstd_jll if !Sys.iswindows() using OpenSSL_jll end @@ -33,17 +33,18 @@ const libcurl = LazyLibrary( end; dependencies = if Sys.iswindows() if Sys.WORD_SIZE == 32 - LazyLibrary[libz, libnghttp2, libssh2, libgcc_s] + LazyLibrary[libz, libzstd, libnghttp2, libssh2, libgcc_s] else - LazyLibrary[libz, libnghttp2, libssh2] + LazyLibrary[libz, libzstd, libnghttp2, libssh2] end else - LazyLibrary[libz, libnghttp2, libssh2, libssl, libcrypto] + LazyLibrary[libz, libzstd, libnghttp2, libssh2, libssl, libcrypto] end ) function eager_mode() Zlib_jll.eager_mode() + Zstd_jll.eager_mode() nghttp2_jll.eager_mode() LibSSH2_jll.eager_mode() @static if @isdefined CompilerSupportLibraries_jll diff --git a/stdlib/LibUnwind_jll/Project.toml b/stdlib/LibUnwind_jll/Project.toml index 5643f0989f6da..3e22155b27530 100644 --- a/stdlib/LibUnwind_jll/Project.toml +++ b/stdlib/LibUnwind_jll/Project.toml @@ -1,6 +1,6 @@ name = "LibUnwind_jll" uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" -version = "1.8.2+0" +version = "1.8.2+1" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl index 04e5a65a61b23..5cb46865a0706 100644 --- a/test/stdlib_dependencies.jl +++ b/test/stdlib_dependencies.jl @@ -234,6 +234,14 @@ try end end + if prop_name == :libspqr + # Allow libstdc++ to not be linked - spqr only uses std::complex, + # which may be header-only, so doesn't get linked on as-needed distributions. + # However, in general, we can't assume that, so we need to take the dependency + # and just allow this here. + extraneous_deps = setdiff(extraneous_deps, ["libstdc++"]) + end + # We expect there to be no missing or extraneous deps deps_mismatch = !isempty(missing_deps) || !isempty(extraneous_deps) From 1a1270d977ba8aab7fcefe0d652f493bcc3dc44e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 3 Aug 2025 20:49:41 -0400 Subject: [PATCH 623/662] Add missing exec paths to new Zstd_jll stdlib (#59198) `Zstd_jll.zstd_path::String` is provided by the non-stdlib JLL, like for all executable products, so not having this is breaking, I believe. Also add a test to catch this for existing/new stdlibs. --- stdlib/Zstd_jll/src/Zstd_jll.jl | 8 +++++-- test/stdlib_dependencies.jl | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl index ce2dda69d5ee3..30481a12e0979 100644 --- a/stdlib/Zstd_jll/src/Zstd_jll.jl +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -18,6 +18,8 @@ const LIBPATH_list = String[] artifact_dir::String = "" libzstd_path::String = "" +zstd_path::String = "" +zstdmt_path::String = "" const libzstd = LazyLibrary( if Sys.iswindows() BundledLazyLibraryPath("libzstd-1.dll") @@ -79,8 +81,8 @@ function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = tr f(zstdmt()) end end -zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) -zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) +zstd() = adjust_ENV(`$zstd_path`) +zstdmt() = adjust_ENV(`$zstdmt_path`) # Function to eagerly dlopen our library and thus resolve all dependencies function eager_mode() @@ -94,6 +96,8 @@ is_available() = true function __init__() global libzstd_path = string(libzstd.path) + global zstd_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe) + global zstdmt_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe) global artifact_dir = dirname(Sys.BINDIR) end diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl index 5cb46865a0706..b1529b7daebca 100644 --- a/test/stdlib_dependencies.jl +++ b/test/stdlib_dependencies.jl @@ -273,6 +273,47 @@ try end end + # Check that any JLL stdlib that defines executables also provides corresponding *_path variables + @testset "Stdlib JLL executable path variables" begin + for (_, (stdlib_name, _)) in Pkg.Types.stdlibs() + if !endswith(stdlib_name, "_jll") + continue + end + + # Import the stdlib, skip it if it's not available on this platform + m = eval(Meta.parse("import $(stdlib_name); $(stdlib_name)")) + if !Base.invokelatest(getproperty(m, :is_available)) + continue + end + + # Look for *_exe constants that indicate executable definitions + exe_constants = Symbol[] + for name in names(m, all=true) + name_str = string(name) + if endswith(name_str, "_exe") && isdefined(m, name) + push!(exe_constants, name) + end + end + + # For each *_exe constant, check if there's a corresponding *_path variable + for exe_const in exe_constants + exe_name_str = string(exe_const) + # Convert from *_exe to *_path (e.g., zstd_exe -> zstd_path) + expected_path_var = Symbol(replace(exe_name_str, "_exe" => "_path")) + + @test isdefined(m, expected_path_var) + + if !isdefined(m, expected_path_var) + @warn("Missing path variable", + jll = stdlib_name, + exe_constant = exe_const, + expected_path_var = expected_path_var + ) + end + end + end + end + finally if prev_env !== nothing Pkg.activate(prev_env) From b58fd8b0fdfb91a960b2389bd130adc5feca7be3 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Mon, 4 Aug 2025 12:34:35 -0400 Subject: [PATCH 624/662] allow passing `--project` to `juliac.jl` (#59160) My main motivation here is that it allows me to `alias juliac=julia /path/to/contrib/juliac.jl` and then specify `juliac --proj`. --- contrib/juliac/juliac.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/juliac/juliac.jl b/contrib/juliac/juliac.jl index ed80d88444639..eb6277785c789 100644 --- a/contrib/juliac/juliac.jl +++ b/contrib/juliac/juliac.jl @@ -86,6 +86,7 @@ end # arguments to forward to julia compilation process julia_args = [] enable_trim::Bool = false +project::String = "--project=$(Base.active_project())" let i = 1 while i <= length(ARGS) @@ -107,6 +108,8 @@ let i = 1 push!(julia_args, arg) # forwarded arg elseif arg == "--experimental" push!(julia_args, arg) # forwarded arg + elseif startswith(arg, "--proj") + global project = arg else if arg[1] == '-' || !isnothing(file) println("Unexpected argument `$arg`") @@ -150,7 +153,7 @@ bc_path = joinpath(tmpdir, "img-bc.a") function precompile_env() # Pre-compile the environment # (otherwise obscure error messages will occur) - cmd = addenv(`$julia_cmd --project=$(Base.active_project()) -e "using Pkg; Pkg.precompile()"`) + cmd = addenv(`$julia_cmd $project -e "using Pkg; Pkg.precompile()"`) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nError encountered during pre-compilation of environment.") @@ -168,7 +171,7 @@ function compile_products(enable_trim::Bool) end # Compile the Julia code - cmd = addenv(`$julia_cmd_target --project=$(Base.active_project()) --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + cmd = addenv(`$julia_cmd_target $project --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nFailed to compile $file") From b35c4f471f05873a4de6a989e0038a5c28cd09ce Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 4 Aug 2025 17:07:22 -0400 Subject: [PATCH 625/662] model ccall binding access in inference (#58872) fixes #57749 --- Compiler/src/abstractinterpretation.jl | 52 ++++++++++++++++++++++++++ test/ccall.jl | 8 ++++ 2 files changed, 60 insertions(+) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 75a7bcaa4eede..3552525f70cf3 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3485,7 +3485,59 @@ function refine_partial_type(@nospecialize t) return t end +abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::IRInterpretationState) = nothing + +function abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::AbsIntState) + if isexpr(e, :call) + n = length(e.args) + argtypes = Vector{Any}(undef, n) + callresult = Future{CallMeta}() + i::Int = 1 + nextstate::UInt8 = 0x0 + local ai, res + function evalargs(interp, sv) + if nextstate === 0x1 + @goto state1 + elseif nextstate === 0x2 + @goto state2 + end + while i <= n + ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv) + if !isready(ai) + nextstate = 0x1 + return false + @label state1 + end + argtypes[i] = ai[].rt + i += 1 + end + res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv) + if !isready(res) + nextstate = 0x2 + return false + @label state2 + end + callresult[] = res[] + return true + end + evalargs(interp, sv) || push!(sv.tasks, evalargs) + return callresult + else + return Future(abstract_eval_basic_statement(interp, e, sstate, sv)) + end +end + function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) + callee = e.args[1] + if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple) + # NOTE these expressions are not properly linearized + abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv) + if length(callee.args) > 2 + abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv) + end + else + abstract_eval_value(interp, callee, sstate, sv) + end mi = frame_instance(sv) t = sp_type_rewrap(e.args[2], mi, true) for i = 3:length(e.args) diff --git a/test/ccall.jl b/test/ccall.jl index f193d16fc09e2..048d74af7c298 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -1979,3 +1979,11 @@ let llvm = sprint(code_llvm, gc_safe_ccall, ()) # check for the gc_safe store @test occursin("store atomic i8 2", llvm) end + +module Test57749 +using Test, Zstd_jll +const prefix = "Zstd version: " +const sym = :ZSTD_versionString +get_zstd_version() = prefix * unsafe_string(ccall((sym, libzstd), Cstring, ())) +@test startswith(get_zstd_version(), "Zstd") +end From db587b4979221dc0f29986378a297ba5fff8570a Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 27 Mar 2025 23:11:18 +0000 Subject: [PATCH 626/662] Support moving --- src/aotcompile.cpp | 1 + src/ast.c | 3 + src/builtins.c | 12 + src/cgutils.cpp | 3 + src/codegen.cpp | 10 + src/datatype.c | 4 +- src/gc-common.c | 5 + src/gc-interface.h | 21 ++ src/gc-mmtk.c | 409 ++++++++++++++++++++++++++-- src/gc-stock.c | 40 +++ src/gc-tls-mmtk.h | 4 + src/genericmemory.c | 4 + src/gf.c | 2 +- src/interpreter.c | 9 + src/ircode.c | 5 +- src/jitlayers.h | 2 + src/jl_exported_funcs.inc | 2 + src/jl_uv.c | 2 + src/julia.h | 62 +++++ src/julia_internal.h | 6 + src/julia_threads.h | 4 + src/llvm-final-gc-lowering.cpp | 2 + src/llvm-gc-interface-passes.h | 9 + src/llvm-late-gc-lowering-mmtk.cpp | 72 +++++ src/llvm-late-gc-lowering-stock.cpp | 4 + src/llvm-late-gc-lowering.cpp | 14 +- src/llvm-pass-helpers.cpp | 48 ++++ src/llvm-pass-helpers.h | 6 + src/method.c | 2 +- src/module.c | 2 +- src/runtime_ccall.cpp | 2 + src/signals-unix.c | 2 + src/staticdata.c | 15 +- src/staticdata_utils.c | 2 +- src/task.c | 12 +- src/toplevel.c | 1 + 36 files changed, 754 insertions(+), 49 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index ae5e826a6152f..8bee81264a3a7 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -72,6 +72,7 @@ typedef struct { SmallVector jl_sysimg_fvars; SmallVector jl_sysimg_gvars; std::map> jl_fvar_map; + // This holds references to the heap. Need to be pinned. SmallVector jl_value_to_llvm; SmallVector jl_external_to_llvm; } jl_native_code_desc_t; diff --git a/src/ast.c b/src/ast.c index 04c87d220f409..d7679f4a6ad51 100644 --- a/src/ast.c +++ b/src/ast.c @@ -776,6 +776,9 @@ static value_t julia_to_list2_noalloc(fl_context_t *fl_ctx, jl_value_t *a, jl_va static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid) { + // The following code will take internal pointers to v's fields. We need to make sure + // that v will not be moved by GC. + OBJ_PIN(v); value_t retval; if (julia_to_scm_noalloc1(fl_ctx, v, &retval)) return retval; diff --git a/src/builtins.c b/src/builtins.c index 2fcdd4281f371..e972a1839a5e0 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -367,6 +367,9 @@ static uintptr_t type_object_id_(jl_value_t *v, jl_varidx_t *env) JL_NOTSAFEPOIN i++; pe = pe->prev; } + // FIXME: Pinning objects that get hashed + // until we implement address space hashing. + OBJ_PIN(v); uintptr_t bits = jl_astaggedvalue(v)->header; if (bits & GC_IN_IMAGE) return ((uintptr_t*)v)[-2]; @@ -423,6 +426,11 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT // a few select pointers (notably symbol) also have special hash values // which may affect the stability of the objectid hash, even though // they don't affect egal comparison + + // FIXME: Pinning objects that get hashed + // until we implement address space hashing. + PTR_PIN(v); // This has to be a pointer pin -- v could be an internal pointer + return bits_hash(v, sz) ^ h; } if (dt == jl_unionall_type) @@ -483,6 +491,10 @@ static uintptr_t NOINLINE jl_object_id__cold(uintptr_t tv, jl_value_t *v) JL_NOT uintptr_t bits = jl_astaggedvalue(v)->header; if (bits & GC_IN_IMAGE) return ((uintptr_t*)v)[-2]; + + // FIXME: Pinning objects that get hashed + // until we implement address space hashing. + OBJ_PIN(v); return inthash((uintptr_t)v); } return immut_id_(dt, v, dt->hash); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 97a58f8aa419c..9d70d4bbda53a 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -397,6 +397,7 @@ static Constant *julia_pgv(jl_codegen_params_t ¶ms, Module *M, const char *c // emit a GlobalVariable for a jl_value_t named "cname" // store the name given so we can reuse it (facilitating merging later) // so first see if there already is a GlobalVariable for this address + OBJ_PIN(addr); // This will be stored in the native heap. We need to pin it. GlobalVariable* &gv = params.global_targets[addr]; StringRef localname; std::string gvname; @@ -564,6 +565,8 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) { if (p == NULL) return Constant::getNullValue(ctx.types().T_pjlvalue); + // Pointers to p will be emitted into the code. Make sure p won't be moved by GC. + OBJ_PIN(p); Value *pgv = literal_pointer_val_slot(ctx.emission_context, jl_Module, p); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto load = ai.decorateInst(maybe_mark_load_dereferenceable( diff --git a/src/codegen.cpp b/src/codegen.cpp index 2d9c94baee9b4..4d7606c25e21f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1752,6 +1752,7 @@ struct jl_cgval_t { promotion_point(nullptr), promotion_ssa(-1) { + OBJ_PIN(typ); // jl_cgval_t could be in the native heap. We have to pin the object references in it. assert(TIndex == nullptr || TIndex->getType() == getInt8Ty(TIndex->getContext())); } jl_cgval_t(Value *Vptr, bool isboxed, jl_value_t *typ, Value *tindex, MDNode *tbaa, Value* inline_roots) = delete; @@ -1768,6 +1769,7 @@ struct jl_cgval_t { promotion_point(nullptr), promotion_ssa(-1) { + OBJ_PIN(typ); // jl_cgval_t could be in the native heap. We have to pin the object references in it. if (Vboxed) assert(Vboxed->getType() == JuliaType::get_prjlvalue_ty(Vboxed->getContext())); assert(tbaa != nullptr); @@ -1788,6 +1790,8 @@ struct jl_cgval_t { promotion_point(nullptr), promotion_ssa(-1) { + OBJ_PIN(typ); // jl_cgval_t could be in the native heap. We have to pin the object references in it. + OBJ_PIN(constant); // jl_cgval_t could be in the native heap. We have to pin the object references in it. assert(jl_is_datatype(typ)); assert(constant); } @@ -1804,6 +1808,8 @@ struct jl_cgval_t { promotion_point(v.promotion_point), promotion_ssa(v.promotion_ssa) { + OBJ_PIN(typ); // jl_cgval_t could be in the native heap. We have to pin the object references in it. + OBJ_PIN(constant); // jl_cgval_t could be in the native heap. We have to pin the object references in it. if (Vboxed) assert(Vboxed->getType() == JuliaType::get_prjlvalue_ty(Vboxed->getContext())); // this constructor expects we had a badly or equivalently typed version @@ -1877,6 +1883,7 @@ class jl_codectx_t { std::map > scope_restore; std::map eh_buffers; SmallVector SAvalues; + // The vector holds reference to Julia obj ref. We need to pin jl_value_t*. SmallVector, jl_value_t *>, 0> PhiNodes; SmallVector ssavalue_assigned; SmallVector ssavalue_usecount; @@ -5744,6 +5751,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) decay_derived(ctx, phi)); jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, best_tbaa(ctx.tbaa(), phiType)); val.Vboxed = ptr_phi; + OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, roots, r)); ctx.SAvalues[idx] = val; ctx.ssavalue_assigned[idx] = true; @@ -5753,6 +5761,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_nrows(edges), "tindex_phi"); Tindex_phi->insertInto(BB, InsertPt); jl_cgval_t val = mark_julia_slot(NULL, phiType, Tindex_phi, ctx.tbaa().tbaa_stack); + OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, (PHINode*)nullptr, roots, r)); ctx.SAvalues[idx] = val; ctx.ssavalue_assigned[idx] = true; @@ -5807,6 +5816,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) value_phi->insertInto(BB, InsertPt); slot = mark_julia_type(ctx, value_phi, isboxed, phiType); } + OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. ctx.PhiNodes.push_back(std::make_tuple(slot, BB, dest, value_phi, roots, r)); ctx.SAvalues[idx] = slot; ctx.ssavalue_assigned[idx] = true; diff --git a/src/datatype.c b/src/datatype.c index 450bf89748474..1dbabbfe96688 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -71,7 +71,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu { jl_task_t *ct = jl_current_task; jl_typename_t *tn = - (jl_typename_t*)jl_gc_alloc(ct->ptls, sizeof(jl_typename_t), + (jl_typename_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_typename_t), jl_typename_type); tn->name = name; tn->module = module; @@ -106,7 +106,7 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_module_t *module, jl_dat jl_datatype_t *jl_new_uninitialized_datatype(void) { jl_task_t *ct = jl_current_task; - jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type); + jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type); jl_set_typetagof(t, jl_datatype_tag, 0); t->hash = 0; t->hasfreetypevars = 0; diff --git a/src/gc-common.c b/src/gc-common.c index 87ddb68abc704..a5067bbf125d9 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -540,6 +540,11 @@ JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, void *ty) return jl_gc_alloc_(ptls, sz, ty); } +JL_DLLEXPORT jl_value_t *(jl_gc_alloc_nonmoving)(jl_ptls_t ptls, size_t sz, void *ty) +{ + return jl_gc_alloc_nonmoving_(ptls, sz, ty); +} + JL_DLLEXPORT void *jl_malloc(size_t sz) { return jl_gc_counted_malloc(sz); diff --git a/src/gc-interface.h b/src/gc-interface.h index 9108cdb331946..446d35f45d336 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -99,6 +99,10 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem); // Runs a GC cycle. This function's parameter determines whether we're running an // incremental, full, or automatic (i.e. heuristic driven) collection. JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection); +// Pinning objects; Returns whether the object has been pinned by this call. +JL_DLLEXPORT unsigned char jl_gc_pin_object(void* obj); +// Pinning objects through a potential internal pointer; Returns whether the object has been pinned by this call. +JL_DLLEXPORT unsigned char jl_gc_pin_pointer(void* ptr); // Returns whether the thread with `tid` is a collector thread JL_DLLEXPORT int gc_is_collector_thread(int tid) JL_NOTSAFEPOINT; // Returns which GC implementation is being used and possibly its version according to the list of supported GCs @@ -108,6 +112,16 @@ JL_DLLEXPORT const char* jl_gc_active_impl(void); // each GC should implement it but it will most likely not be used by other code in the runtime. // It still needs to be annotated with JL_DLLEXPORT since it is called from Rust by MMTk. JL_DLLEXPORT void jl_gc_sweep_stack_pools_and_mtarraylist_buffers(jl_ptls_t ptls) JL_NOTSAFEPOINT; +// Notifies the GC that the given thread is about to yield for a GC. ctx is the ucontext for the thread +// if it is already fetched by the caller, otherwise it is NULL. +JL_DLLEXPORT void jl_gc_notify_thread_yield(jl_ptls_t ptls, void* ctx); + +// TODO: The preserve hook functions may be temporary. We should see the performance impact of the change. + +// Runtime hook for gc preserve begin. The GC needs to make sure that the preserved objects and its children stay alive and won't move. +JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT; +// Runtime hook for gc preserve end. The GC needs to make sure that the preserved objects and its children stay alive and won't move. +JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT; // ========================================================================= // // Metrics @@ -148,6 +162,9 @@ JL_DLLEXPORT uint64_t jl_gc_total_hrtime(void); // **must** also set the type of the returning object to be `ty`. The type `ty` may also be used to record // an allocation of that type in the allocation profiler. struct _jl_value_t *jl_gc_alloc_(struct _jl_tls_states_t * ptls, size_t sz, void *ty); +// Similar to jl_gc_alloc_, except that the GC needs to make sure the object allocated from this function will +// not be moved by the GC. +struct _jl_value_t *jl_gc_alloc_nonmoving_(struct _jl_tls_states_t * ptls, size_t sz, void *ty); // Allocates small objects and increments Julia allocation counterst. Size of the object // header must be included in the object size. The (possibly unused in some implementations) // offset to the arena in which we're allocating is passed in the second parameter, and the @@ -214,6 +231,10 @@ struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty, unsigned align) JL_NOTSAF // The GC may use that information to, for instance, determine that such objects should // be treated as marked and belonged to the old generation in nursery collections. void jl_gc_notify_image_load(const char* img_data, size_t len); +// This function notifies the GC about memory addresses that are set when allocating the boot image. +// The GC may use that information to, for instance, determine that all objects in that chunk of memory should +// be treated as marked and belonged to the old generation in nursery collections. +void jl_gc_notify_image_alloc(const char* img_data, size_t len); // ========================================================================= // // Runtime Write-Barriers diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 9e3dc6f8d87ed..1af93fcc61a9e 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -55,6 +55,11 @@ extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, extern void* mmtk_alloc(void* mutator, size_t size, size_t align, size_t offset, int allocator); extern void mmtk_post_alloc(void* mutator, void* refer, size_t bytes, int allocator); extern void mmtk_store_obj_size_c(void* obj, size_t size); +extern bool mmtk_is_pinned(void* obj); +extern unsigned char mmtk_pin_object(void* obj); +extern bool mmtk_is_object_pinned(void* obj); +extern unsigned char mmtk_pin_pointer(void* ptr); +extern bool mmtk_is_pointer_pinned(void* ptr); extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; @@ -298,6 +303,8 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) gc_num.total_time_to_safepoint += duration; if (!jl_atomic_load_acquire(&jl_gc_disable_counter)) { + // This thread will yield. + jl_gc_notify_thread_yield(ptls, NULL); JL_LOCK_NOGC(&finalizers_lock); // all the other threads are stopped, so this does not make sense, right? otherwise, failing that, this seems like plausibly a deadlock #ifndef __clang_gcanalyzer__ mmtk_block_thread_for_gc(); @@ -327,6 +334,27 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) errno = last_errno; } +JL_DLLEXPORT unsigned char jl_gc_pin_object(void* obj) { + return mmtk_pin_object(obj); +} + +JL_DLLEXPORT unsigned char jl_gc_pin_pointer(void* ptr) { + return mmtk_pin_pointer(ptr); +} + +JL_DLLEXPORT void jl_gc_notify_thread_yield(jl_ptls_t ptls, void* ctx) { + if (ctx == NULL) { + // Save the context for the thread as it was running at the time of the call + int r = getcontext(&ptls->gc_tls.ctx_at_the_time_gc_started); + if (r == -1) { + jl_safe_printf("Failed to save context for conservative scanning\n"); + abort(); + } + return; + } + memcpy(&ptls->gc_tls.ctx_at_the_time_gc_started, ctx, sizeof(ucontext_t)); +} + // ========================================================================= // // GC Statistics // ========================================================================= // @@ -497,43 +525,299 @@ static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWor } } -JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) -{ - // Create a new buf - RootsWorkBuffer buf = (closure->report_nodes_func)((void**)0, 0, 0, closure->data, true); - size_t len = 0; +// staticdata_utils.c +extern jl_array_t *internal_methods; +extern jl_array_t *newly_inferred; +// task.c +extern jl_function_t* task_done_hook_func; +#define TRACE_GLOBALLY_ROOTED(r) add_node_to_roots_buffer(closure, buf, buf_len, r) + +// This is a list of global variables that are marked with JL_GLOBALLY_ROOTED. We need to make sure that they +// won't be moved. Otherwise, when we access those objects from the C global variables, we may see moved references. +void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len) +{ + TRACE_GLOBALLY_ROOTED(cmpswap_names); + TRACE_GLOBALLY_ROOTED(jl_typeinf_func); + TRACE_GLOBALLY_ROOTED(_jl_debug_method_invalidation); + // Max 4096 + for (size_t i = 0; i < N_CALL_CACHE; i++) { + TRACE_GLOBALLY_ROOTED(call_cache[i]); + } + // julia_internal.h + TRACE_GLOBALLY_ROOTED(jl_type_type_mt); + TRACE_GLOBALLY_ROOTED(jl_nonfunction_mt); + TRACE_GLOBALLY_ROOTED(jl_kwcall_mt); + TRACE_GLOBALLY_ROOTED(jl_opaque_closure_method); + TRACE_GLOBALLY_ROOTED(jl_nulldebuginfo); + TRACE_GLOBALLY_ROOTED(_jl_debug_method_invalidation); + TRACE_GLOBALLY_ROOTED(jl_module_init_order); + // TRACE_GLOBALLY_ROOTED(jl_current_modules); -- we cannot trace a htable_t. So we trace each module. + for (size_t i = 0; i < jl_current_modules.size; i += 2) { + if (jl_current_modules.table[i + 1] != HT_NOTFOUND) { + TRACE_GLOBALLY_ROOTED(jl_current_modules.table[i]); + } + } + for (size_t i = 0; i < N_CALL_CACHE; i++) { + jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); + TRACE_GLOBALLY_ROOTED(v); + } + TRACE_GLOBALLY_ROOTED(jl_precompile_toplevel_module); + TRACE_GLOBALLY_ROOTED(jl_global_roots_list); + TRACE_GLOBALLY_ROOTED(jl_global_roots_keyset); + TRACE_GLOBALLY_ROOTED(precompile_field_replace); + // julia.h + TRACE_GLOBALLY_ROOTED(jl_typeofbottom_type); + TRACE_GLOBALLY_ROOTED(jl_datatype_type); + TRACE_GLOBALLY_ROOTED(jl_uniontype_type); + TRACE_GLOBALLY_ROOTED(jl_unionall_type); + TRACE_GLOBALLY_ROOTED(jl_tvar_type); + + TRACE_GLOBALLY_ROOTED(jl_any_type); + TRACE_GLOBALLY_ROOTED(jl_type_type); + TRACE_GLOBALLY_ROOTED(jl_typename_type); + TRACE_GLOBALLY_ROOTED(jl_type_typename); + TRACE_GLOBALLY_ROOTED(jl_symbol_type); + TRACE_GLOBALLY_ROOTED(jl_ssavalue_type); + TRACE_GLOBALLY_ROOTED(jl_slotnumber_type); + TRACE_GLOBALLY_ROOTED(jl_argument_type); + TRACE_GLOBALLY_ROOTED(jl_const_type); + TRACE_GLOBALLY_ROOTED(jl_partial_struct_type); + TRACE_GLOBALLY_ROOTED(jl_partial_opaque_type); + TRACE_GLOBALLY_ROOTED(jl_interconditional_type); + TRACE_GLOBALLY_ROOTED(jl_method_match_type); + TRACE_GLOBALLY_ROOTED(jl_simplevector_type); + TRACE_GLOBALLY_ROOTED(jl_tuple_typename); + TRACE_GLOBALLY_ROOTED(jl_vecelement_typename); + TRACE_GLOBALLY_ROOTED(jl_anytuple_type); + TRACE_GLOBALLY_ROOTED(jl_emptytuple_type); + TRACE_GLOBALLY_ROOTED(jl_anytuple_type_type); + TRACE_GLOBALLY_ROOTED(jl_vararg_type); + TRACE_GLOBALLY_ROOTED(jl_function_type); + TRACE_GLOBALLY_ROOTED(jl_builtin_type); + TRACE_GLOBALLY_ROOTED(jl_opaque_closure_type); + TRACE_GLOBALLY_ROOTED(jl_opaque_closure_typename); + + TRACE_GLOBALLY_ROOTED(jl_bottom_type); + TRACE_GLOBALLY_ROOTED(jl_method_instance_type); + TRACE_GLOBALLY_ROOTED(jl_code_instance_type); + TRACE_GLOBALLY_ROOTED(jl_code_info_type); + TRACE_GLOBALLY_ROOTED(jl_debuginfo_type); + TRACE_GLOBALLY_ROOTED(jl_method_type); + TRACE_GLOBALLY_ROOTED(jl_module_type); + TRACE_GLOBALLY_ROOTED(jl_addrspace_type); + TRACE_GLOBALLY_ROOTED(jl_addrspacecore_type); + TRACE_GLOBALLY_ROOTED(jl_abstractarray_type); + TRACE_GLOBALLY_ROOTED(jl_densearray_type); + TRACE_GLOBALLY_ROOTED(jl_array_type); + TRACE_GLOBALLY_ROOTED(jl_array_typename); + TRACE_GLOBALLY_ROOTED(jl_genericmemory_type); + TRACE_GLOBALLY_ROOTED(jl_genericmemory_typename); + TRACE_GLOBALLY_ROOTED(jl_genericmemoryref_type); + TRACE_GLOBALLY_ROOTED(jl_genericmemoryref_typename); + TRACE_GLOBALLY_ROOTED(jl_weakref_type); + TRACE_GLOBALLY_ROOTED(jl_abstractstring_type); + TRACE_GLOBALLY_ROOTED(jl_string_type); + TRACE_GLOBALLY_ROOTED(jl_errorexception_type); + TRACE_GLOBALLY_ROOTED(jl_argumenterror_type); + TRACE_GLOBALLY_ROOTED(jl_loaderror_type); + TRACE_GLOBALLY_ROOTED(jl_initerror_type); + TRACE_GLOBALLY_ROOTED(jl_typeerror_type); + TRACE_GLOBALLY_ROOTED(jl_methoderror_type); + TRACE_GLOBALLY_ROOTED(jl_undefvarerror_type); + TRACE_GLOBALLY_ROOTED(jl_fielderror_type); + TRACE_GLOBALLY_ROOTED(jl_atomicerror_type); + TRACE_GLOBALLY_ROOTED(jl_missingcodeerror_type); + TRACE_GLOBALLY_ROOTED(jl_lineinfonode_type); + TRACE_GLOBALLY_ROOTED(jl_stackovf_exception); + TRACE_GLOBALLY_ROOTED(jl_memory_exception); + TRACE_GLOBALLY_ROOTED(jl_readonlymemory_exception); + TRACE_GLOBALLY_ROOTED(jl_diverror_exception); + TRACE_GLOBALLY_ROOTED(jl_undefref_exception); + TRACE_GLOBALLY_ROOTED(jl_interrupt_exception); + TRACE_GLOBALLY_ROOTED(jl_precompilable_error); + TRACE_GLOBALLY_ROOTED(jl_boundserror_type); + TRACE_GLOBALLY_ROOTED(jl_an_empty_vec_any); + TRACE_GLOBALLY_ROOTED(jl_an_empty_memory_any); + TRACE_GLOBALLY_ROOTED(jl_an_empty_string); + + TRACE_GLOBALLY_ROOTED(jl_bool_type); + TRACE_GLOBALLY_ROOTED(jl_char_type); + TRACE_GLOBALLY_ROOTED(jl_int8_type); + TRACE_GLOBALLY_ROOTED(jl_uint8_type); + TRACE_GLOBALLY_ROOTED(jl_int16_type); + TRACE_GLOBALLY_ROOTED(jl_uint16_type); + TRACE_GLOBALLY_ROOTED(jl_int32_type); + TRACE_GLOBALLY_ROOTED(jl_uint32_type); + TRACE_GLOBALLY_ROOTED(jl_int64_type); + TRACE_GLOBALLY_ROOTED(jl_uint64_type); + TRACE_GLOBALLY_ROOTED(jl_float16_type); + TRACE_GLOBALLY_ROOTED(jl_float32_type); + TRACE_GLOBALLY_ROOTED(jl_float64_type); + TRACE_GLOBALLY_ROOTED(jl_floatingpoint_type); + TRACE_GLOBALLY_ROOTED(jl_number_type); + TRACE_GLOBALLY_ROOTED(jl_void_type); // deprecated + TRACE_GLOBALLY_ROOTED(jl_nothing_type); + TRACE_GLOBALLY_ROOTED(jl_signed_type); + TRACE_GLOBALLY_ROOTED(jl_voidpointer_type); + TRACE_GLOBALLY_ROOTED(jl_uint8pointer_type); + TRACE_GLOBALLY_ROOTED(jl_pointer_type); + TRACE_GLOBALLY_ROOTED(jl_llvmpointer_type); + TRACE_GLOBALLY_ROOTED(jl_ref_type); + TRACE_GLOBALLY_ROOTED(jl_pointer_typename); + TRACE_GLOBALLY_ROOTED(jl_llvmpointer_typename); + TRACE_GLOBALLY_ROOTED(jl_namedtuple_typename); + TRACE_GLOBALLY_ROOTED(jl_namedtuple_type); + TRACE_GLOBALLY_ROOTED(jl_task_type); + TRACE_GLOBALLY_ROOTED(jl_pair_type); + + TRACE_GLOBALLY_ROOTED(jl_array_uint8_type); + TRACE_GLOBALLY_ROOTED(jl_array_any_type); + TRACE_GLOBALLY_ROOTED(jl_array_symbol_type); + TRACE_GLOBALLY_ROOTED(jl_array_int32_type); + TRACE_GLOBALLY_ROOTED(jl_array_uint32_type); + TRACE_GLOBALLY_ROOTED(jl_array_uint64_type); + TRACE_GLOBALLY_ROOTED(jl_memory_uint8_type); + TRACE_GLOBALLY_ROOTED(jl_memory_uint16_type); + TRACE_GLOBALLY_ROOTED(jl_memory_uint32_type); + TRACE_GLOBALLY_ROOTED(jl_memory_uint64_type); + TRACE_GLOBALLY_ROOTED(jl_memory_any_type); + TRACE_GLOBALLY_ROOTED(jl_memoryref_uint8_type); + TRACE_GLOBALLY_ROOTED(jl_memoryref_any_type); + TRACE_GLOBALLY_ROOTED(jl_expr_type); + TRACE_GLOBALLY_ROOTED(jl_binding_type); + TRACE_GLOBALLY_ROOTED(jl_binding_partition_type); + TRACE_GLOBALLY_ROOTED(jl_globalref_type); + TRACE_GLOBALLY_ROOTED(jl_linenumbernode_type); + TRACE_GLOBALLY_ROOTED(jl_gotonode_type); + TRACE_GLOBALLY_ROOTED(jl_gotoifnot_type); + TRACE_GLOBALLY_ROOTED(jl_enternode_type); + TRACE_GLOBALLY_ROOTED(jl_returnnode_type); + TRACE_GLOBALLY_ROOTED(jl_phinode_type); + TRACE_GLOBALLY_ROOTED(jl_pinode_type); + TRACE_GLOBALLY_ROOTED(jl_phicnode_type); + TRACE_GLOBALLY_ROOTED(jl_upsilonnode_type); + TRACE_GLOBALLY_ROOTED(jl_quotenode_type); + TRACE_GLOBALLY_ROOTED(jl_newvarnode_type); + TRACE_GLOBALLY_ROOTED(jl_intrinsic_type); + TRACE_GLOBALLY_ROOTED(jl_methtable_type); + TRACE_GLOBALLY_ROOTED(jl_typemap_level_type); + TRACE_GLOBALLY_ROOTED(jl_typemap_entry_type); + + TRACE_GLOBALLY_ROOTED(jl_emptysvec); + TRACE_GLOBALLY_ROOTED(jl_emptytuple); + TRACE_GLOBALLY_ROOTED(jl_true); + TRACE_GLOBALLY_ROOTED(jl_false); + TRACE_GLOBALLY_ROOTED(jl_nothing); + TRACE_GLOBALLY_ROOTED(jl_kwcall_func); + + TRACE_GLOBALLY_ROOTED(jl_libdl_dlopen_func); + + TRACE_GLOBALLY_ROOTED(jl_main_module); + TRACE_GLOBALLY_ROOTED(jl_core_module); + TRACE_GLOBALLY_ROOTED(jl_base_module); + TRACE_GLOBALLY_ROOTED(jl_top_module); + TRACE_GLOBALLY_ROOTED(jl_libdl_module); + + // staticdata_utils.c + TRACE_GLOBALLY_ROOTED(internal_methods); + TRACE_GLOBALLY_ROOTED(newly_inferred); + // task.c + TRACE_GLOBALLY_ROOTED(task_done_hook_func); + // threading.c + // TRACE_GLOBALLY_ROOTED(jl_all_tls_states); -- we don't need to pin these. Julia TLS are allocated with calloc. +} + +// These are from gc_mark_roots -- this is not enough for a moving GC. We need to make sure +// all the globally rooted symbols are traced and will not move. This function is unused. +// We use trace_full_globally_rooted() instead. +void trace_partial_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len) +{ // add module - add_node_to_roots_buffer(closure, &buf, &len, jl_main_module); + TRACE_GLOBALLY_ROOTED(jl_main_module); // buildin values - add_node_to_roots_buffer(closure, &buf, &len, jl_an_empty_vec_any); - add_node_to_roots_buffer(closure, &buf, &len, jl_module_init_order); + TRACE_GLOBALLY_ROOTED(jl_an_empty_vec_any); + TRACE_GLOBALLY_ROOTED(jl_module_init_order); + for (size_t i = 0; i < jl_current_modules.size; i += 2) { if (jl_current_modules.table[i + 1] != HT_NOTFOUND) { - add_node_to_roots_buffer(closure, &buf, &len, jl_current_modules.table[i]); + TRACE_GLOBALLY_ROOTED(jl_current_modules.table[i]); } } - add_node_to_roots_buffer(closure, &buf, &len, jl_anytuple_type_type); + TRACE_GLOBALLY_ROOTED(jl_anytuple_type_type); for (size_t i = 0; i < N_CALL_CACHE; i++) { - jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); - add_node_to_roots_buffer(closure, &buf, &len, v); + jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); + TRACE_GLOBALLY_ROOTED(v); } - add_node_to_roots_buffer(closure, &buf, &len, _jl_debug_method_invalidation); + TRACE_GLOBALLY_ROOTED(_jl_debug_method_invalidation); // constants - add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type); - add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names); + TRACE_GLOBALLY_ROOTED(jl_emptytuple_type); + TRACE_GLOBALLY_ROOTED(cmpswap_names); + TRACE_GLOBALLY_ROOTED(jl_global_roots_list); + TRACE_GLOBALLY_ROOTED(jl_global_roots_keyset); + TRACE_GLOBALLY_ROOTED(precompile_field_replace); +} + +JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) +{ + + // Create a new buf + RootsWorkBuffer buf = (closure->report_nodes_func)((void**)0, 0, 0, closure->data, true); + size_t len = 0; + + // globally rooted + trace_full_globally_rooted(closure, &buf, &len); + + // Simply pin things in global roots table + // size_t i; + // for (i = 0; i < jl_array_len(jl_global_roots_table); i++) { + // jl_value_t* root = jl_array_ptr_ref(jl_global_roots_table, i); + // add_node_to_roots_buffer(closure, &buf, &len, root); + // } + // for (i = 0; i < jl_global_roots_list->length; i++) { + // jl_value_t* root = jl_genericmemory_ptr_ref(jl_global_roots_list, i); + // add_node_to_roots_buffer(closure, &buf, &len, root); + // } + // for (i = 0; i < jl_global_roots_keyset->length; i++) { + // jl_value_t* root = jl_genericmemory_ptr_ref(jl_global_roots_keyset, i); + // add_node_to_roots_buffer(closure, &buf, &len, root); + // } + // add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_list); + // add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_keyset); + + // // add module + // add_node_to_roots_buffer(closure, &buf, &len, jl_main_module); + + // // buildin values + // add_node_to_roots_buffer(closure, &buf, &len, jl_an_empty_vec_any); + // add_node_to_roots_buffer(closure, &buf, &len, jl_module_init_order); + // for (size_t i = 0; i < jl_current_modules.size; i += 2) { + // if (jl_current_modules.table[i + 1] != HT_NOTFOUND) { + // add_node_to_roots_buffer(closure, &buf, &len, jl_current_modules.table[i]); + // } + // } + // add_node_to_roots_buffer(closure, &buf, &len, jl_anytuple_type_type); + // for (size_t i = 0; i < N_CALL_CACHE; i++) { + // jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); + // add_node_to_roots_buffer(closure, &buf, &len, v); + // } + // add_node_to_roots_buffer(closure, &buf, &len, _jl_debug_method_invalidation); + + // // constants + // add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type); + // add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names); + // add_node_to_roots_buffer(closure, &buf, &len, precompile_field_replace); // jl_global_roots_table must be transitively pinned + // FIXME: We need to remove transitive pinning of global roots. Otherwise they may pin most of the objects in the heap. RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true); size_t tpinned_len = 0; add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_list); add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_keyset); - // FIXME: transivitely pinning for now, should be removed after we add moving Immix add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, precompile_field_replace); - // Push the result of the work. (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false); (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false); @@ -790,6 +1074,8 @@ JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls, jl_value_t *valu { jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*), jl_weakref_type); wr->value = value; // NOTE: wb not needed here + // Note: we are using MMTk's weak ref processing. If we switch to Julia's weak ref processing, + // we need to make sure the value and the weak ref won't be moved (e.g. pin them) mmtk_add_weak_candidate(wr); return wr; } @@ -846,18 +1132,28 @@ STATIC_INLINE void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* curs } } +STATIC_INLINE void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) { + intptr_t addr = (intptr_t) obj; + uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6); + intptr_t shift = (addr >> 3) & 0b111; + while(1) { + uint8_t old_val = *meta_addr; + uint8_t new_val = old_val | (1 << shift); + if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { + break; + } + } +} + STATIC_INLINE void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_DEFAULT_IMMIX_ALLOCATOR]; return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0); } -inline void mmtk_immix_post_alloc_slow(MMTkMutatorContext* mutator, void* obj, size_t size) { - mmtk_post_alloc(mutator, obj, size, 0); -} - STATIC_INLINE void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - // FIXME: for now, we do nothing - // but when supporting moving, this is where we set the valid object (VO) bit + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); + } } STATIC_INLINE void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { @@ -866,9 +1162,9 @@ STATIC_INLINE void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t } STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - // FIXME: Similarly, for now, we do nothing - // but when supporting moving, this is where we set the valid object (VO) bit - // and log (old gen) bit + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); + } } JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int osize, size_t align, void *ty) @@ -967,6 +1263,15 @@ inline jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) return v; } +inline jl_value_t *jl_gc_alloc_nonmoving_(jl_ptls_t ptls, size_t sz, void *ty) +{ + // TODO: Currently we just alloc and pin the object. We may use a + // different non moving allocator instead. + jl_value_t *v = jl_gc_alloc_(ptls, sz, ty); + OBJ_PIN(v); + return v; +} + // allocation wrappers that track allocation and let collection run JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) { @@ -1085,6 +1390,11 @@ void jl_gc_notify_image_load(const char* img_data, size_t len) mmtk_set_vm_space((void*)img_data, len); } +void jl_gc_notify_image_alloc(const char* img_data, size_t len) +{ + mmtk_immortal_region_post_alloc((void*)img_data, len); +} + // ========================================================================= // // Code specific to stock that is not supported by MMTk // ========================================================================= // @@ -1214,6 +1524,53 @@ JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p) return NULL; } +#define jl_p_gcpreserve_stack (jl_current_task->gcpreserve_stack) + +// This macro currently uses malloc instead of alloca because this function will exit +// after pushing the roots into the gc_preserve_stack, which means that the preserve_begin function's +// stack frame will be destroyed (together with its alloca variables). When we support lowering this code +// inside the same function that is doing the preserve_begin/preserve_end calls we should be able to simple use allocas. +// Note also that we use a separate stack for gc preserve roots to avoid the possibility of calling free +// on a stack that has been allocated with alloca instead of malloc, which could happen depending on the order in which +// JL_GC_POP() and jl_gc_preserve_end_hook() occurs. + +#define JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(rts_var,n) \ + rts_var = ((jl_value_t**)malloc(((n)+2)*sizeof(jl_value_t*)))+2; \ + ((void**)rts_var)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(n); \ + ((void**)rts_var)[-1] = jl_p_gcpreserve_stack; \ + memset((void*)rts_var, 0, (n)*sizeof(jl_value_t*)); \ + jl_p_gcpreserve_stack = (jl_gcframe_t*)&(((void**)rts_var)[-2]); \ + +#define JL_GC_POP_PRESERVE_ROOT_OBJS() \ + jl_gcframe_t *curr = jl_p_gcpreserve_stack; \ + if(curr) { \ + (jl_p_gcpreserve_stack = jl_p_gcpreserve_stack->prev); \ + free(curr); \ + } + +// Add each argument as a tpin root object. +// However, we cannot use JL_GC_PUSH and JL_GC_POP since the slots should live +// beyond this function. Instead, we maintain a tpin stack by mallocing/freeing +// the frames for each of the preserve regions we encounter +JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT +{ + jl_value_t** frame; + JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(frame, n); + if (n == 0) return; + + va_list args; + va_start(args, n); + for (int i = 0; i < n; i++) { + frame[i] = va_arg(args, jl_value_t *); + } + va_end(args); +} + +JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT +{ + JL_GC_POP_PRESERVE_ROOT_OBJS(); +} + #ifdef __cplusplus } #endif diff --git a/src/gc-stock.c b/src/gc-stock.c index 278c53a685dc3..b9c0827f60e65 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -806,6 +806,12 @@ inline jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) return v; } +inline jl_value_t *jl_gc_alloc_nonmoving_(jl_ptls_t ptls, size_t sz, void *ty) +{ + // Just use the normal allocation, as the GC won't move objects anyway. + return jl_gc_alloc_(ptls, sz, ty); +} + int jl_gc_classify_pools(size_t sz, int *osize) { if (sz > GC_MAX_SZCLASS) @@ -3484,6 +3490,11 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) gc_cblist_pre_gc, (collection)); if (!jl_atomic_load_acquire(&jl_gc_disable_counter)) { + // This thread will yield. + // jl_gc_notify_thread_yield does nothing for the stock GC at the point, but it may be non empty in the future, + // and this is a place where we should call jl_gc_notify_thread_yield. + // TODO: This call can be removed if requested. + jl_gc_notify_thread_yield(ptls, NULL); JL_LOCK_NOGC(&finalizers_lock); // all the other threads are stopped, so this does not make sense, right? otherwise, failing that, this seems like plausibly a deadlock #ifndef __clang_gcanalyzer__ if (_jl_gc_collect(ptls, collection)) { @@ -4102,15 +4113,44 @@ JL_DLLEXPORT void jl_gc_schedule_foreign_sweepfunc(jl_ptls_t ptls, jl_value_t *o arraylist_push(&ptls->gc_tls.sweep_objs, obj); } +// added for MMTk integration + void jl_gc_notify_image_load(const char* img_data, size_t len) { // Do nothing } +void jl_gc_notify_image_alloc(const char* img_data, size_t len) +{ + // Do nothing +} + JL_DLLEXPORT const char* jl_gc_active_impl(void) { return "Built with stock GC"; } +JL_DLLEXPORT unsigned char jl_gc_pin_object(void* obj) { + return 0; +} + +JL_DLLEXPORT unsigned char jl_gc_pin_pointer(void* ptr) { + return 0; +} + +JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT +{ + jl_unreachable(); +} + +JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT +{ + jl_unreachable(); +} + +JL_DLLEXPORT void jl_gc_notify_thread_yield(jl_ptls_t ptls, void* ctx) { + // Do nothing before a thread yields +} + #ifdef __cplusplus } #endif diff --git a/src/gc-tls-mmtk.h b/src/gc-tls-mmtk.h index 5b69aef5d55fb..29cf30678babc 100644 --- a/src/gc-tls-mmtk.h +++ b/src/gc-tls-mmtk.h @@ -7,6 +7,9 @@ #include "mmtkMutator.h" #include "julia_atomics.h" +// VO bit is required to support conservative stack scanning and moving. +#define MMTK_NEEDS_VO_BIT (1) + #ifdef __cplusplus extern "C" { #endif @@ -14,6 +17,7 @@ extern "C" { typedef struct { MMTkMutatorContext mmtk_mutator; _Atomic(size_t) malloc_sz_since_last_poll; + ucontext_t ctx_at_the_time_gc_started; } jl_gc_tls_states_t; #ifdef __cplusplus diff --git a/src/genericmemory.c b/src/genericmemory.c index 35ef2b545a32d..2c3116b9d3467 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -41,6 +41,8 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory_unchecked(jl_ptls_t ptls m = (jl_genericmemory_t*)jl_gc_alloc(ptls, tot, mtype); if (pooled) { data = (char*)m + JL_SMALL_BYTE_ALIGNMENT; + // Data is inlined and ptr is an internal pointer. We pin the object so the ptr will not be invalid. + OBJ_PIN(m); } else { int isaligned = 1; // jl_gc_managed_malloc is always aligned @@ -111,6 +113,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str) m->length = jl_string_len(str); m->ptr = jl_string_data(str); jl_genericmemory_data_owner_field(m) = str; + OBJ_PIN(str); return m; } @@ -166,6 +169,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void m->length = nel; jl_genericmemory_data_owner_field(m) = own_buffer ? (jl_value_t*)m : NULL; if (own_buffer) { + OBJ_PIN(m); int isaligned = 0; // TODO: allow passing memalign'd buffers jl_gc_track_malloced_genericmemory(ct->ptls, m, isaligned); size_t allocated_bytes = memory_block_usable_size(data, isaligned); diff --git a/src/gf.c b/src/gf.c index a83587cfde13f..bc8ee029bfc1b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -632,7 +632,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( assert(min_world <= max_world && "attempting to set invalid world constraints"); //assert((!jl_is_method(mi->def.value) || max_world != ~(size_t)0 || min_world <= 1 || edges == NULL || jl_svec_len(edges) != 0) && "missing edges"); jl_task_t *ct = jl_current_task; - jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_gc_alloc(ct->ptls, sizeof(jl_code_instance_t), + jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_code_instance_t), jl_code_instance_type); codeinst->def = (jl_value_t*)mi; codeinst->owner = owner; diff --git a/src/interpreter.c b/src/interpreter.c index 8b1b005349d9d..9d5e7f01827c6 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -52,7 +52,16 @@ extern void JL_GC_ENABLEFRAME(interpreter_state*) JL_NOTSAFEPOINT; #else +#ifdef WITH_THIRD_PARTY_HEAP +#if WITH_THIRD_PARTY_HEAP == 1 // MMTk +#define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<3)|2) +// For roots that are not transitively pinned +#define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) ((((size_t)(n))<<3)|6) +#else #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<2)|2) +#define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) JL_GC_ENCODE_PUSHFRAME(n) +#endif +#endif #define JL_GC_PUSHFRAME(frame,locals,n) \ JL_CPPALLOCA(frame, sizeof(*frame)+(((n)+3)*sizeof(jl_value_t*))); \ diff --git a/src/ircode.c b/src/ircode.c index 65130e46edfe0..f19f2f5a990bd 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -1707,12 +1707,15 @@ void jl_init_serializer(void) assert(LAST_TAG+1+i < 256); for (i = 2; i < 256; i++) { - if (deser_tag[i]) + if (deser_tag[i]) { + OBJHASH_PIN(deser_tag[i]) ptrhash_put(&ser_tag, deser_tag[i], (void*)i); + } } i = 2; while (common_symbols[i-2] != NULL) { + OBJHASH_PIN(common_symbols[i-2]) ptrhash_put(&common_symbol_tag, common_symbols[i-2], (void*)i); deser_symbols[i] = (jl_value_t*)common_symbols[i-2]; i += 1; diff --git a/src/jitlayers.h b/src/jitlayers.h index 961adde887289..8094e1812c362 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -240,6 +240,7 @@ struct jl_codegen_params_t { // outputs jl_workqueue_t workqueue; SmallVector cfuncs; + // This map may hold Julia obj ref in the native heap. We need to pin the void*. std::map global_targets; jl_array_t *temporary_roots = nullptr; SmallSet temporary_roots_set; @@ -343,6 +344,7 @@ Constant *literal_pointer_val_slot(jl_codegen_params_t ¶ms, Module *M, jl_va static inline Constant *literal_static_pointer_val(const void *p, Type *T) JL_NOTSAFEPOINT { + PTR_PIN((void*)p); // This may point to non-mmtk heap memory. // this function will emit a static pointer into the generated code // the generated code will only be valid during the current session, // and thus, this should typically be avoided in new API's diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 64ba74722bdc9..1eb7dbe08b524 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -175,6 +175,8 @@ XX(jl_gc_set_max_memory) \ XX(jl_gc_sync_total_bytes) \ XX(jl_gc_total_hrtime) \ + XX(jl_gc_preserve_begin_hook) \ + XX(jl_gc_preserve_end_hook) \ XX(jl_gdblookup) \ XX(jl_generating_output) \ XX(jl_declare_const_gf) \ diff --git a/src/jl_uv.c b/src/jl_uv.c index 24da8629faf02..13ad74c957111 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -473,6 +473,7 @@ JL_DLLEXPORT void jl_forceclose_uv(uv_handle_t *handle) JL_DLLEXPORT void jl_uv_associate_julia_struct(uv_handle_t *handle, jl_value_t *data) { + OBJ_PIN(data); handle->data = data; } @@ -483,6 +484,7 @@ JL_DLLEXPORT void jl_uv_associate_julia_struct(uv_handle_t *handle, */ JL_DLLEXPORT void jl_uv_disassociate_julia_struct(uv_handle_t *handle) { + // TODO: unpin here -- we need to implement pin count before we can unpin objects. handle->data = NULL; } diff --git a/src/julia.h b/src/julia.h index 7d0900a761421..ab9cdb56a4e37 100644 --- a/src/julia.h +++ b/src/julia.h @@ -77,6 +77,13 @@ typedef struct _jl_tls_states_t *jl_ptls_t; // the common fields are hidden before the pointer, but the following macro is // used to indicate which types below are subtypes of jl_value_t #define JL_DATA_TYPE +// Objects of a type that is JL_NON_MOVING should be allocated with +// jl_gc_alloc_non_moving so they will never be moved by GC. +// Those types are usually frequently referenced by the runtime. +// It is basically a trade-off between allocating the objects as non-moving +// and pinning the objects after allocation. If objects of certain types are +// mostly likely to be pinned, it is a good idea to just allocate them as non moving. +#define JL_NON_MOVING typedef struct _jl_value_t jl_value_t; #include "julia_threads.h" @@ -84,6 +91,19 @@ typedef struct _jl_value_t jl_value_t; extern "C" { #endif +// object pinning ------------------------------------------------------------ + +// FIXME: Pinning objects that get hashed in the ptrhash table +// until we implement address space hashing. +#define OBJHASH_PIN(key) if (key) jl_gc_pin_object(key); +#define PTRHASH_PIN(key) if (key) jl_gc_pin_pointer(key); + +// Called when pinning objects that would cause an error if moved +// The difference: the argument for pin_object needs to pointer to an object (jl_value_t*), +// but the argument for pin_pointer can be an internal pointer. +#define OBJ_PIN(key) if (key) jl_gc_pin_object(key); +#define PTR_PIN(key) if (key) jl_gc_pin_pointer(key); + // core data types ------------------------------------------------------------ struct _jl_taggedvalue_bits { @@ -401,6 +421,7 @@ typedef struct _jl_method_t { // No lock is required to read these fields, set once on construction: // def, specTypes, sparam_vals struct _jl_method_instance_t { + JL_NON_MOVING // Non moving, as it is referenced in a map in JITDebugInfoRegistry JL_DATA_TYPE union { jl_value_t *value; // generic accessor @@ -442,6 +463,7 @@ typedef struct _jl_opaque_closure_t { // def, owner, rettype, exctype, rettype_const, analysis_results, // time_infer_total, time_infer_self typedef struct _jl_code_instance_t { + JL_NON_MOVING // Pin codeinst, as they are referenced by vectors and maps in _jl_codegen_params_t JL_DATA_TYPE jl_value_t *def; // MethodInstance or ABIOverride jl_value_t *owner; // Compiler token this belongs to, `jl_nothing` is reserved for native @@ -527,6 +549,7 @@ typedef struct { // of a type and storing all data common to different instantiations of the type, // including a cache for hash-consed allocation of DataType objects. typedef struct { + JL_NON_MOVING // Typenames should be pinned since they are used as metadata, and are read during scan_object JL_DATA_TYPE jl_sym_t *name; struct _jl_module_t *module; @@ -611,6 +634,7 @@ typedef struct { } jl_datatype_layout_t; typedef struct _jl_datatype_t { + JL_NON_MOVING // Types should not be moved. It is also referenced from the native heap in jl_raw_alloc_t. JL_DATA_TYPE jl_typename_t *name; struct _jl_datatype_t *super; @@ -821,6 +845,7 @@ typedef struct { // name, parent, file, line, build_id, uuid, nospecialize, optlevel, compile, // infer, iistopmod, max_methods typedef struct _jl_module_t { + JL_NON_MOVING // modules are referenced in jl_current_modules (htable). They cannot move. JL_DATA_TYPE jl_sym_t *name; struct _jl_module_t *parent; @@ -1168,9 +1193,46 @@ struct _jl_gcframe_t { #define jl_pgcstack (jl_current_task->gcstack) +#ifndef WITH_THIRD_PARTY_HEAP #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<2) #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<2)|1) #define JL_GC_DECODE_NROOTS(n) (n >> 2) +<<<<<<< HEAD +======= + +#define JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) JL_GC_ENCODE_PUSHARGS(n) +#define JL_GC_ENCODE_PUSH_NO_TPIN(n) JL_GC_ENCODE_PUSH(n) + +#else +#if WITH_THIRD_PARTY_HEAP == 1 // MMTk +// We use an extra bit (100) in the nroots value from the frame to indicate that the roots +// in the frame are/are not transitively pinning. +// There are currently 3 macros that encode passing nroots to the gcframe +// and they use the two lowest bits to encode information about what is in the frame (as below). +// To support the distinction between transtively pinning roots and non transitively pinning roots +// on the stack, we take another bit from nroots to encode information about whether or not to +// transitively pin the roots in the frame. +// +// So the ones that transitively pin look like: +// #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<3) +// #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<3)|1) +// #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<3)|2) +// and the ones that do not look like: +// #define JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) (((size_t)(n))<<3|4) +// #define JL_GC_ENCODE_PUSH_NO_TPIN(n) ((((size_t)(n))<<3)|5) +// #define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) ((((size_t)(n))<<3)|6) + +// these are transitively pinning +#define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<3) +#define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<3)|1) +#define JL_GC_DECODE_NROOTS(n) (n >> 3) + +// these only pin the root object itself +#define JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) (((size_t)(n))<<3|4) +#define JL_GC_ENCODE_PUSH_NO_TPIN(n) ((((size_t)(n))<<3)|5) +#endif +#endif +>>>>>>> b4d216cfeb (Support moving) #ifdef __clang_gcanalyzer__ diff --git a/src/julia_internal.h b/src/julia_internal.h index e93dae8312d51..0bd974f7a67bf 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -577,12 +577,18 @@ static_assert(ARRAY_CACHE_ALIGN_THRESHOLD > GC_MAX_SZCLASS, ""); * safepoints will be caught by the GC analyzer. */ JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty); +JL_DLLEXPORT jl_value_t *jl_gc_alloc_nonmoving(jl_ptls_t ptls, size_t sz, void *ty); + // On GCC, only inline when sz is constant #ifdef __GNUC__ # define jl_gc_alloc(ptls, sz, ty) \ (__builtin_constant_p(sz) ? \ jl_gc_alloc_(ptls, sz, ty) : \ (jl_gc_alloc)(ptls, sz, ty)) +# define jl_gc_alloc_nonmoving(ptls, sz, ty) \ + (__builtin_constant_p(sz) ? \ + jl_gc_alloc_nonmoving_(ptls, sz, ty) : \ + (jl_gc_alloc_nonmoving)(ptls, sz, ty)) #else # define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty) #endif diff --git a/src/julia_threads.h b/src/julia_threads.h index dbe9166f288a9..4b09dbf1dae4a 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -235,6 +235,7 @@ typedef struct _jl_excstack_t jl_excstack_t; typedef struct _jl_handler_t jl_handler_t; typedef struct _jl_task_t { + JL_NON_MOVING // jl_mutex_t (as globals) references tasks JL_DATA_TYPE jl_value_t *next; // invasive linked list for scheduler jl_value_t *queue; // invasive linked list for scheduler @@ -279,6 +280,9 @@ typedef struct _jl_task_t { // uint48_t padding2_64; // saved gc stack top for context switches jl_gcframe_t *gcstack; + // GC stack of objects from gc preserve regions + // These must always be transitively pinned. Only used by MMTK. + jl_gcframe_t *gcpreserve_stack; size_t world_age; // quick lookup for current ptls jl_ptls_t ptls; // == jl_all_tls_states[tid] diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 7d3a233c0a720..b98b3dc741ebe 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -42,6 +42,8 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F) IRBuilder<> builder(target); StoreInst *inst = builder.CreateAlignedStore( + // FIXME: We should use JL_GC_ENCODE_PUSHARGS_NO_TPIN here. + // We need to make sure things are properly pinned before turning this into a non TPIN push. ConstantInt::get(T_size, JL_GC_ENCODE_PUSHARGS(nRoots)), builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0, "frame.nroots"),// GEP of 0 becomes a noop and eats the name Align(sizeof(void*))); diff --git a/src/llvm-gc-interface-passes.h b/src/llvm-gc-interface-passes.h index 0d21ea0a66cd8..bd327ff8cc3d3 100644 --- a/src/llvm-gc-interface-passes.h +++ b/src/llvm-gc-interface-passes.h @@ -361,6 +361,7 @@ struct LateLowerGCFrame: private JuliaPassContext { void PlaceGCFrameReset(State &S, unsigned R, unsigned MinColorRoot, ArrayRef Colors, Value *GCFrame, Instruction *InsertBefore); void PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAssignedColors, State &S, std::map>); void CleanupWriteBarriers(Function &F, State *S, const SmallVector &WriteBarriers, bool *CFGModified); + void CleanupGCPreserve(Function &F, CallInst *CI, Value *callee, Type *T_size); bool CleanupIR(Function &F, State *S, bool *CFGModified); void NoteUseChain(State &S, BBState &BBS, User *TheUser); SmallVector GetPHIRefinements(PHINode *phi, State &S); @@ -416,4 +417,12 @@ struct FinalLowerGC: private JuliaPassContext { bool shouldRunFinalGC(); }; +inline bool isSpecialPtr(Type *Ty) { + PointerType *PTy = dyn_cast(Ty); + if (!PTy) + return false; + unsigned AS = PTy->getAddressSpace(); + return AddressSpace::FirstSpecial <= AS && AS <= AddressSpace::LastSpecial; +} + #endif // LLVM_GC_PASSES_H diff --git a/src/llvm-late-gc-lowering-mmtk.cpp b/src/llvm-late-gc-lowering-mmtk.cpp index 5539c8dbcf153..e3f83be1f9381 100644 --- a/src/llvm-late-gc-lowering-mmtk.cpp +++ b/src/llvm-late-gc-lowering-mmtk.cpp @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "llvm-gc-interface-passes.h" +#include "mmtk.h" Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) { @@ -83,6 +84,31 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t))); auto v_as_ptr = builder.CreateIntToPtr(v_raw, smallAllocFunc->getReturnType()); + + // Post alloc + if (MMTK_NEEDS_VO_BIT) { + auto intptr_ty = Type::getInt64Ty(target->getContext()); + auto i8_ty = Type::getInt8Ty(F.getContext()); + intptr_t metadata_base_address = reinterpret_cast(MMTK_SIDE_VO_BIT_BASE_ADDRESS); + auto metadata_base_val = ConstantInt::get(intptr_ty, metadata_base_address); + auto metadata_base_ptr = ConstantExpr::getIntToPtr(metadata_base_val, PointerType::get(i8_ty, 0)); + // intptr_t addr = (intptr_t) v; + auto addr = v_raw; + // uint8_t* vo_meta_addr = (uint8_t*) (MMTK_SIDE_VO_BIT_BASE_ADDRESS) + (addr >> 6); + auto shr = builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 6)); + auto metadata_ptr = builder.CreateGEP(i8_ty, metadata_base_ptr, shr); + // intptr_t shift = (addr >> 3) & 0b111; + auto shift = builder.CreateAnd(builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 3)), ConstantInt::get(intptr_ty, 7)); + // uint8_t byte_val = *vo_meta_addr; + auto byte_val = builder.CreateAlignedLoad(i8_ty, metadata_ptr, Align()); + // uint8_t new_val = byte_val | (1 << shift); + auto shifted_val = builder.CreateShl(ConstantInt::get(intptr_ty, 1), shift); + auto shifted_val_i8 = builder.CreateTruncOrBitCast(shifted_val, i8_ty); + auto new_val = builder.CreateOr(byte_val, shifted_val_i8); + // (*vo_meta_addr) = new_val; + builder.CreateStore(new_val, metadata_ptr); + } + builder.CreateBr(next_instr->getParent()); phiNode->addIncoming(new_call, slowpath); @@ -94,3 +120,49 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) } return target; } + +void LateLowerGCFrame::CleanupGCPreserve(Function &F, CallInst *CI, Value *callee, Type *T_size) { + if (callee == gc_preserve_begin_func) { + // Initialize an IR builder. + IRBuilder<> builder(CI); + + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + size_t nargs = 0; + State S2(F); + + std::vector args; + for (Use &U : CI->args()) { + Value *V = U; + if (isa(V)) + continue; + if (isa(V->getType())) { + if (isSpecialPtr(V->getType())) { + int Num = Number(S2, V); + if (Num >= 0) { + nargs++; + Value *Val = GetPtrForNumber(S2, Num, CI); + args.push_back(Val); + } + } + } else { + auto Nums = NumberAll(S2, V); + for (int Num : Nums) { + if (Num < 0) + continue; + Value *Val = GetPtrForNumber(S2, Num, CI); + args.push_back(Val); + nargs++; + } + } + } + args.insert(args.begin(), ConstantInt::get(T_size, nargs)); + + ArrayRef args_llvm = ArrayRef(args); + builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveBeginHook), args_llvm ); + } else if (callee == gc_preserve_end_func) { + // Initialize an IR builder. + IRBuilder<> builder(CI); + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveEndHook), {}); + } +} diff --git a/src/llvm-late-gc-lowering-stock.cpp b/src/llvm-late-gc-lowering-stock.cpp index 2a11487773396..b6e517c52f0a3 100644 --- a/src/llvm-late-gc-lowering-stock.cpp +++ b/src/llvm-late-gc-lowering-stock.cpp @@ -7,3 +7,7 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) // Do nothing for the stock GC return target; } + +void LateLowerGCFrame::CleanupGCPreserve(Function &F, CallInst *CI, Value *callee, Type *T_size) { + // Do nothing for the stock GC +} \ No newline at end of file diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index d4e11afcfe2e1..2e86d8e3ab824 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -15,14 +15,6 @@ static bool isTrackedValue(Value *V) { return PT && PT->getAddressSpace() == AddressSpace::Tracked; } -static bool isSpecialPtr(Type *Ty) { - PointerType *PTy = dyn_cast(Ty); - if (!PTy) - return false; - unsigned AS = PTy->getAddressSpace(); - return AddressSpace::FirstSpecial <= AS && AS <= AddressSpace::LastSpecial; -} - // return how many Special pointers are in T (count > 0), // and if there is anything else in T (all == false) CountTrackedPointers::CountTrackedPointers(Type *T, bool ignore_loaded) { @@ -2064,9 +2056,11 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { continue; } Value *callee = CI->getCalledOperand(); - if (callee && (callee == gc_flush_func || callee == gc_preserve_begin_func - || callee == gc_preserve_end_func)) { + if (callee && callee == gc_flush_func) { /* No replacement */ + } else if (callee && (callee == gc_preserve_begin_func + || callee == gc_preserve_end_func)) { + CleanupGCPreserve(F, CI, callee, T_size); } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) { auto *obj = CI->getOperand(0); #if JL_LLVM_VERSION >= 200000 diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index a6a16a7f4956c..1278a5475adbe 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -262,6 +262,8 @@ namespace jl_well_known { static const char *GC_SMALL_ALLOC_NAME = XSTR(jl_gc_small_alloc); static const char *GC_QUEUE_ROOT_NAME = XSTR(jl_gc_queue_root); static const char *GC_ALLOC_TYPED_NAME = XSTR(jl_gc_alloc_typed); + static const char *GC_PRESERVE_BEGIN_HOOK_NAME = XSTR(jl_gc_preserve_begin_hook); + static const char *GC_PRESERVE_END_HOOK_NAME = XSTR(jl_gc_preserve_end_hook); using jl_intrinsics::addGCAllocAttributes; @@ -330,4 +332,50 @@ namespace jl_well_known { allocTypedFunc->addFnAttr(Attribute::getWithAllocSizeArgs(ctx, 1, None)); return addGCAllocAttributes(allocTypedFunc); }); + + const WellKnownFunctionDescription GCPreserveBeginHook( + GC_PRESERVE_BEGIN_HOOK_NAME, + [](Type *T_size) { + auto &ctx = T_size->getContext(); + auto func = Function::Create( + FunctionType::get( + Type::getVoidTy(ctx), + { T_size }, + true), + Function::ExternalLinkage, + GC_PRESERVE_BEGIN_HOOK_NAME); + +#if JL_LLVM_VERSION >= 160000 + func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly()); +#else + func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); +#endif + return func; + }); + + const WellKnownFunctionDescription GCPreserveEndHook( + GC_PRESERVE_END_HOOK_NAME, + [](Type *T_size) { + auto &ctx = T_size->getContext(); + auto func = Function::Create( + FunctionType::get( + Type::getVoidTy(ctx), + { }, + false), + Function::ExternalLinkage, + GC_PRESERVE_END_HOOK_NAME); +#if JL_LLVM_VERSION >= 160000 + func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly()); +#else + func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); +#endif + return func; + }); +} + +void setName(llvm::Value *V, const llvm::Twine &Name, int debug_info) +{ + if (debug_info >= 2 && !llvm::isa(V)) { + V->setName(Name); + } } diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index d79470818c287..fe81b316ca6ef 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -156,6 +156,12 @@ namespace jl_well_known { // `jl_gc_alloc_typed`: allocates bytes. extern const WellKnownFunctionDescription GCAllocTyped; + + // `jl_gc_preserve_begin_hook`: called at the beginning of gc preserve regions, if required + extern const WellKnownFunctionDescription GCPreserveBeginHook; + + // `jl_gc_preserve_end_hook`: called at the end of gc preserve regions, if required + extern const WellKnownFunctionDescription GCPreserveEndHook; } void setName(llvm::Value *V, const llvm::Twine &Name, int debug_info); diff --git a/src/method.c b/src/method.c index 459c8088ea0f1..5aaf221cc38a6 100644 --- a/src/method.c +++ b/src/method.c @@ -600,7 +600,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void) { jl_task_t *ct = jl_current_task; jl_method_instance_t *mi = - (jl_method_instance_t*)jl_gc_alloc(ct->ptls, sizeof(jl_method_instance_t), + (jl_method_instance_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_method_instance_t), jl_method_instance_type); mi->def.value = NULL; mi->specTypes = NULL; diff --git a/src/module.c b/src/module.c index d7ba12609ee80..7e542590327ae 100644 --- a/src/module.c +++ b/src/module.c @@ -493,7 +493,7 @@ static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) { jl_task_t *ct = jl_current_task; const jl_uuid_t uuid_zero = {0, 0}; - jl_module_t *m = (jl_module_t*)jl_gc_alloc(ct->ptls, sizeof(jl_module_t), + jl_module_t *m = (jl_module_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_module_t), jl_module_type); jl_set_typetagof(m, jl_module_tag, 0); assert(jl_is_symbol(name)); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index a653b027a861a..2efa8bb001903 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -320,6 +320,8 @@ jl_value_t *jl_get_cfunction_trampoline( tramp = trampoline_alloc(); ((void**)result)[0] = tramp; init_trampoline(tramp, nval); + OBJHASH_PIN((void*)fobj) + OBJHASH_PIN(result) ptrhash_put(cache, (void*)fobj, result); uv_mutex_unlock(&trampoline_lock); return result; diff --git a/src/signals-unix.c b/src/signals-unix.c index 2db397050420e..dca53efb40474 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -410,6 +410,8 @@ JL_NO_ASAN static void segv_handler(int sig, siginfo_t *info, void *context) return; } if (sig == SIGSEGV && info->si_code == SEGV_ACCERR && jl_addr_is_safepoint((uintptr_t)info->si_addr) && !is_write_fault(context)) { + // TODO: We should do the same for other platforms + jl_gc_notify_thread_yield(ct->ptls, context); jl_set_gc_and_wait(ct); // Do not raise sigint on worker thread if (jl_atomic_load_relaxed(&ct->tid) != 0) diff --git a/src/staticdata.c b/src/staticdata.c index be1188572f3b0..d70c8c04620da 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -664,8 +664,11 @@ static int needs_uniquing(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFE static void record_field_change(jl_value_t **addr, jl_value_t *newval) JL_NOTSAFEPOINT { - if (*addr != newval) + if (*addr != newval){ + OBJHASH_PIN((void*)addr) + OBJHASH_PIN((void*)newval) ptrhash_put(&field_replace, (void*)addr, newval); + } } static jl_value_t *get_replaceable_field(jl_value_t **addr, int mutabl) JL_GC_DISABLED @@ -2569,6 +2572,7 @@ static jl_svec_t *jl_prune_type_cache_hash(jl_svec_t *cache) JL_GC_DISABLED assert(serialization_queue.items[from_seroder_entry(idx)] == cache); cache = cache_rehash_set(cache, sz); // redirect all references to the old cache to relocate to the new cache object + OBJHASH_PIN((void*)cache) ptrhash_put(&serialization_order, cache, idx); serialization_queue.items[from_seroder_entry(idx)] = cache; return cache; @@ -3617,6 +3621,7 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) ios_seek_end(&f); size_t len = ios_pos(&f); char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + jl_gc_notify_image_alloc(sysimg, len); ios_seek(&f, 0); if (ios_readall(&f, sysimg, len) != len) @@ -4040,6 +4045,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, assert(tag == 0); arraylist_push(&delay_list, obj); arraylist_push(&delay_list, pfld); + OBJHASH_PIN(obj) ptrhash_put(&new_dt_objs, (void*)obj, obj); // mark obj as invalid *pfld = (uintptr_t)NULL; continue; @@ -4074,6 +4080,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, } static_assert(offsetof(jl_datatype_t, name) == 0, ""); newdt->name = dt->name; + OBJHASH_PIN(newdt) + OBJHASH_PIN(dt) ptrhash_put(&new_dt_objs, (void*)newdt, dt); } else { @@ -4411,9 +4419,10 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im char *sysimg; int success = !needs_permalloc; ios_seek(f, datastartpos); - if (needs_permalloc) + if (needs_permalloc) { sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - else + jl_gc_notify_image_alloc(sysimg, len); + } else sysimg = &f->buf[f->bpos]; if (needs_permalloc) success = ios_readall(f, sysimg, len) == len; diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 69b60a4314a5a..c3d4ace606f9b 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -82,7 +82,7 @@ static uint64_t jl_worklist_key(jl_array_t *worklist) JL_NOTSAFEPOINT return 0; } -static jl_array_t *newly_inferred JL_GLOBALLY_ROOTED /*FIXME*/; +jl_array_t *newly_inferred JL_GLOBALLY_ROOTED /*FIXME*/; // Mutex for newly_inferred jl_mutex_t newly_inferred_mutex; extern jl_mutex_t world_counter_lock; diff --git a/src/task.c b/src/task.c index ffef3441be62f..01d91cc20a220 100644 --- a/src/task.c +++ b/src/task.c @@ -311,7 +311,7 @@ CFI_NORETURN #endif /* Rooted by the base module */ -static _Atomic(jl_function_t*) task_done_hook_func JL_GLOBALLY_ROOTED = NULL; +_Atomic(jl_function_t*) task_done_hook_func JL_GLOBALLY_ROOTED = NULL; void JL_NORETURN jl_finish_task(jl_task_t *ct) { @@ -1076,7 +1076,7 @@ void jl_rng_split(uint64_t dst[JL_RNG_SIZE], uint64_t src[JL_RNG_SIZE]) JL_NOTSA JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion_future, size_t ssize) { jl_task_t *ct = jl_current_task; - jl_task_t *t = (jl_task_t*)jl_gc_alloc(ct->ptls, sizeof(jl_task_t), jl_task_type); + jl_task_t *t = (jl_task_t*)jl_gc_alloc_nonmoving(ct->ptls, sizeof(jl_task_t), jl_task_type); jl_set_typetagof(t, jl_task_tag, 0); JL_PROBE_RT_NEW_TASK(ct, t); t->ctx.copy_stack = 0; @@ -1109,6 +1109,12 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion t->start = start; t->result = jl_nothing; t->donenotify = completion_future; + // completion_future is a GenericCondition with SpinLock. + // I am not sure why we have to pin this. But, if we don't pin it, + // it may get moved, and we still use the invalid old reference somehow. + // See https://github.com/mmtk/mmtk-julia/issues/179. + // TODO: We should understand where we get the invalid reference from. + OBJ_PIN(completion_future); jl_atomic_store_relaxed(&t->_isexception, 0); // Inherit scope from parent task t->scope = ct->scope; @@ -1540,7 +1546,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi) bootstrap_task.value.ptls = ptls; if (jl_nothing == NULL) // make a placeholder jl_nothing = jl_gc_permobj(0, jl_nothing_type, 0); - jl_task_t *ct = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), jl_task_type); + jl_task_t *ct = (jl_task_t*)jl_gc_alloc_nonmoving(ptls, sizeof(jl_task_t), jl_task_type); jl_set_typetagof(ct, jl_task_tag, 0); memset(ct, 0, sizeof(jl_task_t)); void *stack = stack_lo; diff --git a/src/toplevel.c b/src/toplevel.c index ff5aabec748ec..9c19164d37812 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -137,6 +137,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex jl_value_t *form = (jl_value_t*)newm; JL_GC_PUSH1(&form); JL_LOCK(&jl_modules_mutex); + OBJHASH_PIN(newm) ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1)); JL_UNLOCK(&jl_modules_mutex); // copy parent environment info into submodule From d1fabab6df48ec1009baf568e8bdfb6baedbe0fc Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 3 Apr 2025 00:08:28 +0000 Subject: [PATCH 627/662] Minor fix for the macro --- src/interpreter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpreter.c b/src/interpreter.c index 9d5e7f01827c6..708955360350f 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -57,11 +57,11 @@ extern void JL_GC_ENABLEFRAME(interpreter_state*) JL_NOTSAFEPOINT; #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<3)|2) // For roots that are not transitively pinned #define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) ((((size_t)(n))<<3)|6) +#endif // endif MMTk #else #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<2)|2) #define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) JL_GC_ENCODE_PUSHFRAME(n) #endif -#endif #define JL_GC_PUSHFRAME(frame,locals,n) \ JL_CPPALLOCA(frame, sizeof(*frame)+(((n)+3)*sizeof(jl_value_t*))); \ From 983d2c471001eecf0cdfd08544dd0bd2d4f46649 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 3 Apr 2025 02:32:10 +0000 Subject: [PATCH 628/662] jl_pinned_ref example in CPP --- src/aotcompile.cpp | 6 +++--- src/cgutils.cpp | 3 +-- src/jitlayers.cpp | 6 +++--- src/jitlayers.h | 2 +- src/julia.h | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 8bee81264a3a7..91c4dfddac169 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -377,7 +377,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo { for (size_t i = 0; i < jl_array_dim0(params.temporary_roots); i++) { jl_value_t *val = jl_array_ptr_ref(params.temporary_roots, i); - auto ref = params.global_targets.find((void*)val); + auto ref = params.global_targets.find(jl_pinned_ref_assume(void, val)); if (ref == params.global_targets.end()) continue; auto get_global_root = [val, &method_roots]() { @@ -392,7 +392,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo if (mval != val) { GlobalVariable *GV = ref->second; params.global_targets.erase(ref); - auto mref = params.global_targets.find((void*)mval); + auto mref = params.global_targets.find(jl_pinned_ref_assume(void, mval)); if (mref != params.global_targets.end()) { // replace ref with mref in all Modules std::string OldName(GV->getName()); @@ -416,7 +416,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo assert(GV == nullptr); } else { - params.global_targets[(void*)mval] = GV; + params.global_targets[jl_pinned_ref_create(void, mval)] = GV; } } } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9d70d4bbda53a..5175c9d56731f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -397,8 +397,7 @@ static Constant *julia_pgv(jl_codegen_params_t ¶ms, Module *M, const char *c // emit a GlobalVariable for a jl_value_t named "cname" // store the name given so we can reuse it (facilitating merging later) // so first see if there already is a GlobalVariable for this address - OBJ_PIN(addr); // This will be stored in the native heap. We need to pin it. - GlobalVariable* &gv = params.global_targets[addr]; + GlobalVariable* &gv = params.global_targets[jl_pinned_ref_create(void, addr)]; StringRef localname; std::string gvname; if (!gv) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 7eee739f4f550..1ade8581be2f0 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -183,7 +183,7 @@ static void jl_optimize_roots(jl_codegen_params_t ¶ms, jl_method_instance_t JL_LOCK(&m->writelock); for (size_t i = 0; i < jl_array_dim0(params.temporary_roots); i++) { jl_value_t *val = jl_array_ptr_ref(params.temporary_roots, i); - auto ref = params.global_targets.find((void*)val); + auto ref = params.global_targets.find(jl_pinned_ref_assume(void, val)); if (ref == params.global_targets.end()) continue; auto get_global_root = [val, m]() { @@ -204,13 +204,13 @@ static void jl_optimize_roots(jl_codegen_params_t ¶ms, jl_method_instance_t if (mval != val) { GlobalVariable *GV = ref->second; params.global_targets.erase(ref); - auto mref = params.global_targets.find((void*)mval); + auto mref = params.global_targets.find(jl_pinned_ref_assume(void, mval)); if (mref != params.global_targets.end()) { GV->replaceAllUsesWith(mref->second); GV->eraseFromParent(); } else { - params.global_targets[(void*)mval] = GV; + params.global_targets[jl_pinned_ref_create(void, mval)] = GV; } } } diff --git a/src/jitlayers.h b/src/jitlayers.h index 8094e1812c362..d8f41d13b6910 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -241,7 +241,7 @@ struct jl_codegen_params_t { jl_workqueue_t workqueue; SmallVector cfuncs; // This map may hold Julia obj ref in the native heap. We need to pin the void*. - std::map global_targets; + std::map global_targets; jl_array_t *temporary_roots = nullptr; SmallSet temporary_roots_set; std::map, GlobalVariable*> external_fns; diff --git a/src/julia.h b/src/julia.h index ab9cdb56a4e37..d95c6de2fed60 100644 --- a/src/julia.h +++ b/src/julia.h @@ -104,6 +104,45 @@ extern "C" { #define OBJ_PIN(key) if (key) jl_gc_pin_object(key); #define PTR_PIN(key) if (key) jl_gc_pin_pointer(key); +#ifdef __cplusplus +} // extern "C" +#endif + +#ifdef __cplusplus + +// C++ template version +template +class pinned_ref { + T* ptr; +public: + explicit pinned_ref(void* p) : ptr(static_cast(p)) {} + operator void*() const { return ptr; } + T* get() const { return ptr; } + static pinned_ref create(void* p) { OBJ_PIN(p); return pinned_ref(p); } + static pinned_ref assume(void* p) { return pinned_ref(p); } +}; + +// Redefine macros for C++ to use the template version +#define jl_pinned_ref(T) pinned_ref +#define jl_pinned_ref_assume(T, ptr) pinned_ref::assume(ptr) +#define jl_pinned_ref_create(T, ptr) pinned_ref::create(ptr) +#define jl_pinned_ref_get(ref) (ref).get() + +#else + +// Primary type definition +#define jl_pinned_ref(T) union { T* t; void* unused; } +#define jl_pinned_ref_assume(T, ptr) ((jl_pinned_ref(T)){ .t = (ptr) }) +// Creation macro +#define jl_pinned_ref_create(T, ptr) OBJ_PIN(ptr); jl_pinned_ref_assume(T, ptr) +// Getter macro +#define jl_pinned_ref_get(ref) ((ref).t) + +#endif + +#ifdef __cplusplus +extern "C" { +#endif // core data types ------------------------------------------------------------ struct _jl_taggedvalue_bits { From 5c5b378ac9abc0124a495251a86174a490cc050b Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 9 Apr 2025 06:52:46 +0000 Subject: [PATCH 629/662] Changing C macro; Making jl_bt_element_t safe by wrapping jl_value_t* in jl_pinned_ref type --- src/interpreter.c | 5 +++-- src/julia.h | 4 ++-- src/julia_internal.h | 6 +++--- src/rtutils.c | 2 +- src/signal-handling.c | 2 +- src/signals-unix.c | 2 +- src/task.c | 2 +- src/threading.h | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/interpreter.c b/src/interpreter.c index 708955360350f..38ae990f93540 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -945,13 +945,14 @@ JL_DLLEXPORT size_t jl_capture_interp_frame(jl_bt_element_t *bt_entry, uintptr_t entry_tags = jl_bt_entry_descriptor(njlvalues, 0, JL_BT_INTERP_FRAME_TAG, s->ip); bt_entry[0].uintptr = JL_BT_NON_PTR_ENTRY; bt_entry[1].uintptr = entry_tags; - bt_entry[2].jlvalue = s->ci ? (jl_value_t*)s->ci : + jl_value_t* result = s->ci ? (jl_value_t*)s->ci : s->mi ? (jl_value_t*)s->mi : s->src ? (jl_value_t*)s->src : (jl_value_t*)jl_nothing; + jl_pinned_ref_set(bt_entry[2].jlvalue, result); if (need_module) { // If we only have a CodeInfo (s->src), we are in a top level thunk and // need to record the module separately. - bt_entry[3].jlvalue = (jl_value_t*)s->module; + jl_pinned_ref_set(bt_entry[3].jlvalue, (jl_value_t*)s->module); } return required_space; } diff --git a/src/julia.h b/src/julia.h index d95c6de2fed60..3eb495ee5be83 100644 --- a/src/julia.h +++ b/src/julia.h @@ -133,8 +133,8 @@ class pinned_ref { // Primary type definition #define jl_pinned_ref(T) union { T* t; void* unused; } #define jl_pinned_ref_assume(T, ptr) ((jl_pinned_ref(T)){ .t = (ptr) }) -// Creation macro -#define jl_pinned_ref_create(T, ptr) OBJ_PIN(ptr); jl_pinned_ref_assume(T, ptr) +// Assignment macro +#define jl_pinned_ref_set(lhs, ptr) OBJ_PIN(ptr); jl_pinned_ref_get(lhs) = ptr; // Getter macro #define jl_pinned_ref_get(ref) ((ref).t) diff --git a/src/julia_internal.h b/src/julia_internal.h index 0bd974f7a67bf..d3eca3f2a5029 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1395,7 +1395,7 @@ void jl_ctor_def(jl_value_t *ty, jl_value_t *functionloc); typedef struct _jl_bt_element_t { union { uintptr_t uintptr; // Metadata or native instruction ptr - jl_value_t* jlvalue; // Pointer to GC-managed value + jl_pinned_ref(jl_value_t) jlvalue; // Pointer to GC-managed value }; } jl_bt_element_t; @@ -1444,7 +1444,7 @@ STATIC_INLINE uintptr_t jl_bt_entry_header(jl_bt_element_t *bt_entry) JL_NOTSAFE // The returned value is rooted for the lifetime of the parent exception stack. STATIC_INLINE jl_value_t *jl_bt_entry_jlvalue(jl_bt_element_t *bt_entry, size_t i) JL_NOTSAFEPOINT { - return bt_entry[2 + i].jlvalue; + return jl_pinned_ref_get(bt_entry[2 + i].jlvalue); } #define JL_BT_INTERP_FRAME_TAG 1 // An interpreter frame @@ -1563,7 +1563,7 @@ STATIC_INLINE jl_bt_element_t *jl_excstack_raw(jl_excstack_t *stack) JL_NOTSAFEP STATIC_INLINE jl_value_t *jl_excstack_exception(jl_excstack_t *stack JL_PROPAGATES_ROOT, size_t itr) JL_NOTSAFEPOINT { - return jl_excstack_raw(stack)[itr-1].jlvalue; + return jl_pinned_ref_get(jl_excstack_raw(stack)[itr-1].jlvalue); } STATIC_INLINE size_t jl_excstack_bt_size(jl_excstack_t *stack, size_t itr) JL_NOTSAFEPOINT { diff --git a/src/rtutils.c b/src/rtutils.c index 2976e56b4195d..98cea3b6305cd 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -397,7 +397,7 @@ void jl_push_excstack(jl_task_t *ct, jl_excstack_t **stack JL_REQUIRE_ROOTED_SLO memcpy(rawstack + s->top, bt_data, sizeof(jl_bt_element_t)*bt_size); s->top += bt_size + 2; rawstack[s->top-2].uintptr = bt_size; - rawstack[s->top-1].jlvalue = exception; + jl_pinned_ref_set(rawstack[s->top-1].jlvalue, exception); } // conversion ----------------------------------------------------------------- diff --git a/src/signal-handling.c b/src/signal-handling.c index 70299104d751e..f65be903c2d5b 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -295,7 +295,7 @@ void jl_profile_task(void) profile_bt_data_prof[profile_bt_size_cur++].uintptr = (uintptr_t)r.tid + 1; // store task id (never null) - profile_bt_data_prof[profile_bt_size_cur++].jlvalue = (jl_value_t*)t; + jl_pinned_ref_set(profile_bt_data_prof[profile_bt_size_cur++].jlvalue, (jl_value_t*)t); // store cpu cycle clock profile_bt_data_prof[profile_bt_size_cur++].uintptr = cycleclock(); diff --git a/src/signals-unix.c b/src/signals-unix.c index dca53efb40474..130394a903816 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -908,7 +908,7 @@ static void do_profile(void *ctx) profile_bt_data_prof[profile_bt_size_cur++].uintptr = ptls2->tid + 1; // store task id (never null) - profile_bt_data_prof[profile_bt_size_cur++].jlvalue = (jl_value_t*)jl_atomic_load_relaxed(&ptls2->current_task); + jl_pinned_ref_set(profile_bt_data_prof[profile_bt_size_cur++].jlvalue, (jl_value_t*)jl_atomic_load_relaxed(&ptls2->current_task)); // store cpu cycle clock profile_bt_data_prof[profile_bt_size_cur++].uintptr = cycleclock(); diff --git a/src/task.c b/src/task.c index 01d91cc20a220..6d48380e14bf8 100644 --- a/src/task.c +++ b/src/task.c @@ -817,7 +817,7 @@ JL_DLLEXPORT void jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED) if (!excstack || excstack->top == 0) jl_error("rethrow(exc) not allowed outside a catch block"); // overwrite exception on top of stack. see jl_excstack_exception - jl_excstack_raw(excstack)[excstack->top-1].jlvalue = e; + jl_pinned_ref_set(jl_excstack_raw(excstack)[excstack->top-1].jlvalue, e); JL_GC_PROMISE_ROOTED(e); throw_internal(ct, NULL); } diff --git a/src/threading.h b/src/threading.h index cb26537699713..b908cd83fec32 100644 --- a/src/threading.h +++ b/src/threading.h @@ -19,7 +19,7 @@ extern _Atomic(jl_ptls_t*) jl_all_tls_states JL_GLOBALLY_ROOTED; /* thread local typedef struct _jl_threadarg_t { int16_t tid; uv_barrier_t *barrier; - void *arg; + void* arg; // can this be a heap object? } jl_threadarg_t; // each thread must initialize its TLS From dd609cdde66421bcea10c6a2d0d93dbad4be662e Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 14 Apr 2025 02:19:15 +0000 Subject: [PATCH 630/662] Wrapping types from jl_codectx_t with jl_pinned_ref --- src/ccall.cpp | 10 ++++---- src/codegen.cpp | 66 ++++++++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index d9d25c7a939f0..947653e3cb3b0 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -734,15 +734,15 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar jl_value_t *ir_arg = args[1]; JL_GC_PUSH4(&ir, &rt, &at, &entry); if (jl_is_ssavalue(ir_arg)) - ir_arg = jl_array_ptr_ref((jl_array_t*)ctx.source->code, ((jl_ssavalue_t*)ir_arg)->id - 1); + ir_arg = jl_array_ptr_ref((jl_array_t*)(jl_pinned_ref_get(ctx.source)->code), ((jl_ssavalue_t*)ir_arg)->id - 1); ir = static_eval(ctx, ir_arg); if (!ir) { emit_error(ctx, "error statically evaluating llvm IR argument"); JL_GC_POP(); return jl_cgval_t(); } - if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) { - jl_value_t *rtt = jl_array_ptr_ref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[2])->id - 1); + if (jl_is_ssavalue(args[2]) && !jl_is_long(jl_pinned_ref_get(ctx.source)->ssavaluetypes)) { + jl_value_t *rtt = jl_array_ptr_ref((jl_array_t*)jl_pinned_ref_get(ctx.source)->ssavaluetypes, ((jl_ssavalue_t*)args[2])->id - 1); if (jl_is_type_type(rtt)) rt = jl_tparam0(rtt); } @@ -754,8 +754,8 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar return jl_cgval_t(); } } - if (jl_is_ssavalue(args[3]) && !jl_is_long(ctx.source->ssavaluetypes)) { - jl_value_t *att = jl_array_ptr_ref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[3])->id - 1); + if (jl_is_ssavalue(args[3]) && !jl_is_long(jl_pinned_ref_get(ctx.source)->ssavaluetypes)) { + jl_value_t *att = jl_array_ptr_ref((jl_array_t*)jl_pinned_ref_get(ctx.source)->ssavaluetypes, ((jl_ssavalue_t*)args[3])->id - 1); if (jl_is_type_type(att)) at = jl_tparam0(att); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 4d7606c25e21f..a290acd25c101 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1881,10 +1881,10 @@ class jl_codectx_t { SmallVector slots; std::map phic_slots; std::map > scope_restore; - std::map eh_buffers; + std::map eh_buffers; SmallVector SAvalues; // The vector holds reference to Julia obj ref. We need to pin jl_value_t*. - SmallVector, jl_value_t *>, 0> PhiNodes; + SmallVector, jl_pinned_ref(jl_value_t)>, 0> PhiNodes; SmallVector ssavalue_assigned; SmallVector ssavalue_usecount; jl_module_t *module = NULL; @@ -1892,9 +1892,9 @@ class jl_codectx_t { jl_tbaacache_t tbaa_cache; jl_noaliascache_t aliasscope_cache; jl_method_instance_t *linfo = NULL; - jl_value_t *rettype = NULL; - jl_code_info_t *source = NULL; - jl_array_t *code = NULL; + jl_pinned_ref(jl_value_t) rettype = jl_pinned_ref_assume(jl_value_t, NULL); + jl_pinned_ref(jl_code_info_t) source = jl_pinned_ref_assume(jl_code_info_t, NULL); + jl_pinned_ref(jl_array_t) code = jl_pinned_ref_assume(jl_array_t, NULL); size_t min_world = 0; size_t max_world = -1; const char *name = NULL; @@ -2391,7 +2391,7 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ static jl_sym_t *slot_symbol(jl_codectx_t &ctx, int s) { - return (jl_sym_t*)jl_array_ptr_ref(ctx.source->slotnames, s); + return (jl_sym_t*)jl_array_ptr_ref(jl_pinned_ref_get(ctx.source)->slotnames, s); } static void store_def_flag(jl_codectx_t &ctx, const jl_varinfo_t &vi, bool val) @@ -5213,17 +5213,17 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR FunctionType *ft = f->getFunctionType(); if (ft == ctx.types().T_jlfunc) { Value *ret = emit_jlcall(ctx, f, nullptr, argv, nargs, julia_call); - result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); + result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, jl_pinned_ref_get(ctx.rettype)), rt); } else if (ft == ctx.types().T_jlfuncparams) { Value *ret = emit_jlcall(ctx, f, ctx.spvals_ptr, argv, nargs, julia_call2); - result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); + result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, jl_pinned_ref_get(ctx.rettype)), rt); } else { unsigned return_roots = 0; jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; StringRef protoname = f->getName(); - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, jl_pinned_ref_get(ctx.rettype), protoname, nullptr, argv, nargs, &cc, &return_roots, rt); } handled = true; } @@ -5709,7 +5709,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) { - jl_value_t *ssavalue_types = (jl_value_t*)ctx.source->ssavaluetypes; + jl_value_t *ssavalue_types = (jl_value_t*)jl_pinned_ref_get(ctx.source)->ssavaluetypes; jl_value_t *phiType = NULL; if (jl_is_array(ssavalue_types)) { phiType = jl_array_ptr_ref(ssavalue_types, idx); @@ -5751,8 +5751,8 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) decay_derived(ctx, phi)); jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, best_tbaa(ctx.tbaa(), phiType)); val.Vboxed = ptr_phi; - OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. - ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, roots, r)); + // OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. + ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, roots, jl_pinned_ref_create(jl_value_t, r))); ctx.SAvalues[idx] = val; ctx.ssavalue_assigned[idx] = true; return; @@ -5761,8 +5761,8 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_nrows(edges), "tindex_phi"); Tindex_phi->insertInto(BB, InsertPt); jl_cgval_t val = mark_julia_slot(NULL, phiType, Tindex_phi, ctx.tbaa().tbaa_stack); - OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. - ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, (PHINode*)nullptr, roots, r)); + // OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. + ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, (PHINode*)nullptr, roots, jl_pinned_ref_create(jl_value_t, r))); ctx.SAvalues[idx] = val; ctx.ssavalue_assigned[idx] = true; return; @@ -5816,8 +5816,8 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) value_phi->insertInto(BB, InsertPt); slot = mark_julia_type(ctx, value_phi, isboxed, phiType); } - OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. - ctx.PhiNodes.push_back(std::make_tuple(slot, BB, dest, value_phi, roots, r)); + // OBJ_PIN(r); // r will be saved to a data structure in the native heap, make sure it won't be moved by GC. + ctx.PhiNodes.push_back(std::make_tuple(slot, BB, dest, value_phi, roots, jl_pinned_ref_create(jl_value_t, r))); ctx.SAvalues[idx] = slot; ctx.ssavalue_assigned[idx] = true; return; @@ -5844,7 +5844,7 @@ static void emit_ssaval_assign(jl_codectx_t &ctx, ssize_t ssaidx_0based, jl_valu if (slot.isboxed || slot.TIndex) { // see if inference suggested a different type for the ssavalue than the expression // e.g. sometimes the information is inconsistent after inlining getfield on a Tuple - jl_value_t *ssavalue_types = (jl_value_t*)ctx.source->ssavaluetypes; + jl_value_t *ssavalue_types = (jl_value_t*)jl_pinned_ref_get(ctx.source)->ssavaluetypes; if (jl_is_array(ssavalue_types)) { jl_value_t *declType = jl_array_ptr_ref(ssavalue_types, ssaidx_0based); if (declType != slot.typ) { @@ -6120,7 +6120,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result) ctx.builder.CreateCall(prepare_call(gc_preserve_end_func), {token}); } if (jl_enternode_catch_dest(enter_stmt)) { - handler_to_end.push_back(ctx.eh_buffers[enter_stmt]); + handler_to_end.push_back(ctx.eh_buffers[jl_pinned_ref_assume(jl_value_t, enter_stmt)]); // We're not actually setting up the exception frames for these, so // we don't need to exit them. scope_to_restore = nullptr; // restored by exception handler @@ -6337,14 +6337,14 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } else if (head == jl_invoke_sym) { assert(ssaidx_0based >= 0); - jl_value_t *expr_t = jl_is_long(ctx.source->ssavaluetypes) ? (jl_value_t*)jl_any_type : - jl_array_ptr_ref(ctx.source->ssavaluetypes, ssaidx_0based); + jl_value_t *expr_t = jl_is_long(jl_pinned_ref_get(ctx.source)->ssavaluetypes) ? (jl_value_t*)jl_any_type : + jl_array_ptr_ref(jl_pinned_ref_get(ctx.source)->ssavaluetypes, ssaidx_0based); return emit_invoke(ctx, ex, expr_t); } else if (head == jl_invoke_modify_sym) { assert(ssaidx_0based >= 0); - jl_value_t *expr_t = jl_is_long(ctx.source->ssavaluetypes) ? (jl_value_t*)jl_any_type : - jl_array_ptr_ref(ctx.source->ssavaluetypes, ssaidx_0based); + jl_value_t *expr_t = jl_is_long(jl_pinned_ref_get(ctx.source)->ssavaluetypes) ? (jl_value_t*)jl_any_type : + jl_array_ptr_ref(jl_pinned_ref_get(ctx.source)->ssavaluetypes, ssaidx_0based); return emit_invoke_modify(ctx, ex, expr_t); } else if (head == jl_call_sym) { @@ -6354,7 +6354,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ // TODO: this case is needed for the call to emit_expr in emit_llvmcall expr_t = (jl_value_t*)jl_any_type; else { - expr_t = jl_is_long(ctx.source->ssavaluetypes) ? (jl_value_t*)jl_any_type : jl_array_ptr_ref(ctx.source->ssavaluetypes, ssaidx_0based); + expr_t = jl_is_long(jl_pinned_ref_get(ctx.source)->ssavaluetypes) ? (jl_value_t*)jl_any_type : jl_array_ptr_ref(jl_pinned_ref_get(ctx.source)->ssavaluetypes, ssaidx_0based); is_promotable = ctx.ssavalue_usecount[ssaidx_0based] == 1; } jl_cgval_t res = emit_call(ctx, ex, expr_t, is_promotable); @@ -7044,7 +7044,7 @@ static void emit_fptr1_wrapper(Module *M, StringRef gf_thunk_name, Value *target jl_codectx_t ctx(M->getContext(), params, 0, 0); ctx.f = w; - ctx.rettype = declrt; + ctx.rettype = jl_pinned_ref_create(jl_value_t, declrt); BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); ctx.builder.SetInsertPoint(b0); @@ -7824,7 +7824,7 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_va jl_codectx_t ctx(M->getContext(), params, 0, 0); ctx.f = w; ctx.linfo = lam; - ctx.rettype = jlretty; + ctx.rettype = jl_pinned_ref_create(jl_value_t, jlretty); BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); ctx.builder.SetInsertPoint(b0); @@ -8135,8 +8135,8 @@ static jl_llvm_functions_t jl_codectx_t ctx(*params.tsctx.getContext(), params, min_world, max_world); jl_datatype_t *vatyp = NULL; JL_GC_PUSH2(&ctx.code, &vatyp); - ctx.code = src->code; - ctx.source = src; + ctx.code = jl_pinned_ref_create(jl_array_t, src->code); + ctx.source = jl_pinned_ref_create(jl_code_info_t, src); ctx.module = jl_is_method(lam->def.method) ? lam->def.method->module : lam->def.module; ctx.linfo = lam; @@ -8157,10 +8157,10 @@ static jl_llvm_functions_t if (vn != jl_unused_sym) ctx.vaSlot = ctx.nargs - 1; } - ctx.rettype = jlrettype; + ctx.rettype = jl_pinned_ref_create(jl_value_t, jlrettype); ctx.funcName = ctx.name; ctx.spvals_ptr = NULL; - jl_array_t *stmts = ctx.code; + jl_array_t *stmts = jl_pinned_ref_get(jl_pinned_ref_create(jl_array_t, ctx.code)); size_t stmtslen = jl_array_dim0(stmts); // step 1b. unpack debug information @@ -9373,7 +9373,7 @@ static jl_llvm_functions_t auto *handler_sz64 = ConstantInt::get(Type::getInt64Ty(ctx.builder.getContext()), sizeof(jl_handler_t)); AllocaInst* ehbuff = emit_static_alloca(ctx, sizeof(jl_handler_t), Align(16)); - ctx.eh_buffers[stmt] = ehbuff; + ctx.eh_buffers[jl_pinned_ref_create(jl_value_t, stmt)] = ehbuff; ctx.builder.CreateLifetimeStart(ehbuff, handler_sz64); ctx.builder.CreateCall(prepare_call(jlenter_func), {ct, ehbuff}); CallInst *sj; @@ -9447,14 +9447,14 @@ static jl_llvm_functions_t for (auto &tup : ctx.PhiNodes) { jl_cgval_t phi_result; PHINode *VN; - jl_value_t *r; + jl_pinned_ref(jl_value_t) r = jl_pinned_ref_assume(jl_value_t, NULL); AllocaInst *dest; SmallVector roots; BasicBlock *PhiBB; std::tie(phi_result, PhiBB, dest, VN, roots, r) = tup; jl_value_t *phiType = phi_result.typ; - jl_array_t *edges = (jl_array_t*)jl_fieldref_noalloc(r, 0); - jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(r, 1); + jl_array_t *edges = (jl_array_t*)jl_fieldref_noalloc(jl_pinned_ref_get(r), 0); + jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(jl_pinned_ref_get(r), 1); PHINode *TindexN = cast_or_null(phi_result.TIndex); DenseSet preds; for (size_t i = 0; i < jl_array_nrows(edges); ++i) { From d302280fec8705690fe8bb7788beb5c7665c6db1 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 14 Apr 2025 03:16:52 +0000 Subject: [PATCH 631/662] Wrapping types from jl_native_code_desc_t with jl_pinned_ref --- src/aotcompile.cpp | 4 ++-- src/julia.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 91c4dfddac169..d68c5e36d7e6d 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -73,7 +73,7 @@ typedef struct { SmallVector jl_sysimg_gvars; std::map> jl_fvar_map; // This holds references to the heap. Need to be pinned. - SmallVector jl_value_to_llvm; + SmallVector jl_value_to_llvm; SmallVector jl_external_to_llvm; } jl_native_code_desc_t; @@ -857,7 +857,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm gvars[idx] = global.second->getName().str(); assert(gvars_set.insert(global.second).second && "Duplicate gvar in params!"); assert(gvars_names.insert(gvars[idx]).second && "Duplicate gvar name in params!"); - data->jl_value_to_llvm[idx] = global.first; + data->jl_value_to_llvm[idx] = jl_pinned_ref_assume(void, global.first); idx++; } CreateNativeMethods += compiled_functions.size(); diff --git a/src/julia.h b/src/julia.h index 3eb495ee5be83..0820cd52420d5 100644 --- a/src/julia.h +++ b/src/julia.h @@ -115,6 +115,7 @@ template class pinned_ref { T* ptr; public: + explicit pinned_ref() : ptr(static_cast(assume(NULL))) {} explicit pinned_ref(void* p) : ptr(static_cast(p)) {} operator void*() const { return ptr; } T* get() const { return ptr; } From f6ea9db69aedb7921a103973282ad408987d11b9 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 14 Apr 2025 04:32:00 +0000 Subject: [PATCH 632/662] Wrapping types from jl_cgval_t with jl_pinned_ref --- src/ccall.cpp | 10 +- src/cgutils.cpp | 102 +++++++------- src/codegen.cpp | 338 ++++++++++++++++++++++----------------------- src/intrinsics.cpp | 114 +++++++-------- src/julia.h | 3 - 5 files changed, 282 insertions(+), 285 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 947653e3cb3b0..e395ced9a8d6f 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -484,10 +484,10 @@ static const std::string make_errmsg(const char *fname, int n, const char *err) static jl_cgval_t typeassert_input(jl_codectx_t &ctx, const jl_cgval_t &jvinfo, jl_value_t *jlto, jl_unionall_t *jlto_env, int argn) { - if (jlto != (jl_value_t*)jl_any_type && !jl_subtype(jvinfo.typ, jlto)) { + if (jlto != (jl_value_t*)jl_any_type && !jl_subtype(jl_pinned_ref_get(jvinfo.typ), jlto)) { if (jlto == (jl_value_t*)jl_voidpointer_type) { // allow a bit more flexibility for what can be passed to (void*) due to Ref{T} conversion behavior in input - if (!jl_is_cpointer_type(jvinfo.typ)) { + if (!jl_is_cpointer_type(jl_pinned_ref_get(jvinfo.typ))) { // emit a typecheck, if not statically known to be correct emit_cpointercheck(ctx, jvinfo, make_errmsg("ccall", argn + 1, "")); return update_julia_type(ctx, jvinfo, (jl_value_t*)jl_pointer_type); @@ -596,7 +596,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va } } jl_cgval_t arg1 = emit_expr(ctx, arg); - jl_value_t *ptr_ty = arg1.typ; + jl_value_t *ptr_ty = jl_pinned_ref_get(arg1.typ); if (!jl_is_cpointer_type(ptr_ty)) { if (!ccall) return; @@ -2016,9 +2016,9 @@ jl_cgval_t function_sig_t::emit_a_ccall( Value *v; if (jl_is_abstract_ref_type(jargty)) { - if (!jl_is_cpointer_type(arg.typ)) { + if (!jl_is_cpointer_type(jl_pinned_ref_get(arg.typ))) { emit_cpointercheck(ctx, arg, "ccall: argument to Ref{T} is not a pointer"); - arg.typ = (jl_value_t*)jl_voidpointer_type; + arg.typ = jl_pinned_ref_create(jl_value_t, (jl_value_t*)jl_voidpointer_type); arg.isboxed = false; } jargty_in_env = (jl_value_t*)jl_voidpointer_type; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 5175c9d56731f..3617341b46694 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -375,8 +375,8 @@ static llvm::SmallVector get_gc_roots_for(jl_codectx_t &ctx, const jl_ assert(x.V->getType()->getPointerAddressSpace() != AddressSpace::Tracked); return {x.V}; } - else if (jl_is_concrete_immutable(x.typ) && !jl_is_pointerfree(x.typ)) { - jl_value_t *jltype = x.typ; + else if (jl_is_concrete_immutable(jl_pinned_ref_get(x.typ)) && !jl_is_pointerfree(jl_pinned_ref_get(x.typ))) { + jl_value_t *jltype = jl_pinned_ref_get(x.typ); Type *T = julia_type_to_llvm(ctx, jltype); Value *agg = emit_unbox(ctx, T, x, jltype); SmallVector perm_offsets; @@ -977,11 +977,11 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x) assert(x.ispointer()); Value *data; if (x.constant) { - Constant *val = julia_const_to_llvm(ctx, x.constant); + Constant *val = julia_const_to_llvm(ctx, jl_pinned_ref_get(x.constant)); if (val) - data = get_pointer_to_constant(ctx.emission_context, val, Align(julia_alignment(jl_typeof(x.constant))), "_j_const", *jl_Module); + data = get_pointer_to_constant(ctx.emission_context, val, Align(julia_alignment(jl_typeof(jl_pinned_ref_get(x.constant)))), "_j_const", *jl_Module); else - data = literal_pointer_val(ctx, x.constant); + data = literal_pointer_val(ctx, jl_pinned_ref_get(x.constant)); } else if (x.V == NULL) { // might be a ghost union with tindex but no actual pointer @@ -1077,8 +1077,8 @@ static std::pair split_value_size(jl_datatype_t *typ) // take a value `x` and split its bits into dst and the roots into inline_roots static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align_src, Value *dst, Align align_dst, jl_aliasinfo_t const &dst_ai, Value *inline_roots_ptr, jl_aliasinfo_t const &roots_ai, bool isVolatileStore=false) { - jl_datatype_t *typ = (jl_datatype_t*)x.typ; - assert(jl_is_concrete_type(x.typ)); + jl_datatype_t *typ = (jl_datatype_t*)jl_pinned_ref_get(x.typ); + assert(jl_is_concrete_type(jl_pinned_ref_get(x.typ))); auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); Type *T_prjlvalue = ctx.types().T_prjlvalue; if (!x.inline_roots.empty()) { @@ -1141,8 +1141,8 @@ static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align_src, Value *dst, Align align_dst, jl_aliasinfo_t const &dst_ai, MutableArrayRef inline_roots) { - jl_datatype_t *typ = (jl_datatype_t*)x.typ; - assert(jl_is_concrete_type(x.typ)); + jl_datatype_t *typ = (jl_datatype_t*)jl_pinned_ref_get(x.typ); + assert(jl_is_concrete_type(jl_pinned_ref_get(x.typ))); auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); Type *T_prjlvalue = ctx.types().T_prjlvalue; if (!x.inline_roots.empty()) { @@ -1201,7 +1201,7 @@ static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align static std::pair> split_value(jl_codectx_t &ctx, const jl_cgval_t &x, Align x_alignment) { - jl_datatype_t *typ = (jl_datatype_t*)x.typ; + jl_datatype_t *typ = (jl_datatype_t*)jl_pinned_ref_get(x.typ); auto sizes = split_value_size(typ); Align align_dst(julia_alignment((jl_value_t*)typ)); AllocaInst *bits = sizes.first > 0 ? emit_static_alloca(ctx, sizes.first, align_dst) : nullptr; @@ -1242,11 +1242,11 @@ static std::pair split_value_field(jl_datatype_t *typ, unsigned // This does not respect roots, so you must call emit_write_multibarrier afterwards. static void recombine_value(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dst, jl_aliasinfo_t const &dst_ai, Align alignment, bool isVolatileStore) { - jl_datatype_t *typ = (jl_datatype_t*)x.typ; - assert(jl_is_concrete_type(x.typ)); + jl_datatype_t *typ = (jl_datatype_t*)jl_pinned_ref_get(x.typ); + assert(jl_is_concrete_type(jl_pinned_ref_get(x.typ))); assert(typ->layout->first_ptr >= 0 && !x.inline_roots.empty()); Align align_dst = alignment; - Align align_src(julia_alignment(x.typ)); + Align align_src(julia_alignment(jl_pinned_ref_get(x.typ))); Value *src = x.V; auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); size_t dst_off = 0; @@ -1300,9 +1300,9 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull // given p, compute its type jl_datatype_t *dt = NULL; if (p.constant) - dt = (jl_datatype_t*)jl_typeof(p.constant); - else if (jl_is_concrete_type(p.typ)) - dt = (jl_datatype_t*)p.typ; + dt = (jl_datatype_t*)jl_typeof(jl_pinned_ref_get(p.constant)); + else if (jl_is_concrete_type(jl_pinned_ref_get(p.typ))) + dt = (jl_datatype_t*)jl_pinned_ref_get(p.typ); if (dt) { if (justtag) return emit_tagfrom(ctx, dt); @@ -1333,10 +1333,10 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull return true; }; if (p.isboxed) - return emit_typeof(ctx, p.V, maybenull, justtag, notag(p.typ)); + return emit_typeof(ctx, p.V, maybenull, justtag, notag(jl_pinned_ref_get(p.typ))); if (p.TIndex) { Value *tindex = ctx.builder.CreateAnd(p.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); - bool allunboxed = is_uniontype_allunboxed(p.typ); + bool allunboxed = is_uniontype_allunboxed(jl_pinned_ref_get(p.typ)); Type *expr_type = justtag ? ctx.types().T_size : ctx.types().T_pjlvalue; Value *datatype_or_p = Constant::getNullValue(PointerType::getUnqual(expr_type->getContext())); unsigned counter = 0; @@ -1353,7 +1353,7 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull datatype_or_p = ctx.builder.CreateSelect(cmp, ptr, datatype_or_p); setName(ctx.emission_context, datatype_or_p, "typetag_ptr"); }, - p.typ, + jl_pinned_ref_get(p.typ), counter); auto emit_unboxty = [&] () -> Value* { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); @@ -1370,7 +1370,7 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull BasicBlock *mergeBB = BasicBlock::Create(ctx.builder.getContext(), "merge", ctx.f); ctx.builder.CreateCondBr(isnull, boxBB, unboxBB); ctx.builder.SetInsertPoint(boxBB); - auto boxTy = emit_typeof(ctx, p.Vboxed, maybenull, justtag, notag(p.typ)); + auto boxTy = emit_typeof(ctx, p.Vboxed, maybenull, justtag, notag(jl_pinned_ref_get(p.typ))); ctx.builder.CreateBr(mergeBB); boxBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(unboxBB); @@ -1811,7 +1811,7 @@ static Value *emit_exactly_isa(jl_codectx_t &ctx, const jl_cgval_t &arg, jl_data { assert(jl_is_concrete_type((jl_value_t*)dt) || is_uniquerep_Type((jl_value_t*)dt)); if (arg.TIndex) { - unsigned tindex = get_box_tindex(dt, arg.typ); + unsigned tindex = get_box_tindex(dt, jl_pinned_ref_get(arg.typ)); if (tindex > 0) { // optimize more when we know that this is a split union-type where tindex = 0 is invalid Value *xtindex = ctx.builder.CreateAnd(arg.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), ~UNION_BOX_MARKER)); @@ -1895,11 +1895,11 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, std::optional known_isa; jl_value_t *intersected_type = type; if (x.constant) - known_isa = jl_isa(x.constant, type); - else if (jl_is_not_broken_subtype(x.typ, type) && jl_subtype(x.typ, type)) { + known_isa = jl_isa(jl_pinned_ref_get(x.constant), type); + else if (jl_is_not_broken_subtype(jl_pinned_ref_get(x.typ), type) && jl_subtype(jl_pinned_ref_get(x.typ), type)) { known_isa = true; } else { - intersected_type = jl_type_intersection(x.typ, type); + intersected_type = jl_type_intersection(jl_pinned_ref_get(x.typ), type); if (intersected_type == (jl_value_t*)jl_bottom_type) known_isa = false; } @@ -2500,9 +2500,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else if (cmpop.isboxed || cmpop.constant || jl_pointer_egal(jltype)) { Compare = boxed(ctx, cmpop); - needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(cmpop.typ); + needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(jl_pinned_ref_get(cmpop.typ)); if (needloop && !cmpop.isboxed) // try to use the same box in the compare now and later - cmpop = mark_julia_type(ctx, Compare, true, cmpop.typ); + cmpop = mark_julia_type(ctx, Compare, true, jl_pinned_ref_get(cmpop.typ)); } else { Compare = Constant::getNullValue(ctx.types().T_prjlvalue); // TODO: does this need to be an invalid bit pattern? @@ -2678,7 +2678,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(DoneBB); if (needlock) emit_lockstate_value(ctx, needlock, false); - if (parent != NULL && tracked_pointers && (!isboxed || !type_is_permalloc(rhs.typ))) { + if (parent != NULL && tracked_pointers && (!isboxed || !type_is_permalloc(jl_pinned_ref_get(rhs.typ)))) { if (isreplacefield || issetfieldonce) { BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "xchg_wb", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg_wb", ctx.f); @@ -2698,7 +2698,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, r = emit_unbox(ctx, intcast_eltyp, rhs, jltype); } if (!isboxed) - emit_write_multibarrier(ctx, parent, r, rhs.typ); + emit_write_multibarrier(ctx, parent, r, jl_pinned_ref_get(rhs.typ)); else emit_write_barrier(ctx, parent, r); } @@ -2777,7 +2777,7 @@ static bool field_may_be_null(const jl_cgval_t &strct, jl_datatype_t *stt, size_ if (!jl_field_isptr(stt, idx) && !jl_type_hasptr(jl_field_type(stt, idx))) return false; if (strct.constant) { - if ((jl_is_immutable(stt) || jl_field_isconst(stt, idx)) && jl_field_isdefined(strct.constant, idx)) + if ((jl_is_immutable(stt) || jl_field_isconst(stt, idx)) && jl_field_isdefined(jl_pinned_ref_get(strct.constant), idx)) return false; } return true; @@ -3407,8 +3407,8 @@ static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tb static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t &v) { - MDNode *tbaa = jl_is_mutable(v.typ) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; - unsigned alignment = julia_alignment(v.typ); + MDNode *tbaa = jl_is_mutable(jl_pinned_ref_get(v.typ)) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; + unsigned alignment = julia_alignment(jl_pinned_ref_get(v.typ)); unsigned newv_align = std::max(alignment, (unsigned)sizeof(void*)); newv = maybe_decay_tracked(ctx, newv); emit_unbox_store(ctx, v, newv, tbaa, Align(alignment), Align(newv_align)); @@ -3501,7 +3501,7 @@ static Value *call_with_attrs(jl_codectx_t &ctx, JuliaFunction *intr, static Value *as_value(jl_codectx_t &ctx, Type *to, const jl_cgval_t &v) { assert(!v.isboxed); - return emit_unbox(ctx, to, v, v.typ); + return emit_unbox(ctx, to, v, jl_pinned_ref_get(v.typ)); } static Value *load_i8box(jl_codectx_t &ctx, Value *v, jl_datatype_t *ty) @@ -3520,7 +3520,7 @@ static Value *load_i8box(jl_codectx_t &ctx, Value *v, jl_datatype_t *ty) // Returns ctx.types().T_prjlvalue static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t) { - jl_value_t *jt = vinfo.typ; + jl_value_t *jt = jl_pinned_ref_get(vinfo.typ); if (jt == (jl_value_t*)jl_bool_type) return track_pjlvalue(ctx, julia_bool(ctx, ctx.builder.CreateTrunc(as_value(ctx, t, vinfo), getInt1Ty(ctx.builder.getContext())))); if (t == getInt1Ty(ctx.builder.getContext())) @@ -3604,11 +3604,11 @@ static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, j if (val.typ == jl_bottom_type) return UndefValue::get(getInt8Ty(ctx.builder.getContext())); if (val.constant) - return ConstantInt::get(getInt8Ty(ctx.builder.getContext()), get_box_tindex((jl_datatype_t*)jl_typeof(val.constant), typ)); + return ConstantInt::get(getInt8Ty(ctx.builder.getContext()), get_box_tindex((jl_datatype_t*)jl_typeof(jl_pinned_ref_get(val.constant)), typ)); if (val.TIndex) return ctx.builder.CreateAnd(val.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); Value *typof = emit_typeof(ctx, val, maybenull, true); - return compute_box_tindex(ctx, typof, val.typ, typ); + return compute_box_tindex(ctx, typof, jl_pinned_ref_get(val.typ), typ); } @@ -3710,7 +3710,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB box_merge->addIncoming(box, tempBB); ctx.builder.CreateBr(postBB); }, - vinfo.typ, + jl_pinned_ref_get(vinfo.typ), counter); ctx.builder.SetInsertPoint(defaultBB); if (skip.size() > 0) { @@ -3805,12 +3805,12 @@ static void recursively_adjust_ptr_type(llvm::Value *Val, unsigned FromAS, unsig // Returns ctx.types().T_prjlvalue static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotable) { - jl_value_t *jt = vinfo.typ; + jl_value_t *jt = jl_pinned_ref_get(vinfo.typ); if (jt == jl_bottom_type || jt == NULL) // We have an undef value on a (hopefully) dead branch return UndefValue::get(ctx.types().T_prjlvalue); if (vinfo.constant) - return track_pjlvalue(ctx, literal_pointer_val(ctx, vinfo.constant)); + return track_pjlvalue(ctx, literal_pointer_val(ctx, jl_pinned_ref_get(vinfo.constant))); // This can happen in early bootstrap for `gc_preserve_begin` return value. if (jt == (jl_value_t*)jl_nothing_type) return track_pjlvalue(ctx, literal_pointer_val(ctx, jl_nothing)); @@ -3868,21 +3868,21 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con // TODO: make this a lifetime_end & dereferenceable annotation? ctx.builder.CreateAlignedStore(UndefValue::get(ai->getAllocatedType()), ai, ai->getAlign()); if (src.constant) { - jl_value_t *typ = jl_typeof(src.constant); + jl_value_t *typ = jl_typeof(jl_pinned_ref_get(src.constant)); assert(skip || jl_is_pointerfree(typ)); if (jl_is_pointerfree(typ)) { emit_guarded_test(ctx, skip, nullptr, [&] { unsigned alignment = julia_alignment(typ); - emit_unbox_store(ctx, mark_julia_const(ctx, src.constant), dest, tbaa_dst, Align(alignment), Align(alignment), isVolatile); + emit_unbox_store(ctx, mark_julia_const(ctx, jl_pinned_ref_get(src.constant)), dest, tbaa_dst, Align(alignment), Align(alignment), isVolatile); return nullptr; }); } } - else if (jl_is_concrete_type(src.typ)) { - assert(skip || jl_is_pointerfree(src.typ)); - if (jl_is_pointerfree(src.typ)) { + else if (jl_is_concrete_type(jl_pinned_ref_get(src.typ))) { + assert(skip || jl_is_pointerfree(jl_pinned_ref_get(src.typ))); + if (jl_is_pointerfree(jl_pinned_ref_get(src.typ))) { emit_guarded_test(ctx, skip, nullptr, [&] { - unsigned alignment = julia_alignment(src.typ); + unsigned alignment = julia_alignment(jl_pinned_ref_get(src.typ)); emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), Align(alignment), isVolatile); return nullptr; }); @@ -3922,7 +3922,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con } ctx.builder.CreateBr(postBB); }, - src.typ, + jl_pinned_ref_get(src.typ), counter); ctx.builder.SetInsertPoint(defaultBB); if (!skip && allunboxed && (src.V == NULL || isa(src.V))) { @@ -4769,7 +4769,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg Value *i = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); Value *offset = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); setName(ctx.emission_context, offset, "memoryref_offset"); - Value *elsz = emit_genericmemoryelsize(ctx, mem, ref.typ, false); + Value *elsz = emit_genericmemoryelsize(ctx, mem, jl_pinned_ref_get(ref.typ), false); bool bc = bounds_check_enabled(ctx, inbounds); #if 1 Value *ovflw = nullptr; @@ -4785,7 +4785,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg BasicBlock *failBB, *endBB; failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); Value *inbound = ctx.builder.CreateICmpULT(newdata, mlen); setName(ctx.emission_context, offset, "memoryref_isinbounds"); ctx.builder.CreateCondBr(inbound, endBB, failBB); @@ -4817,7 +4817,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg // n.b. we could boundscheck that -len<=offset<=len instead of using smul.ovflw, // since we know that len*elsz does not overflow, // and we can further rearrange that as ovflw = !( offset+len < len+len ) as unsigned math - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); ovflw = ctx.builder.CreateICmpUGE(ctx.builder.CreateAdd(offset, mlen), ctx.builder.CreateNUWAdd(mlen, mlen)); setName(ctx.emission_context, ovflw, "memoryref_ovflw"); } @@ -4831,7 +4831,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg BasicBlock *failBB, *endBB; failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); Value *mptr = emit_genericmemoryptr(ctx, mem, layout, 0); #if 0 Value *mend = mptr; @@ -4871,7 +4871,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg ctx.builder.SetInsertPoint(endBB); } } - return _emit_memoryref(ctx, mem, newdata, layout, ref.typ); + return _emit_memoryref(ctx, mem, newdata, layout, jl_pinned_ref_get(ref.typ)); } static jl_cgval_t emit_memoryref_offset(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) @@ -4890,7 +4890,7 @@ static jl_cgval_t emit_memoryref_offset(jl_codectx_t &ctx, const jl_cgval_t &ref ctx.builder.CreatePtrToInt(data, ctx.types().T_size), ctx.builder.CreatePtrToInt(mptr, ctx.types().T_size)); setName(ctx.emission_context, offset, "memoryref_offset"); - Value *elsz = emit_genericmemoryelsize(ctx, mem, ref.typ, false); + Value *elsz = emit_genericmemoryelsize(ctx, mem, jl_pinned_ref_get(ref.typ), false); offset = ctx.builder.CreateExactUDiv(offset, elsz); setName(ctx.emission_context, offset, "memoryref_offsetidx"); } diff --git a/src/codegen.cpp b/src/codegen.cpp index a290acd25c101..c8c9ff5f97606 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1721,8 +1721,8 @@ struct jl_cgval_t { Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 SmallVector inline_roots; // if present, `V` is a pointer, but not in canonical layout - jl_value_t *constant; // constant value (rooted in linfo.def.roots) - jl_value_t *typ; // the original type of V, never nullptr + jl_pinned_ref(jl_value_t) constant; // constant value (rooted in linfo.def.roots) + jl_pinned_ref(jl_value_t) typ; // the original type of V, never nullptr bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag bool isghost; // whether this value is "ghost" MDNode *tbaa; // The related tbaa node. Non-nullptr iff this holds an address. @@ -2212,7 +2212,7 @@ static inline jl_cgval_t ghostValue(jl_codectx_t &ctx, jl_value_t *typ) assert(is_uniquerep_Type(typ)); // replace T::Type{T} with T, by assuming that T must be a leaftype of some sort jl_cgval_t constant(NULL, true, typ, NULL, best_tbaa(ctx.tbaa(), typ), None); - constant.constant = jl_tparam0(typ); + constant.constant = jl_pinned_ref_create(jl_value_t, jl_tparam0(typ)); if (typ == (jl_value_t*)jl_typeofbottom_type->super) constant.isghost = true; return constant; @@ -2236,7 +2236,7 @@ static inline jl_cgval_t mark_julia_const(jl_codectx_t &ctx, jl_value_t *jv) return ghostValue(ctx, typ); } jl_cgval_t constant(NULL, true, typ, NULL, best_tbaa(ctx.tbaa(), typ), None); - constant.constant = jv; + constant.constant = jl_pinned_ref_create(jl_value_t, jv); return constant; } @@ -2291,17 +2291,17 @@ static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, const jl_cgval_t &v // ctx.builder.CreateAlignedStore(v.inline_roots[i], emit_ptrgep(ctx, loc, i * sizeof(void*)), Align(sizeof(void*))); // return mark_julia_slot(loc, v.typ, v.TIndex, ctx.tbaa().tbaa_gcframe); //} - Align align(julia_alignment(v.typ)); - Type *ty = julia_type_to_llvm(ctx, v.typ); + Align align(julia_alignment(jl_pinned_ref_get(v.typ))); + Type *ty = julia_type_to_llvm(ctx, jl_pinned_ref_get(v.typ)); AllocaInst *loc = emit_static_alloca(ctx, ty, align); auto tbaa = v.V == nullptr ? ctx.tbaa().tbaa_gcframe : ctx.tbaa().tbaa_stack; auto stack_ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); recombine_value(ctx, v, loc, stack_ai, align, false); - return mark_julia_slot(loc, v.typ, v.TIndex, tbaa); + return mark_julia_slot(loc, jl_pinned_ref_get(v.typ), v.TIndex, tbaa); } if (v.ispointer()) return v; - return value_to_pointer(ctx, v.V, v.typ, v.TIndex); + return value_to_pointer(ctx, v.V, jl_pinned_ref_get(v.typ), v.TIndex); } static inline jl_cgval_t mark_julia_type(jl_codectx_t &ctx, Value *v, bool isboxed, jl_value_t *typ) @@ -2339,11 +2339,11 @@ static inline jl_cgval_t mark_julia_type(jl_codectx_t &ctx, Value *v, bool isbox // see if it might be profitable (and cheap) to change the type of v to typ static inline jl_cgval_t update_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ) { - if (v.typ == jl_bottom_type || typ == (jl_value_t*)jl_any_type || jl_egal(v.typ, typ)) + if (v.typ == jl_bottom_type || typ == (jl_value_t*)jl_any_type || jl_egal(jl_pinned_ref_get(v.typ), typ)) return v; // fast-path if (v.constant) - return jl_isa(v.constant, typ) ? v : jl_cgval_t(); - if (jl_is_concrete_type(v.typ) && !jl_is_kind(v.typ)) { + return jl_isa(jl_pinned_ref_get(v.constant), typ) ? v : jl_cgval_t(); + if (jl_is_concrete_type(jl_pinned_ref_get(v.typ)) && !jl_is_kind(jl_pinned_ref_get(v.typ))) { if (jl_is_concrete_type(typ) && !jl_is_kind(typ)) { // type mismatch: changing from one leaftype to another CreateTrap(ctx.builder); @@ -2499,7 +2499,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & } skip_box.resize(idx + 1, t); }, - v.typ, + jl_pinned_ref_get(v.typ), counter); } setName(ctx.emission_context, new_tindex, "tindex"); @@ -2544,7 +2544,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & for_each_uniontype_small( // for each new union-split value [&](unsigned idx, jl_datatype_t *jt) { - unsigned old_idx = get_box_tindex(jt, v.typ); + unsigned old_idx = get_box_tindex(jt, jl_pinned_ref_get(v.typ)); if (old_idx == 0) { // didn't handle this item before, select its new union index maybe_setup_union_isa(); @@ -2617,14 +2617,14 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ { if (typ == (jl_value_t*)jl_typeofbottom_type) return ghostValue(ctx, typ); // normalize TypeofBottom to Type{Union{}} - if (v.typ == jl_bottom_type || jl_egal(v.typ, typ)) + if (v.typ == jl_bottom_type || jl_egal(jl_pinned_ref_get(v.typ), typ)) return v; // fast-path Type *T = julia_type_to_llvm(ctx, typ); if (type_is_ghost(T)) return ghostValue(ctx, typ); Value *new_tindex = NULL; if (jl_is_concrete_type(typ)) { - if (jl_is_concrete_type(v.typ)) { + if (jl_is_concrete_type(jl_pinned_ref_get(v.typ))) { // type mismatch: changing from one leaftype to another if (skip) *skip = ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); @@ -2655,8 +2655,8 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ } else if (!v.isboxed && jl_is_uniontype(typ)) { // previous value was unboxed (leaftype), statically compute union tindex - assert(jl_is_concrete_type(v.typ)); - unsigned new_idx = get_box_tindex((jl_datatype_t*)v.typ, typ); + assert(jl_is_concrete_type(jl_pinned_ref_get(v.typ))); + unsigned new_idx = get_box_tindex((jl_datatype_t*)jl_pinned_ref_get(v.typ), typ); if (new_idx) { new_tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), new_idx); if (v.V && v.inline_roots.empty() && !v.ispointer()) { @@ -2664,7 +2664,7 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return jl_cgval_t(value_to_pointer(ctx, v), typ, new_tindex); } } - else if (jl_subtype(v.typ, typ)) { + else if (jl_subtype(jl_pinned_ref_get(v.typ), typ)) { makeboxed = true; } else if (skip) { @@ -2874,7 +2874,7 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef arg for (size_t i = 0; i < nargs; i++) { if (!args[i].constant) return NULL; - v[i] = args[i].constant; + v[i] = jl_pinned_ref_get(args[i].constant); } assert(v[0] == BUILTIN(apply_type)); size_t last_age = jl_current_task->world_age; @@ -2919,7 +2919,7 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) ssize_t idx = ((jl_ssavalue_t*)ex)->id - 1; assert(idx >= 0); if (ctx.ssavalue_assigned[idx]) { - return ctx.SAvalues[idx].constant; + return jl_pinned_ref_get(ctx.SAvalues[idx].constant); } return NULL; } @@ -3320,7 +3320,7 @@ static Value *emit_box_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const Value *nullcheck1, Value *nullcheck2) { ++EmittedBoxCompares; - if (jl_pointer_egal(arg1.typ) || jl_pointer_egal(arg2.typ)) { + if (jl_pointer_egal(jl_pinned_ref_get(arg1.typ)) || jl_pointer_egal(jl_pinned_ref_get(arg2.typ))) { // if we can be certain we won't try to load from the pointer (because // we know boxed is trivial), we can skip the separate null checks // and just do the ICmpEQ test @@ -3330,7 +3330,7 @@ static Value *emit_box_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const return emit_nullcheck_guard2(ctx, nullcheck1, nullcheck2, [&] { Value *varg1 = decay_derived(ctx, boxed(ctx, arg1)); Value *varg2 = decay_derived(ctx, boxed(ctx, arg2)); - if (jl_pointer_egal(arg1.typ) || jl_pointer_egal(arg2.typ)) { + if (jl_pointer_egal(jl_pinned_ref_get(arg1.typ)) || jl_pointer_egal(jl_pinned_ref_get(arg2.typ))) { return ctx.builder.CreateICmpEQ(varg1, varg2); } Value *neq = ctx.builder.CreateICmpNE(varg1, varg2); @@ -3350,7 +3350,7 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2) { ++EmittedBitsUnionCompares; - assert(jl_egal(arg1.typ, arg2.typ) && arg1.TIndex && arg2.TIndex && jl_is_uniontype(arg1.typ) && "unimplemented"); + assert(jl_egal(jl_pinned_ref_get(arg1.typ), jl_pinned_ref_get(arg2.typ)) && arg1.TIndex && arg2.TIndex && jl_is_uniontype(jl_pinned_ref_get(arg1.typ)) && "unimplemented"); Value *tindex = arg1.TIndex; tindex = ctx.builder.CreateAnd(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); Value *tindex2 = arg2.TIndex; @@ -3378,7 +3378,7 @@ static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, phi->addIncoming(cmp, tempBB); ctx.builder.CreateBr(postBB); }, - arg1.typ, + jl_pinned_ref_get(arg1.typ), counter); assert(allunboxed); (void)allunboxed; ctx.builder.SetInsertPoint(defaultBB); @@ -3399,10 +3399,10 @@ static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t arg2) { ++EmittedBitsCompares; - jl_value_t *argty = (arg1.constant ? jl_typeof(arg1.constant) : arg1.typ); + jl_value_t *argty = (jl_pinned_ref_get(arg1.constant) ? jl_typeof(jl_pinned_ref_get(arg1.constant)) : jl_pinned_ref_get(arg1.typ)); bool isboxed; - Type *at = julia_type_to_llvm(ctx, arg1.typ, &isboxed); - assert(jl_is_datatype(arg1.typ) && arg1.typ == (arg2.constant ? jl_typeof(arg2.constant) : arg2.typ) && !isboxed); + Type *at = julia_type_to_llvm(ctx, jl_pinned_ref_get(arg1.typ), &isboxed); + assert(jl_is_datatype(jl_pinned_ref_get(arg1.typ)) && jl_pinned_ref_get(arg1.typ) == (jl_pinned_ref_get(arg2.constant) ? jl_typeof(jl_pinned_ref_get(arg2.constant)) : arg2.typ) && !isboxed); if (type_is_ghost(at)) return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); @@ -3513,10 +3513,10 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva ++EmittedEgals; // handle simple static expressions with no side-effects if (arg1.constant && arg2.constant) - return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), jl_egal(arg1.constant, arg2.constant)); + return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), jl_egal(jl_pinned_ref_get(arg1.constant), jl_pinned_ref_get(arg2.constant))); - jl_value_t *rt1 = (arg1.constant ? jl_typeof(arg1.constant) : arg1.typ); - jl_value_t *rt2 = (arg2.constant ? jl_typeof(arg2.constant) : arg2.typ); + jl_value_t *rt1 = (jl_pinned_ref_get(arg1.constant) ? jl_typeof(jl_pinned_ref_get(arg1.constant)) : jl_pinned_ref_get(arg1.typ)); + jl_value_t *rt2 = (jl_pinned_ref_get(arg2.constant) ? jl_typeof(jl_pinned_ref_get(arg2.constant)) : jl_pinned_ref_get(arg2.typ)); if (jl_is_concrete_type(rt1) && jl_is_concrete_type(rt2) && !jl_is_kind(rt1) && !jl_is_kind(rt2) && rt1 != rt2) { // disjoint concrete leaf types are never equal (quick test) return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0); @@ -3538,8 +3538,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva // not TIndex && not boxed implies it is an unboxed value of a different type from this singleton // (which was probably caught above, but just to be safe, we repeat it here explicitly) return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0); - Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : arg1.Vboxed; - Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : arg2.Vboxed; + Value *varg1 = arg1.constant ? literal_pointer_val(ctx, jl_pinned_ref_get(arg1.constant)) : arg1.Vboxed; + Value *varg2 = arg2.constant ? literal_pointer_val(ctx, jl_pinned_ref_get(arg2.constant)) : arg2.Vboxed; // rooting these values isn't needed since we won't load this pointer // and we know at least one of them is a unique Singleton // which is already enough to ensure pointer uniqueness for this test @@ -3561,8 +3561,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva if (typ == jl_bool_type) { // aka jl_pointer_egal // some optimizations for bool, since pointer comparison may be better if ((arg1.isboxed || arg1.constant) && (arg2.isboxed || arg2.constant)) { // aka have-fast-pointer - Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : arg1.Vboxed; - Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : arg2.Vboxed; + Value *varg1 = arg1.constant ? literal_pointer_val(ctx, jl_pinned_ref_get(arg1.constant)) : arg1.Vboxed; + Value *varg2 = arg2.constant ? literal_pointer_val(ctx, jl_pinned_ref_get(arg2.constant)) : arg2.Vboxed; return ctx.builder.CreateICmpEQ(decay_derived(ctx, varg1), decay_derived(ctx, varg2)); } } @@ -3588,8 +3588,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva // TODO: handle the case where arg1.typ is not exactly arg2.typ, or when // one of these isn't union, or when the union can be pointer - if (arg1.TIndex && arg2.TIndex && jl_egal(arg1.typ, arg2.typ) && - jl_is_uniontype(arg1.typ) && is_uniontype_allunboxed(arg1.typ)) + if (arg1.TIndex && arg2.TIndex && jl_egal(jl_pinned_ref_get(arg1.typ), jl_pinned_ref_get(arg2.typ)) && + jl_is_uniontype(jl_pinned_ref_get(arg1.typ)) && is_uniontype_allunboxed(jl_pinned_ref_get(arg1.typ))) return emit_nullcheck_guard2(ctx, nullcheck1, nullcheck2, [&] { return emit_bitsunion_compare(ctx, arg1, arg2); }); @@ -3617,7 +3617,7 @@ static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetglobal, true); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), !issetglobal, true); } enum jl_memory_order fail_order = order; if ((isreplaceglobal || issetglobalonce) && nargs == (isreplaceglobal ? 6 : 5)) { @@ -3625,7 +3625,7 @@ static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + fail_order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -3651,9 +3651,9 @@ static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - if (sym.constant && jl_is_symbol(sym.constant)) { - if (mod.constant && jl_is_module(mod.constant)) { - *ret = emit_globalop(ctx, (jl_module_t*)mod.constant, (jl_sym_t*)sym.constant, val, cmp, + if (jl_pinned_ref_get(sym.constant) && jl_is_symbol(jl_pinned_ref_get(sym.constant))) { + if (jl_pinned_ref_get(mod.constant) && jl_is_module(jl_pinned_ref_get(mod.constant))) { + *ret = emit_globalop(ctx, (jl_module_t*)jl_pinned_ref_get(mod.constant), (jl_sym_t*)jl_pinned_ref_get(sym.constant), val, cmp, get_llvm_atomic_order(order), get_llvm_atomic_order(fail_order), issetglobal, isreplaceglobal, @@ -3690,7 +3690,7 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), !issetfield, true); } enum jl_memory_order fail_order = order; if ((isreplacefield || issetfieldonce) && nargs == (isreplacefield ? 6 : 5)) { @@ -3698,7 +3698,7 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + fail_order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -3706,14 +3706,14 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - jl_datatype_t *uty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); + jl_datatype_t *uty = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(obj.typ)); if (jl_is_datatype(uty) && jl_struct_try_layout(uty)) { ssize_t idx = -1; - if (fld.constant && jl_is_symbol(fld.constant)) { - idx = jl_field_index(uty, (jl_sym_t*)fld.constant, 0); + if (fld.constant && jl_is_symbol(jl_pinned_ref_get(fld.constant))) { + idx = jl_field_index(uty, (jl_sym_t*)jl_pinned_ref_get(fld.constant), 0); } else if (fld.constant && fld.typ == (jl_value_t*)jl_long_type) { - ssize_t i = jl_unbox_long(fld.constant); + ssize_t i = jl_unbox_long(jl_pinned_ref_get(fld.constant)); if (i > 0 && i <= (ssize_t)jl_datatype_nfields(uty)) idx = i - 1; } @@ -3829,7 +3829,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_cgval_t undefval; const jl_cgval_t &ref = argv[1]; jl_cgval_t val = argv[isreplacememory || ismodifymemory ? 3 : 2]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (!jl_is_genericmemoryref_type(mty_dt) || !jl_is_concrete_type(mty_dt)) return false; @@ -3848,7 +3848,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetmemory, true); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), !issetmemory, true); } enum jl_memory_order fail_order = order; if (isreplacememory || issetmemoryonce) { @@ -3856,7 +3856,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + fail_order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -3864,7 +3864,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - jl_value_t *boundscheck = argv[nargs].constant; + jl_value_t *boundscheck = jl_pinned_ref_get(argv[nargs].constant); emit_typecheck(ctx, argv[nargs], (jl_value_t*)jl_bool_type, fname); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; bool isboxed = layout->flags.arrayelem_isboxed; @@ -3905,7 +3905,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } Value *mem = emit_memoryref_mem(ctx, ref, layout); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); if (bounds_check_enabled(ctx, boundscheck)) { BasicBlock *failBB, *endBB; failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); @@ -4025,9 +4025,9 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(typeof) && nargs == 1) { const jl_cgval_t &p = argv[1]; if (p.constant) - *ret = mark_julia_const(ctx, jl_typeof(p.constant)); - else if (jl_is_concrete_type(p.typ)) - *ret = mark_julia_const(ctx, p.typ); + *ret = mark_julia_const(ctx, jl_typeof(jl_pinned_ref_get(p.constant))); + else if (jl_is_concrete_type(jl_pinned_ref_get(p.typ))) + *ret = mark_julia_const(ctx, jl_pinned_ref_get(p.typ)); else *ret = mark_julia_type(ctx, emit_typeof(ctx, p, false, false), true, jl_datatype_type); return true; @@ -4036,13 +4036,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(typeassert) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; - if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { - jl_value_t *tp0 = jl_tparam0(ty.typ); + if (jl_is_type_type(jl_pinned_ref_get(ty.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ty.typ))) { + jl_value_t *tp0 = jl_tparam0(jl_pinned_ref_get(ty.typ)); emit_typecheck(ctx, arg, tp0, "typeassert"); *ret = update_julia_type(ctx, arg, tp0); return true; } - if (jl_subtype(ty.typ, (jl_value_t*)jl_type_type)) { + if (jl_subtype(jl_pinned_ref_get(ty.typ), (jl_value_t*)jl_type_type)) { Value *rt_arg = boxed(ctx, arg); Value *rt_ty = boxed(ctx, ty); ctx.builder.CreateCall(prepare_call(jltypeassert_func), {rt_arg, rt_ty}); @@ -4054,8 +4054,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(isa) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; - if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { - jl_value_t *tp0 = jl_tparam0(ty.typ); + if (jl_is_type_type(jl_pinned_ref_get(ty.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ty.typ))) { + jl_value_t *tp0 = jl_tparam0(jl_pinned_ref_get(ty.typ)); Value *isa_result = emit_isa(ctx, arg, tp0, Twine()).first; *ret = mark_julia_type(ctx, isa_result, false, jl_bool_type); return true; @@ -4065,9 +4065,9 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(issubtype) && nargs == 2) { const jl_cgval_t &ta = argv[1]; const jl_cgval_t &tb = argv[2]; - if (jl_is_type_type(ta.typ) && !jl_has_free_typevars(ta.typ) && - jl_is_type_type(tb.typ) && !jl_has_free_typevars(tb.typ)) { - int issub = jl_subtype(jl_tparam0(ta.typ), jl_tparam0(tb.typ)); + if (jl_is_type_type(jl_pinned_ref_get(ta.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ta.typ)) && + jl_is_type_type(jl_pinned_ref_get(tb.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(tb.typ))) { + int issub = jl_subtype(jl_tparam0(jl_pinned_ref_get(ta.typ)), jl_tparam0(jl_pinned_ref_get(tb.typ))); *ret = mark_julia_type(ctx, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), issub), false, jl_bool_type); return true; } @@ -4112,16 +4112,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_cgval_t &memty = argv[1]; if (!memty.constant) return false; - jl_datatype_t *typ = (jl_datatype_t*) memty.constant; + jl_datatype_t *typ = (jl_datatype_t*) jl_pinned_ref_get(memty.constant); if (!jl_is_concrete_type((jl_value_t*)typ) || !jl_is_genericmemory_type(typ)) return false; jl_genericmemory_t *inst = (jl_genericmemory_t*)((jl_datatype_t*)typ)->instance; if (inst == NULL) return false; if (argv[2].constant) { - if (!jl_is_long(argv[2].constant)) + if (!jl_is_long(jl_pinned_ref_get(argv[2].constant))) return false; - size_t nel = jl_unbox_long(argv[2].constant); + size_t nel = jl_unbox_long(jl_pinned_ref_get(argv[2].constant)); if (nel < 0) return false; *ret = emit_const_len_memorynew(ctx, typ, nel, inst); @@ -4134,7 +4134,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryrefnew) && nargs == 1) { const jl_cgval_t &mem = argv[1]; - jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(mem.typ); + jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(mem.typ)); if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(mty_dt->parameters), jl_svec_len(mty_dt->parameters)); const jl_datatype_layout_t *layout = mty_dt->layout; @@ -4145,11 +4145,11 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryrefnew) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &ref = argv[1]; - jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(ref.typ); + jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { mty_dt = (jl_datatype_t*)jl_field_type_concrete(mty_dt, 1); const jl_datatype_layout_t *layout = mty_dt->layout; - jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; + jl_value_t *boundscheck = nargs == 3 ? jl_pinned_ref_get(argv[3].constant) : nullptr; if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); @@ -4168,7 +4168,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryrefoffset) && nargs == 1) { const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; @@ -4179,7 +4179,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryrefget) && nargs == 3) { const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { jl_value_t *kind = jl_tparam0(mty_dt); jl_value_t *ety = jl_tparam1(mty_dt); @@ -4194,7 +4194,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -4215,11 +4215,11 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (order == jl_memory_order_unspecified) { order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } - jl_value_t *boundscheck = argv[3].constant; + jl_value_t *boundscheck = jl_pinned_ref_get(argv[3].constant); emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefget"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); if (bounds_check_enabled(ctx, boundscheck)) { BasicBlock *failBB, *endBB; failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); @@ -4304,7 +4304,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(memoryref_isassigned) && nargs == 3) { const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { jl_value_t *kind = jl_tparam0(mty_dt); mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); @@ -4317,7 +4317,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -4339,10 +4339,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (order == jl_memory_order_unspecified) { order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } - jl_value_t *boundscheck = argv[3].constant; + jl_value_t *boundscheck = jl_pinned_ref_get(argv[3].constant); emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, fname); Value *mem = emit_memoryref_mem(ctx, ref, layout); - Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; bool isboxed = layout->flags.arrayelem_isboxed; if (isboxed || layout->first_ptr >= 0) { @@ -4414,14 +4414,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, inb, (jl_value_t*)jl_bool_type, "getfield"); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); if (inb.constant == jl_false) boundscheck = jl_false; } else if (nargs == 3) { const jl_cgval_t &arg3 = argv[3]; - if (arg3.constant && jl_is_symbol(arg3.constant)) - order = jl_get_atomic_order((jl_sym_t*)arg3.constant, true, false); + if (arg3.constant && jl_is_symbol(jl_pinned_ref_get(arg3.constant))) + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(arg3.constant), true, false); else if (arg3.constant == jl_false) boundscheck = jl_false; else if (arg3.typ != (jl_value_t*)jl_bool_type) @@ -4433,14 +4433,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - jl_datatype_t *utt = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); + jl_datatype_t *utt = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(obj.typ)); if (jl_is_type_type((jl_value_t*)utt) && jl_is_concrete_type(jl_tparam0(utt))) utt = (jl_datatype_t*)jl_typeof(jl_tparam0(utt)); - if (fld.constant && jl_is_symbol(fld.constant)) { - jl_sym_t *name = (jl_sym_t*)fld.constant; - if (obj.constant && jl_is_module(obj.constant)) { - *ret = emit_globalref(ctx, (jl_module_t*)obj.constant, name, order == jl_memory_order_unspecified ? AtomicOrdering::Unordered : get_llvm_atomic_order(order)); + if (fld.constant && jl_is_symbol(jl_pinned_ref_get(fld.constant))) { + jl_sym_t *name = (jl_sym_t*)jl_pinned_ref_get(fld.constant); + if (obj.constant && jl_is_module(jl_pinned_ref_get(obj.constant))) { + *ret = emit_globalref(ctx, (jl_module_t*)jl_pinned_ref_get(obj.constant), name, order == jl_memory_order_unspecified ? AtomicOrdering::Unordered : get_llvm_atomic_order(order)); return true; } @@ -4480,7 +4480,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, size_t nfields = jl_datatype_nfields(utt); // integer index size_t idx; - if (fld.constant && (idx = jl_unbox_long(fld.constant) - 1) < nfields) { + if (fld.constant && (idx = jl_unbox_long(jl_pinned_ref_get(fld.constant)) - 1) < nfields) { if (!jl_has_free_typevars(jl_field_type(utt, idx))) { // known index *ret = emit_getfield_knownidx(ctx, obj, idx, utt, order); @@ -4517,7 +4517,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, vidx = ctx.builder.CreateSub(vidx, ConstantInt::get(ctx.types().T_size, 1)); } else { - vidx = emit_bounds_check(ctx, ptrobj, (jl_value_t*)ptrobj.typ, vidx, + vidx = emit_bounds_check(ctx, ptrobj, (jl_value_t*)jl_pinned_ref_get(ptrobj.typ), vidx, emit_datatype_nfields(ctx, emit_typeof(ctx, ptrobj, false, false)), jl_true); } @@ -4568,8 +4568,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (nargs == 3) { const jl_cgval_t &arg3 = argv[3]; - if (arg3.constant && jl_is_symbol(arg3.constant)) - order = jl_get_atomic_order((jl_sym_t*)arg3.constant, true, false); + if (arg3.constant && jl_is_symbol(jl_pinned_ref_get(arg3.constant))) + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(arg3.constant), true, false); else return false; } @@ -4582,10 +4582,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - if (sym.constant && jl_is_symbol(sym.constant)) { - jl_sym_t *name = (jl_sym_t*)sym.constant; - if (mod.constant && jl_is_module(mod.constant)) { - *ret = emit_globalref(ctx, (jl_module_t*)mod.constant, name, get_llvm_atomic_order(order)); + if (sym.constant && jl_is_symbol(jl_pinned_ref_get(sym.constant))) { + jl_sym_t *name = (jl_sym_t*)jl_pinned_ref_get(sym.constant); + if (mod.constant && jl_is_module(jl_pinned_ref_get(mod.constant))) { + *ret = emit_globalref(ctx, (jl_module_t*)jl_pinned_ref_get(mod.constant), name, get_llvm_atomic_order(order)); return true; } } @@ -4622,15 +4622,15 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } ssize_t nf = -1; if (obj.constant) { - nf = jl_datatype_nfields(jl_typeof(obj.constant)); + nf = jl_datatype_nfields(jl_typeof(jl_pinned_ref_get(obj.constant))); } - else if (jl_is_type_type(obj.typ)) { - jl_value_t *tp0 = jl_tparam0(obj.typ); + else if (jl_is_type_type(jl_pinned_ref_get(obj.typ))) { + jl_value_t *tp0 = jl_tparam0(jl_pinned_ref_get(obj.typ)); if (jl_is_datatype(tp0) && jl_is_datatype_singleton((jl_datatype_t*)tp0)) nf = jl_datatype_nfields((jl_value_t*)jl_datatype_type); } - else if (jl_is_concrete_type(obj.typ)) { - nf = jl_datatype_nfields(obj.typ); + else if (jl_is_concrete_type(jl_pinned_ref_get(obj.typ))) { + nf = jl_datatype_nfields(jl_pinned_ref_get(obj.typ)); } Value *sz; if (nf != -1) @@ -4644,15 +4644,15 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(fieldtype) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &typ = argv[1]; const jl_cgval_t &fld = argv[2]; - if ((jl_is_type_type(typ.typ) && jl_is_concrete_type(jl_tparam0(typ.typ))) || - (typ.constant && jl_is_concrete_type(typ.constant))) { + if ((jl_is_type_type(jl_pinned_ref_get(typ.typ)) && jl_is_concrete_type(jl_tparam0(jl_pinned_ref_get(typ.typ)))) || + (typ.constant && jl_is_concrete_type(jl_pinned_ref_get(typ.constant)))) { if (fld.typ == (jl_value_t*)jl_long_type) { assert(typ.isboxed); Value *tyv = boxed(ctx, typ); Value *types_svec = emit_datatype_types(ctx, tyv); Value *types_len = emit_datatype_nfields(ctx, tyv); Value *idx = emit_unbox(ctx, ctx.types().T_size, fld, (jl_value_t*)jl_long_type); - jl_value_t *boundscheck = (nargs == 3 ? argv[3].constant : jl_true); + jl_value_t *boundscheck = (nargs == 3 ? jl_pinned_ref_get(argv[3].constant) : jl_true); if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "fieldtype"); emit_bounds_check(ctx, typ, (jl_value_t*)jl_datatype_type, idx, types_len, boundscheck); @@ -4668,16 +4668,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == BUILTIN(sizeof) && nargs == 1) { const jl_cgval_t &obj = argv[1]; - jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); + jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(obj.typ)); assert(jl_string_type->name->mutabl); if (sty == jl_string_type || sty == jl_simplevector_type) { if (obj.constant) { size_t sz; if (sty == jl_string_type) { - sz = jl_string_len(obj.constant); + sz = jl_string_len(jl_pinned_ref_get(obj.constant)); } else { - sz = (1 + jl_svec_len(obj.constant)) * sizeof(void*); + sz = (1 + jl_svec_len(jl_pinned_ref_get(obj.constant))) * sizeof(void*); } *ret = mark_julia_type(ctx, ConstantInt::get(ctx.types().T_size, sz), false, jl_long_type); return true; @@ -4705,7 +4705,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (jl_is_genericmemory_type(sty)) { Value *v = boxed(ctx, obj); auto len = emit_genericmemorylen(ctx, v, (jl_value_t*)sty); - auto elsize = emit_genericmemoryelsize(ctx, v, obj.typ, true); + auto elsize = emit_genericmemoryelsize(ctx, v, jl_pinned_ref_get(obj.typ), true); *ret = mark_julia_type(ctx, ctx.builder.CreateMul(len, elsize), false, jl_long_type); return true; } @@ -4733,16 +4733,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (nargs >= 3) { const jl_cgval_t &arg3 = argv[3]; - if (arg3.constant && jl_is_bool(arg3.constant)) - allow_import = jl_unbox_bool(arg3.constant); + if (arg3.constant && jl_is_bool(jl_pinned_ref_get(arg3.constant))) + allow_import = jl_unbox_bool(jl_pinned_ref_get(arg3.constant)); else return false; } if (nargs == 4) { const jl_cgval_t &arg4 = argv[4]; - if (arg4.constant && jl_is_symbol(arg4.constant)) - order = jl_get_atomic_order((jl_sym_t*)arg4.constant, true, false); + if (arg4.constant && jl_is_symbol(jl_pinned_ref_get(arg4.constant))) + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(arg4.constant), true, false); else return false; } @@ -4753,18 +4753,18 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - if (!mod.constant || !sym.constant || !jl_is_symbol(sym.constant) || !jl_is_module(mod.constant)) { + if (!mod.constant || !sym.constant || !jl_is_symbol(jl_pinned_ref_get(sym.constant)) || !jl_is_module(jl_pinned_ref_get(mod.constant))) { return false; } - *ret = emit_isdefinedglobal(ctx, (jl_module_t*)mod.constant, (jl_sym_t*)sym.constant, allow_import, order); + *ret = emit_isdefinedglobal(ctx, (jl_module_t*)jl_pinned_ref_get(mod.constant), (jl_sym_t*)jl_pinned_ref_get(sym.constant), allow_import, order); return true; } else if (f == BUILTIN(isdefined) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; - jl_datatype_t *stt = (jl_datatype_t*)obj.typ; + jl_datatype_t *stt = (jl_datatype_t*)jl_pinned_ref_get(obj.typ); ssize_t fieldidx = -1; if (jl_is_type_type((jl_value_t*)stt)) { // the representation type of Type{T} is either typeof(T), or unknown @@ -4781,12 +4781,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } assert(jl_is_datatype(stt)); - if (fld.constant && jl_is_symbol(fld.constant)) { - jl_sym_t *sym = (jl_sym_t*)fld.constant; + if (fld.constant && jl_is_symbol(jl_pinned_ref_get(fld.constant))) { + jl_sym_t *sym = (jl_sym_t*)jl_pinned_ref_get(fld.constant); fieldidx = jl_field_index(stt, sym, 0); } else if (fld.constant && fld.typ == (jl_value_t*)jl_long_type) { - fieldidx = jl_unbox_long(fld.constant) - 1; + fieldidx = jl_unbox_long(jl_pinned_ref_get(fld.constant)) - 1; } else { isdefined_unknown_idx: @@ -4805,7 +4805,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, "isdefined"); if (!ord.constant) return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); } if (order == jl_memory_order_invalid) { emit_atomic_error(ctx, "invalid atomic ordering"); @@ -5196,11 +5196,11 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR if (lival.constant) { jl_method_instance_t *mi; jl_value_t *ci = nullptr; - if (jl_is_method_instance(lival.constant)) { - mi = (jl_method_instance_t*)lival.constant; + if (jl_is_method_instance(jl_pinned_ref_get(lival.constant))) { + mi = (jl_method_instance_t*)jl_pinned_ref_get(lival.constant); } - else if (jl_is_code_instance(lival.constant)) { - ci = lival.constant; + else if (jl_is_code_instance(jl_pinned_ref_get(lival.constant))) { + ci = jl_pinned_ref_get(lival.constant); mi = jl_get_ci_mi((jl_code_instance_t*)ci); } else { emit_error(ctx, "(Internal ERROR - IR Validity): Invoke target is not a method instance or code instance"); @@ -5357,8 +5357,8 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } - else if (jl_typetagis(f.constant, jl_intrinsic_type)) { - JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); + else if (jl_typetagis(jl_pinned_ref_get(f.constant), jl_intrinsic_type)) { + JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(jl_pinned_ref_get(f.constant)); if (fi == JL_I::atomic_pointermodify && jl_intrinsic_nargs((int)fi) == nargs - 1) return emit_atomic_pointerop(ctx, fi, ArrayRef(argv).drop_front(), nargs - 1, &lival); } @@ -5416,8 +5416,8 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo // a couple intrinsics (really just llvmcall, though partly cglobal too) // have non-standard (aka invalid) evaluation semantics, so we must handle these first - if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { - JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); + if (f.constant && jl_typetagis(jl_pinned_ref_get(f.constant), jl_intrinsic_type)) { + JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(jl_pinned_ref_get(f.constant)); return emit_intrinsic(ctx, fi, args, nargs - 1); } @@ -5449,14 +5449,14 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo } // handle calling an OpaqueClosure - if (jl_is_concrete_type(f.typ) && jl_subtype(f.typ, (jl_value_t*)jl_opaque_closure_type)) { - jl_value_t *oc_argt = jl_tparam0(f.typ); - jl_value_t *oc_rett = jl_tparam1(f.typ); + if (jl_is_concrete_type(jl_pinned_ref_get(f.typ)) && jl_subtype(jl_pinned_ref_get(f.typ), (jl_value_t*)jl_opaque_closure_type)) { + jl_value_t *oc_argt = jl_tparam0(jl_pinned_ref_get(f.typ)); + jl_value_t *oc_rett = jl_tparam1(jl_pinned_ref_get(f.typ)); if (jl_is_datatype(oc_argt) && jl_tupletype_length_compat(oc_argt, nargs-1)) { - jl_value_t *sigtype = jl_argtype_with_function_type((jl_value_t*)f.typ, (jl_value_t*)oc_argt); + jl_value_t *sigtype = jl_argtype_with_function_type((jl_value_t*)jl_pinned_ref_get(f.typ), (jl_value_t*)oc_argt); if (uses_specsig(sigtype, false, oc_rett, true)) { JL_GC_PUSH1(&sigtype); - jl_cgval_t r = emit_specsig_oc_call(ctx, f.typ, sigtype, argv, nargs); + jl_cgval_t r = emit_specsig_oc_call(ctx, jl_pinned_ref_get(f.typ), sigtype, argv, nargs); JL_GC_POP(); return r; } @@ -5607,7 +5607,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va Value *tindex = NULL; if (vi.pTIndex) tindex = ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), vi.pTIndex, Align(1), vi.isVolatile); - v = mark_julia_slot(ssaslot, vi.value.typ, tindex, ctx.tbaa().tbaa_stack, None); + v = mark_julia_slot(ssaslot, jl_pinned_ref_get(vi.value.typ), tindex, ctx.tbaa().tbaa_stack, None); } if (vi.inline_roots) { AllocaInst *varslot = vi.inline_roots; @@ -5630,7 +5630,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va Value *box_isnull = NULL; if (vi.usedUndef) box_isnull = ctx.builder.CreateICmpNE(boxed, Constant::getNullValue(ctx.types().T_prjlvalue)); - maybe_mark_load_dereferenceable(boxed, vi.usedUndef || vi.pTIndex, vi.value.typ); + maybe_mark_load_dereferenceable(boxed, vi.usedUndef || vi.pTIndex, jl_pinned_ref_get(vi.value.typ)); if (vi.pTIndex) { // value is either boxed in the stack slot, or unboxed in value // as indicated by testing (pTIndex & UNION_BOX_MARKER) @@ -5646,7 +5646,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va v.Vboxed = boxed; } else { - v = mark_julia_type(ctx, boxed, true, vi.value.typ); + v = mark_julia_type(ctx, boxed, true, jl_pinned_ref_get(vi.value.typ)); if (vi.usedUndef) isnull = box_isnull; } @@ -5680,7 +5680,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu if (!vi.value.constant) { // check that this is not a virtual store assert(vi.inline_roots || vi.value.ispointer() || (vi.pTIndex && vi.value.V == NULL)); // store value - rval_info = update_julia_type(ctx, rval_info, vi.value.typ); + rval_info = update_julia_type(ctx, rval_info, jl_pinned_ref_get(vi.value.typ)); if (rval_info.typ == jl_bottom_type) return; if (vi.pTIndex && vi.value.V) // TODO: use lifetime-end here instead @@ -5694,7 +5694,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu if (rval_info.TIndex) emit_unionmove(ctx, vi.value.V, tbaa, rval_info, /*skip*/isboxed, vi.isVolatile); else { - Align align(julia_alignment(rval_info.typ)); + Align align(julia_alignment(jl_pinned_ref_get(rval_info.typ))); if (vi.inline_roots) split_value_into(ctx, rval_info, align, vi.value.V, align, jl_aliasinfo_t::fromTBAA(ctx, tbaa), vi.inline_roots, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe), vi.isVolatile); else @@ -5862,7 +5862,7 @@ static void emit_varinfo_assign(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_cgval_t return; // convert rval-type to lval-type - jl_value_t *slot_type = vi.value.typ; + jl_value_t *slot_type = jl_pinned_ref_get(vi.value.typ); // If allow_mismatch is set, type mismatches will not result in traps. // This is used for upsilon nodes, where the destination can have a narrower // type than the store, if inference determines that the store is never read. @@ -5886,7 +5886,7 @@ static void emit_varinfo_assign(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_cgval_t } else { assert(rval_info.isboxed || rval_info.constant); - tindex = compute_tindex_unboxed(ctx, rval_info, vi.value.typ); + tindex = compute_tindex_unboxed(ctx, rval_info, jl_pinned_ref_get(vi.value.typ)); if (vi.boxroot) tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), UNION_BOX_MARKER)); else @@ -6029,7 +6029,7 @@ static Value *emit_condition(jl_codectx_t &ctx, const jl_cgval_t &condV, const T if (!isbool) { if (condV.TIndex) { // check whether this might be bool - isbool = jl_subtype((jl_value_t*)jl_bool_type, condV.typ); + isbool = jl_subtype((jl_value_t*)jl_bool_type, jl_pinned_ref_get(condV.typ)); } emit_typecheck(ctx, condV, (jl_value_t*)jl_bool_type, msg); } @@ -6475,7 +6475,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ for (size_t i = 0; i < nargs; ++i) { argv[i] = emit_expr(ctx, args[i]); } - jl_value_t *ty = argv[0].typ; + jl_value_t *ty = jl_pinned_ref_get(argv[0].typ); if (jl_is_type_type(ty) && jl_is_datatype(jl_tparam0(ty)) && jl_is_concrete_type(jl_tparam0(ty))) { @@ -6519,11 +6519,11 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ emit_error(ctx, "(INTERNAL ERROR - IR Validity): opaque closure source must be constant"); return jl_cgval_t(); } - bool can_optimize = argt.constant != NULL && lb.constant != NULL && ub.constant != NULL && - jl_is_tuple_type(argt.constant) && - jl_is_type(lb.constant) && jl_is_type(ub.constant) && jl_is_method(source.constant) && - ((jl_method_t*)source.constant)->nargs > 0 && - jl_is_valid_oc_argtype((jl_tupletype_t*)argt.constant, (jl_method_t*)source.constant); + bool can_optimize = jl_pinned_ref_get(argt.constant) != NULL && jl_pinned_ref_get(lb.constant) != NULL && jl_pinned_ref_get(ub.constant) != NULL && + jl_is_tuple_type(jl_pinned_ref_get(argt.constant)) && + jl_is_type(jl_pinned_ref_get(lb.constant)) && jl_is_type(jl_pinned_ref_get(ub.constant)) && jl_is_method(jl_pinned_ref_get(source.constant)) && + ((jl_method_t*)jl_pinned_ref_get(source.constant))->nargs > 0 && + jl_is_valid_oc_argtype((jl_tupletype_t*)jl_pinned_ref_get(argt.constant), (jl_method_t*)jl_pinned_ref_get(source.constant)); if (can_optimize) { @@ -6534,7 +6534,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ size_t ncapture_args = nargs-5; SmallVector env_component_ts(ncapture_args); for (size_t i = 0; i < ncapture_args; ++i) { - jl_value_t *typ = argv[nargs-ncapture_args+i].typ; + jl_value_t *typ = jl_pinned_ref_get(argv[nargs-ncapture_args+i].typ); if (typ == jl_bottom_type) { JL_GC_POP(); return jl_cgval_t(); @@ -6545,9 +6545,9 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ env_t = jl_apply_tuple_type_v(env_component_ts.data(), ncapture_args); // we need to know the full env type to look up the right specialization if (jl_is_concrete_type(env_t)) { - jl_tupletype_t *argt_typ = (jl_tupletype_t*)argt.constant; + jl_tupletype_t *argt_typ = (jl_tupletype_t*)jl_pinned_ref_get(argt.constant); Function *F, *specF; - std::tie(F, specF) = get_oc_function(ctx, (jl_method_t*)source.constant, (jl_tupletype_t*)env_t, argt_typ, ub.constant); + std::tie(F, specF) = get_oc_function(ctx, (jl_method_t*)jl_pinned_ref_get(source.constant), (jl_tupletype_t*)env_t, argt_typ, jl_pinned_ref_get(ub.constant)); if (F) { jl_cgval_t jlcall_ptr = mark_julia_type(ctx, F, false, jl_voidpointer_type); jl_cgval_t world_age = mark_julia_type(ctx, get_tls_world_age(ctx), false, jl_long_type); @@ -6568,7 +6568,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ fptr }; - closure_t = jl_apply_type2((jl_value_t*)jl_opaque_closure_type, (jl_value_t*)argt_typ, ub.constant); + closure_t = jl_apply_type2((jl_value_t*)jl_opaque_closure_type, (jl_value_t*)argt_typ, jl_pinned_ref_get(ub.constant)); jl_cgval_t ret = emit_new_struct(ctx, closure_t, 5, closure_fields); JL_GC_POP(); @@ -7526,7 +7526,7 @@ static Function *gen_cfun_wrapper( else if (!type_is_ghost(sig.lrt)) { Type *prt = sig.prt; bool issigned = jl_signed_type && jl_subtype(declrt, (jl_value_t*)jl_signed_type); - Value *v = emit_unbox(ctx, sig.lrt, retval, retval.typ); + Value *v = emit_unbox(ctx, sig.lrt, retval, jl_pinned_ref_get(retval.typ)); r = llvm_type_rewrite(ctx, v, prt, issigned); if (sig.sret) { ctx.builder.CreateStore(r, sretPtr); @@ -7656,7 +7656,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con bool approx = false; sigt = (jl_value_t*)jl_alloc_svec(nargt + 1); jl_svecset(sigt, 0, fexpr_val.typ); - if (!fexpr_val.constant && (!jl_is_concrete_type(fexpr_val.typ) || jl_is_kind(fexpr_val.typ))) + if (!fexpr_val.constant && (!jl_is_concrete_type(jl_pinned_ref_get(fexpr_val.typ)) || jl_is_kind(jl_pinned_ref_get(fexpr_val.typ)))) approx = true; for (size_t i = 0; i < nargt; i++) { jl_value_t *jargty = jl_svecref(argt, i); @@ -7693,10 +7693,10 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } } - const char *name = derive_sigt_name(fexpr_val.typ); + const char *name = derive_sigt_name(jl_pinned_ref_get(fexpr_val.typ)); Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, - sig, fexpr_val.constant, name, + sig, jl_pinned_ref_get(fexpr_val.constant), name, declrt, sigt, unionall_env, sparam_vals, &closure_types); bool outboxed; @@ -7743,7 +7743,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.decorateInst(ctx.builder.CreateStore(F, derived_strct)); ai.decorateInst(ctx.builder.CreateStore( - ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, fexpr_val.constant), ctx.types().T_size), + ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, jl_pinned_ref_get(fexpr_val.constant)), ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 1))); ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 2))); @@ -8477,7 +8477,7 @@ static jl_llvm_functions_t topfile, // File toplineno == -1 ? 0 : toplineno, // Line // Variable type - julia_type_to_di(ctx, debugcache, varinfo.value.typ, &dbuilder, false), + julia_type_to_di(ctx, debugcache, jl_pinned_ref_get(varinfo.value.typ), &dbuilder, false), AlwaysPreserve, // May be deleted if optimized out DINode::FlagZero); // Flags (TODO: Do we need any) } @@ -8488,7 +8488,7 @@ static jl_llvm_functions_t has_sret + nreq + 1, // Argument number (1-based) topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx, debugcache, ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), + julia_type_to_di(ctx, debugcache, jl_pinned_ref_get(ctx.slots[ctx.vaSlot].value.typ), &dbuilder, false), AlwaysPreserve, // May be deleted if optimized out DINode::FlagZero); // Flags (TODO: Do we need any) } @@ -8503,7 +8503,7 @@ static jl_llvm_functions_t jl_symbol_name(s), // Variable name topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx, debugcache, varinfo.value.typ, &dbuilder, false), // Variable type + julia_type_to_di(ctx, debugcache, jl_pinned_ref_get(varinfo.value.typ), &dbuilder, false), // Variable type AlwaysPreserve, // May be deleted if optimized out DINode::FlagZero // Flags (TODO: Do we need any) ); @@ -8597,7 +8597,7 @@ static jl_llvm_functions_t // step 7. allocate local variables slots // must be in the first basic block for the llvm mem2reg pass to work auto allocate_local = [&ctx, &dbuilder, &debugcache, topdebugloc, va, debug_enabled](jl_varinfo_t &varinfo, jl_sym_t *s, int i) { - jl_value_t *jt = varinfo.value.typ; + jl_value_t *jt = jl_pinned_ref_get(varinfo.value.typ); assert(!varinfo.boxroot); // variables shouldn't have memory locs already if (varinfo.value.constant) { // no need to explicitly load/store a constant/ghost value @@ -8790,7 +8790,7 @@ static jl_llvm_functions_t // Load closure env, which is always a boxed value (usually some Tuple) currently Value *envaddr = emit_ptrgep(ctx, oc_this, offsetof(jl_opaque_closure_t, captures)); - theArg = typed_load(ctx, envaddr, NULL, (jl_value_t*)vi.value.typ, + theArg = typed_load(ctx, envaddr, NULL, (jl_value_t*)jl_pinned_ref_get(vi.value.typ), nullptr, nullptr, /*isboxed*/true, AtomicOrdering::NotAtomic, false, sizeof(void*)); } else { @@ -8816,15 +8816,15 @@ static jl_llvm_functions_t else { if (i == 0) { // first (function) arg is separate in jlcall - theArg = mark_julia_type(ctx, fArg, true, vi.value.typ); + theArg = mark_julia_type(ctx, fArg, true, jl_pinned_ref_get(vi.value.typ)); } else { Value *argPtr = emit_ptrgep(ctx, argArray, (i - 1) * ctx.types().sizeof_ptr); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *load = ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, argPtr, Align(sizeof(void*))), - false, vi.value.typ)); - theArg = mark_julia_type(ctx, load, true, vi.value.typ); + false, jl_pinned_ref_get(vi.value.typ))); + theArg = mark_julia_type(ctx, load, true, jl_pinned_ref_get(vi.value.typ)); if (debug_enabled && vi.dinfo && !vi.boxroot) { SmallVector addr; addr.push_back(llvm::dwarf::DW_OP_deref); @@ -8877,14 +8877,14 @@ static jl_llvm_functions_t Type *llvmArgType = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, argType); vargs[i - nreq] = get_specsig_arg(argType, llvmArgType, isboxed); } - if (jl_is_concrete_type(vi.value.typ)) { - jl_cgval_t tuple = emit_new_struct(ctx, vi.value.typ, ctx.nvargs, vargs); + if (jl_is_concrete_type(jl_pinned_ref_get(vi.value.typ))) { + jl_cgval_t tuple = emit_new_struct(ctx, jl_pinned_ref_get(vi.value.typ), ctx.nvargs, vargs); emit_varinfo_assign(ctx, vi, tuple); } else { restTuple = emit_jlcall(ctx, jltuple_func, Constant::getNullValue(ctx.types().T_prjlvalue), vargs, ctx.nvargs, julia_call); - jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, vi.value.typ); + jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, jl_pinned_ref_get(vi.value.typ)); emit_varinfo_assign(ctx, vi, tuple); } } @@ -9452,7 +9452,7 @@ static jl_llvm_functions_t SmallVector roots; BasicBlock *PhiBB; std::tie(phi_result, PhiBB, dest, VN, roots, r) = tup; - jl_value_t *phiType = phi_result.typ; + jl_value_t *phiType = jl_pinned_ref_get(phi_result.typ); jl_array_t *edges = (jl_array_t*)jl_fieldref_noalloc(jl_pinned_ref_get(r), 0); jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(jl_pinned_ref_get(r), 1); PHINode *TindexN = cast_or_null(phi_result.TIndex); @@ -9506,7 +9506,7 @@ static jl_llvm_functions_t ctx.builder.CreateLifetimeStart(dest); jl_cgval_t val = emit_expr(ctx, value); if (val.constant) - val = mark_julia_const(ctx, val.constant); // be over-conservative at making sure `.typ` is set concretely, not tindex + val = mark_julia_const(ctx, jl_pinned_ref_get(val.constant)); // be over-conservative at making sure `.typ` is set concretely, not tindex if (!jl_is_uniontype(phiType) || !TindexN) { if (VN) { assert(roots.empty() && !dest); @@ -9572,8 +9572,8 @@ static jl_llvm_functions_t V = undef_value_for_type(VN->getType()); RTindex = UndefValue::get(getInt8Ty(ctx.builder.getContext())); } - else if (jl_is_concrete_type(val.typ) || val.constant) { - size_t tindex = get_box_tindex((jl_datatype_t*)(val.constant ? jl_typeof(val.constant) : val.typ), phiType); + else if (jl_is_concrete_type(jl_pinned_ref_get(val.typ)) || val.constant) { + size_t tindex = get_box_tindex((jl_datatype_t*)(val.constant ? jl_typeof(jl_pinned_ref_get(val.constant)) : val.typ), phiType); if (tindex == 0) { if (VN) V = boxed(ctx, val); @@ -9583,7 +9583,7 @@ static jl_llvm_functions_t if (VN) V = Constant::getNullValue(ctx.types().T_prjlvalue); if (dest) { - Align align(julia_alignment(val.typ)); + Align align(julia_alignment(jl_pinned_ref_get(val.typ))); emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, align, align); } RTindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), tindex); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ae25c3cc83ca5..dfb6ff116ec50 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -451,7 +451,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va return UndefValue::get(to); // type mismatch error } - Constant *c = x.constant ? julia_const_to_llvm(ctx, x.constant) : nullptr; + Constant *c = jl_pinned_ref_get(x.constant) ? julia_const_to_llvm(ctx, jl_pinned_ref_get(x.constant)) : nullptr; if ((x.inline_roots.empty() && !x.ispointer()) || c != nullptr) { // already unboxed, but sometimes need conversion Value *unboxed = c ? c : x.V; assert(unboxed); // clang-sa doesn't know that !x.ispointer() implies x.V does have a value @@ -459,7 +459,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va } // bools stored as int8, so an extra Trunc is needed to get an int1 - Value *p = x.constant ? literal_pointer_val(ctx, x.constant) : x.V; + Value *p = jl_pinned_ref_get(x.constant) ? literal_pointer_val(ctx, jl_pinned_ref_get(x.constant)) : x.V; if (jt == (jl_value_t*)jl_bool_type || to->isIntegerTy(1)) { assert(p && x.inline_roots.empty()); // clang-sa doesn't know that x.ispointer() implied these are true @@ -522,13 +522,13 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest Value *src = data_pointer(ctx, x); auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); - emit_memcpy(ctx, dest, dest_ai, src, src_ai, jl_datatype_size(x.typ), Align(align_dst), align_src ? *align_src : Align(julia_alignment(x.typ)), isVolatile); + emit_memcpy(ctx, dest, dest_ai, src, src_ai, jl_datatype_size(jl_pinned_ref_get(x.typ)), Align(align_dst), align_src ? *align_src : Align(julia_alignment(jl_pinned_ref_get(x.typ))), isVolatile); } static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) { // evaluate an argument at compile time to determine what type it is - jl_value_t *unw = jl_unwrap_unionall(targ.typ); + jl_value_t *unw = jl_unwrap_unionall(jl_pinned_ref_get(targ.typ)); if (jl_is_type_type(unw)) { jl_value_t *bt = jl_tparam0(unw); if (jl_is_primitivetype(bt)) @@ -571,11 +571,11 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) // Examine the second argument // bool isboxed; - Type *vxt = julia_type_to_llvm(ctx, v.typ, &isboxed); - if (!jl_is_primitivetype(v.typ) || jl_datatype_size(v.typ) != nb) { + Type *vxt = julia_type_to_llvm(ctx, jl_pinned_ref_get(v.typ), &isboxed); + if (!jl_is_primitivetype(v.typ) || jl_datatype_size(jl_pinned_ref_get(v.typ)) != nb) { Value *typ = emit_typeof(ctx, v, false, false); if (!jl_is_primitivetype(v.typ)) { - if (jl_is_datatype(v.typ) && !jl_is_abstracttype(v.typ)) { + if (jl_is_datatype(jl_pinned_ref_get(v.typ)) && !jl_is_abstracttype(v.typ)) { emit_error(ctx, "bitcast: value not a primitive type"); return jl_cgval_t(); } @@ -584,7 +584,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) error_unless(ctx, isprimitive, "bitcast: value not a primitive type"); } } - if (jl_is_datatype(v.typ) && !jl_is_abstracttype(v.typ)) { + if (jl_is_datatype(jl_pinned_ref_get(v.typ)) && !jl_is_abstracttype(v.typ)) { emit_error(ctx, "bitcast: argument size does not match size of target type"); return jl_cgval_t(); } @@ -603,7 +603,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) if (!v.ispointer()) vx = v.V; else if (v.constant) - vx = julia_const_to_llvm(ctx, v.constant); + vx = julia_const_to_llvm(ctx, jl_pinned_ref_get(v.constant)); if (v.ispointer() && vx == NULL) { // try to load as original Type, to preserve llvm optimizations @@ -670,11 +670,11 @@ static jl_cgval_t generic_cast( const jl_cgval_t &targ = argv[0]; const jl_cgval_t &v = argv[1]; jl_datatype_t *jlto = staticeval_bitstype(targ); - if (!jlto || !jl_is_primitivetype(v.typ)) + if (!jlto || !jl_is_primitivetype(jl_pinned_ref_get(v.typ))) return emit_runtime_call(ctx, f, argv, 2); uint32_t nb = jl_datatype_size(jlto); Type *to = bitstype_to_llvm((jl_value_t*)jlto, ctx.builder.getContext(), true); - Type *vt = bitstype_to_llvm(v.typ, ctx.builder.getContext(), true); + Type *vt = bitstype_to_llvm(jl_pinned_ref_get(v.typ), ctx.builder.getContext(), true); // fptrunc and fpext depend on the specific floating point // format to work correctly, and so do not pun their argument types. @@ -698,7 +698,7 @@ static jl_cgval_t generic_cast( if (!to || !vt) return emit_runtime_call(ctx, f, argv, 2); - Value *from = emit_unbox(ctx, vt, v, v.typ); + Value *from = emit_unbox(ctx, vt, v, jl_pinned_ref_get(v.typ)); if (!CastInst::castIsValid(Op, from, to)) return emit_runtime_call(ctx, f, argv, 2); if (Op == Instruction::FPExt) { @@ -745,13 +745,13 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) const jl_cgval_t &i = argv[1]; const jl_cgval_t &align = argv[2]; - if (align.constant == NULL || !jl_is_long(align.constant)) + if (align.constant == NULL || !jl_is_long(jl_pinned_ref_get(align.constant))) return emit_runtime_pointerref(ctx, argv); - unsigned align_nb = jl_unbox_long(align.constant); + unsigned align_nb = jl_unbox_long(jl_pinned_ref_get(align.constant)); if (i.typ != (jl_value_t*)jl_long_type) return emit_runtime_pointerref(ctx, argv); - jl_value_t *aty = e.typ; + jl_value_t *aty = jl_pinned_ref_get(e.typ); if (!jl_is_cpointer_type(aty)) return emit_runtime_pointerref(ctx, argv); jl_value_t *ety = jl_tparam0(aty); @@ -767,7 +767,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) setName(ctx.emission_context, im1, "pointerref_idx"); if (ety == (jl_value_t*)jl_any_type) { - Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); + Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, jl_pinned_ref_get(e.typ)); if (isa(thePtr) && !thePtr->hasName()) setName(ctx.emission_context, thePtr, "unbox_any_ptr"); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, thePtr, im1), Align(align_nb)); @@ -784,7 +784,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) im1 = ctx.builder.CreateMul(im1, ConstantInt::get(ctx.types().T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); setName(ctx.emission_context, im1, "pointerref_offset"); - Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, jl_pinned_ref_get(e.typ)); thePtr = emit_ptrgep(ctx, thePtr, im1); setName(ctx.emission_context, thePtr, "pointerref_src"); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); @@ -796,7 +796,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed); assert(!isboxed); if (!type_is_ghost(ptrty)) { - Value *thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, jl_pinned_ref_get(e.typ)); thePtr = ctx.builder.CreateInBoundsGEP(ptrty, thePtr, im1); auto load = typed_load(ctx, thePtr, nullptr, ety, ctx.tbaa().tbaa_data, nullptr, isboxed, AtomicOrdering::NotAtomic, false, align_nb); setName(ctx.emission_context, load.V, "pointerref"); @@ -821,19 +821,19 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) const jl_cgval_t &i = argv[2]; const jl_cgval_t &align = argv[3]; - if (align.constant == NULL || !jl_is_long(align.constant)) + if (align.constant == NULL || !jl_is_long(jl_pinned_ref_get(align.constant))) return emit_runtime_pointerset(ctx, argv); - unsigned align_nb = jl_unbox_long(align.constant); + unsigned align_nb = jl_unbox_long(jl_pinned_ref_get(align.constant)); if (i.typ != (jl_value_t*)jl_long_type) return emit_runtime_pointerset(ctx, argv); - jl_value_t *aty = e.typ; + jl_value_t *aty = jl_pinned_ref_get(e.typ); if (!jl_is_cpointer_type(aty)) return emit_runtime_pointerset(ctx, argv); jl_value_t *ety = jl_tparam0(aty); if (jl_is_typevar(ety)) return emit_runtime_pointerset(ctx, argv); - if (align.constant == NULL || !jl_is_long(align.constant)) + if (align.constant == NULL || !jl_is_long(jl_pinned_ref_get(align.constant))) return emit_runtime_pointerset(ctx, argv); if (!is_valid_intrinsic_elptr(ety)) { emit_error(ctx, "pointerset: invalid pointer type"); @@ -848,7 +848,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(ctx.types().T_size, 1)); setName(ctx.emission_context, im1, "pointerset_idx"); - Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, jl_pinned_ref_get(e.typ)); if (ety == (jl_value_t*)jl_any_type) { // unsafe_store to Ptr{Any} is allowed to implicitly drop GC roots. auto gep = ctx.builder.CreateInBoundsGEP(ctx.types().T_size, thePtr, im1); @@ -889,8 +889,8 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) static jl_cgval_t emit_pointerarith(jl_codectx_t &ctx, intrinsic f, ArrayRef argv) { - jl_value_t *ptrtyp = argv[0].typ; - jl_value_t *offtyp = argv[1].typ; + jl_value_t *ptrtyp = jl_pinned_ref_get(argv[0].typ); + jl_value_t *offtyp = jl_pinned_ref_get(argv[1].typ); if (!jl_is_cpointer_type(ptrtyp) || offtyp != (jl_value_t *)jl_ulong_type) return emit_runtime_call(ctx, f, argv, argv.size()); assert(f == add_ptr || f == sub_ptr); @@ -915,8 +915,8 @@ static jl_cgval_t emit_pointerarith(jl_codectx_t &ctx, intrinsic f, static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &ord = argv[0]; - if (ord.constant && jl_is_symbol(ord.constant)) { - enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, true); + if (ord.constant && jl_is_symbol(jl_pinned_ref_get(ord.constant))) { + enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, true); if (order == jl_memory_order_invalid) { emit_atomic_error(ctx, "invalid atomic ordering"); return jl_cgval_t(); // unreachable @@ -932,13 +932,13 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef { const jl_cgval_t &e = argv[0]; const jl_cgval_t &ord = argv[1]; - jl_value_t *aty = e.typ; - if (!jl_is_cpointer_type(aty) || !ord.constant || !jl_is_symbol(ord.constant)) + jl_value_t *aty = jl_pinned_ref_get(e.typ); + if (!jl_is_cpointer_type(aty) || !ord.constant || !jl_is_symbol(jl_pinned_ref_get(ord.constant))) return emit_runtime_call(ctx, atomic_pointerref, argv, 2); jl_value_t *ety = jl_tparam0(aty); if (jl_is_typevar(ety)) return emit_runtime_call(ctx, atomic_pointerref, argv, 2); - enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), true, false); if (order == jl_memory_order_invalid) { emit_atomic_error(ctx, "invalid atomic ordering"); return jl_cgval_t(); // unreachable @@ -946,7 +946,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef AtomicOrdering llvm_order = get_llvm_atomic_order(order); if (ety == (jl_value_t*)jl_any_type) { - Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); + Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, jl_pinned_ref_get(e.typ)); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, thePtr, Align(sizeof(jl_value_t*))); setName(ctx.emission_context, load, "atomic_pointerref"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); @@ -970,7 +970,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef assert(jl_is_datatype(ety)); Value *strct = emit_allocobj(ctx, (jl_datatype_t*)ety, true); setName(ctx.emission_context, strct, "atomic_pointerref_box"); - Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, jl_pinned_ref_get(e.typ)); Type *loadT = Type::getIntNTy(ctx.builder.getContext(), nb * 8); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); LoadInst *load = ctx.builder.CreateAlignedLoad(loadT, thePtr, Align(nb)); @@ -988,7 +988,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed); assert(!isboxed); if (!type_is_ghost(ptrty)) { - Value *thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, jl_pinned_ref_get(e.typ)); auto load = typed_load(ctx, thePtr, nullptr, ety, ctx.tbaa().tbaa_data, nullptr, isboxed, llvm_order, false, nb); setName(ctx.emission_context, load.V, "atomic_pointerref"); return load; @@ -1018,18 +1018,18 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef const jl_cgval_t &ord = isreplacefield || ismodifyfield ? argv[3] : argv[2]; const jl_cgval_t &failord = isreplacefield ? argv[4] : undefval; - jl_value_t *aty = e.typ; - if (!jl_is_cpointer_type(aty) || !ord.constant || !jl_is_symbol(ord.constant)) + jl_value_t *aty = jl_pinned_ref_get(e.typ); + if (!jl_is_cpointer_type(aty) || !jl_pinned_ref_get(ord.constant)|| !jl_is_symbol(jl_pinned_ref_get(ord.constant))) return emit_runtime_call(ctx, f, argv, nargs); if (isreplacefield) { - if (!failord.constant || !jl_is_symbol(failord.constant)) + if (!jl_pinned_ref_get(failord.constant) || !jl_is_symbol(jl_pinned_ref_get(failord.constant))) return emit_runtime_call(ctx, f, argv, nargs); } jl_value_t *ety = jl_tparam0(aty); if (jl_is_typevar(ety)) return emit_runtime_call(ctx, f, argv, nargs); - enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); - enum jl_memory_order failorder = isreplacefield ? jl_get_atomic_order((jl_sym_t*)failord.constant, true, false) : order; + enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(ord.constant), !issetfield, true); + enum jl_memory_order failorder = isreplacefield ? jl_get_atomic_order((jl_sym_t*)jl_pinned_ref_get(failord.constant), true, false) : order; if (order == jl_memory_order_invalid || failorder == jl_memory_order_invalid || failorder > order) { emit_atomic_error(ctx, "invalid atomic ordering"); return jl_cgval_t(); // unreachable @@ -1040,7 +1040,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef if (ety == (jl_value_t*)jl_any_type) { // unsafe_store to Ptr{Any} is allowed to implicitly drop GC roots. // n.b.: the expected value (y) must be rooted, but not the others - Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); + Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, jl_pinned_ref_get(e.typ)); bool isboxed = true; jl_cgval_t ret = typed_store(ctx, thePtr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, llvm_order, llvm_failorder, sizeof(jl_value_t*), nullptr, issetfield, isreplacefield, isswapfield, ismodifyfield, false, false, modifyop, "atomic_pointermodify", nullptr, nullptr); @@ -1082,7 +1082,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef assert(!isboxed); Value *thePtr; if (!type_is_ghost(ptrty)) - thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, e.typ); + thePtr = emit_unbox(ctx, PointerType::getUnqual(ptrty->getContext()), e, jl_pinned_ref_get(e.typ)); else thePtr = nullptr; // could use any value here, since typed_store will not use it jl_cgval_t ret = typed_store(ctx, thePtr, x, y, ety, ctx.tbaa().tbaa_data, nullptr, nullptr, isboxed, @@ -1154,8 +1154,8 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ { Value *isfalse = emit_condition(ctx, c, "ifelse"); setName(ctx.emission_context, isfalse, "ifelse_cond"); - jl_value_t *t1 = x.typ; - jl_value_t *t2 = y.typ; + jl_value_t *t1 = jl_pinned_ref_get(x.typ); + jl_value_t *t2 =jl_pinned_ref_get(y.typ); // handle cases where the condition is irrelevant based on type info if (t1 == jl_bottom_type && t2 == jl_bottom_type) return jl_cgval_t(); // undefined @@ -1177,8 +1177,8 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ // to instantiate a union-split optimization x = convert_julia_type(ctx, x, rt_hint); y = convert_julia_type(ctx, y, rt_hint); - t1 = x.typ; - t2 = y.typ; + t1 = jl_pinned_ref_get(x.typ); + t2 = jl_pinned_ref_get(y.typ); } Value *ifelse_result; @@ -1235,10 +1235,10 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ } Value *tindex; if (!x_tindex && x.constant) { - x_tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80 | get_box_tindex((jl_datatype_t*)jl_typeof(x.constant), rt_hint)); + x_tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80 | get_box_tindex((jl_datatype_t*)jl_typeof(jl_pinned_ref_get(x.constant)), rt_hint)); } if (!y_tindex && y.constant) { - y_tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80 | get_box_tindex((jl_datatype_t*)jl_typeof(y.constant), rt_hint)); + y_tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80 | get_box_tindex((jl_datatype_t*)jl_typeof(jl_pinned_ref_get(y.constant)), rt_hint)); } if (x_tindex && y_tindex) { tindex = ctx.builder.CreateSelect(isfalse, y_tindex, x_tindex); @@ -1398,19 +1398,19 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar const jl_cgval_t &x = argv[0]; if (!jl_is_primitivetype(x.typ)) return emit_runtime_call(ctx, f, argv, nargs); - Type *xt = INTT(bitstype_to_llvm(x.typ, ctx.builder.getContext(), true), DL); - Value *from = emit_unbox(ctx, xt, x, x.typ); + Type *xt = INTT(bitstype_to_llvm(jl_pinned_ref_get(x.typ), ctx.builder.getContext(), true), DL); + Value *from = emit_unbox(ctx, xt, x, jl_pinned_ref_get(x.typ)); Value *ans = ctx.builder.CreateNot(from); - return mark_julia_type(ctx, ans, false, x.typ); + return mark_julia_type(ctx, ans, false, jl_pinned_ref_get(x.typ)); } case have_fma: { ++Emitted_have_fma; assert(nargs == 1); const jl_cgval_t &x = argv[0]; - if (!x.constant || !jl_is_datatype(x.constant)) + if (!x.constant || !jl_is_datatype(jl_pinned_ref_get(x.constant))) return emit_runtime_call(ctx, f, argv, nargs); - jl_datatype_t *dt = (jl_datatype_t*) x.constant; + jl_datatype_t *dt = (jl_datatype_t*) jl_pinned_ref_get(x.constant); // select the appropriated overloaded intrinsic std::string intr_name = "julia.cpu.have_fma."; @@ -1431,9 +1431,9 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar const jl_cgval_t &xinfo = argv[0]; // verify argument types - if (!jl_is_primitivetype(xinfo.typ)) + if (!jl_is_primitivetype(jl_pinned_ref_get(xinfo.typ))) return emit_runtime_call(ctx, f, argv, nargs); - Type *xtyp = bitstype_to_llvm(xinfo.typ, ctx.builder.getContext(), true); + Type *xtyp = bitstype_to_llvm(jl_pinned_ref_get(xinfo.typ), ctx.builder.getContext(), true); if (float_func()[f]) { if (!xtyp->isFloatingPointTy()) return emit_runtime_call(ctx, f, argv, nargs); @@ -1459,7 +1459,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar if (f == shl_int || f == lshr_int || f == ashr_int) { if (!jl_is_primitivetype(argv[1].typ)) return emit_runtime_call(ctx, f, argv, nargs); - argt[1] = INTT(bitstype_to_llvm(argv[1].typ, ctx.builder.getContext(), true), DL); + argt[1] = INTT(bitstype_to_llvm(jl_pinned_ref_get(argv[1].typ), ctx.builder.getContext(), true), DL); } else { for (size_t i = 1; i < nargs; ++i) { @@ -1472,12 +1472,12 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar // unbox the arguments SmallVector argvalues(nargs); for (size_t i = 0; i < nargs; ++i) { - argvalues[i] = emit_unbox(ctx, argt[i], argv[i], argv[i].typ); + argvalues[i] = emit_unbox(ctx, argt[i], argv[i], jl_pinned_ref_get(argv[i].typ)); } // call the intrinsic - jl_value_t *newtyp = xinfo.typ; - Value *r = emit_untyped_intrinsic(ctx, f, argvalues, nargs, (jl_datatype_t**)&newtyp, xinfo.typ); + jl_value_t *newtyp = jl_pinned_ref_get(xinfo.typ); + Value *r = emit_untyped_intrinsic(ctx, f, argvalues, nargs, (jl_datatype_t**)&newtyp, jl_pinned_ref_get(xinfo.typ)); // Turn Bool operations into mod 1 now, if needed if (newtyp == (jl_value_t*)jl_bool_type && !r->getType()->isIntegerTy(1)) r = ctx.builder.CreateTrunc(r, getInt1Ty(ctx.builder.getContext())); diff --git a/src/julia.h b/src/julia.h index 0820cd52420d5..7f059a7cd46f2 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1237,8 +1237,6 @@ struct _jl_gcframe_t { #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<2) #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<2)|1) #define JL_GC_DECODE_NROOTS(n) (n >> 2) -<<<<<<< HEAD -======= #define JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) JL_GC_ENCODE_PUSHARGS(n) #define JL_GC_ENCODE_PUSH_NO_TPIN(n) JL_GC_ENCODE_PUSH(n) @@ -1272,7 +1270,6 @@ struct _jl_gcframe_t { #define JL_GC_ENCODE_PUSH_NO_TPIN(n) ((((size_t)(n))<<3)|5) #endif #endif ->>>>>>> b4d216cfeb (Support moving) #ifdef __clang_gcanalyzer__ From 2070bb443370692f763657234b7ec6de2551a27c Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 14 Apr 2025 04:38:13 +0000 Subject: [PATCH 633/662] Wrapping types from InferKey with jl_pinned_ref --- src/engine.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/engine.cpp b/src/engine.cpp index 858f37b55e85e..4cd423d36a301 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -18,7 +18,7 @@ struct ReservationInfo { struct InferKey { jl_method_instance_t *mi = nullptr; - jl_value_t *owner = nullptr; + jl_pinned_ref(jl_value_t) owner = jl_pinned_ref_assume(jl_value_t, nullptr); }; template<> struct llvm::DenseMapInfo { @@ -27,17 +27,17 @@ template<> struct llvm::DenseMapInfo { static inline InferKey getEmptyKey() { return InferKey{FirstInfo::getEmptyKey(), - SecondInfo::getEmptyKey()}; + jl_pinned_ref_create(jl_value_t, SecondInfo::getEmptyKey())}; } static inline InferKey getTombstoneKey() { return InferKey{FirstInfo::getTombstoneKey(), - SecondInfo::getTombstoneKey()}; + jl_pinned_ref_create(jl_value_t, SecondInfo::getTombstoneKey())}; } static unsigned getHashValue(const InferKey& PairVal) { return detail::combineHashValue(FirstInfo::getHashValue(PairVal.mi), - SecondInfo::getHashValue(PairVal.owner)); + SecondInfo::getHashValue(jl_pinned_ref_get(PairVal.owner))); } static bool isEqual(const InferKey &LHS, const InferKey &RHS) { @@ -66,7 +66,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto tid = jl_atomic_load_relaxed(&ct->tid); if (([tid, m, owner, ci] () -> bool { // necessary scope block / lambda for unique_lock jl_unique_gcsafe_lock lock(engine_lock); - InferKey key{m, owner}; + InferKey key{m, jl_pinned_ref_create(jl_value_t, owner)}; if ((signed)Awaiting.size() < tid + 1) Awaiting.resize(tid + 1); while (1) { @@ -106,7 +106,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner int jl_engine_hasreserved(jl_method_instance_t *m, jl_value_t *owner) { jl_task_t *ct = jl_current_task; - InferKey key = {m, owner}; + InferKey key = {m, jl_pinned_ref_create(jl_value_t, owner)}; std::unique_lock lock(engine_lock); auto record = Reservations.find(key); return record != Reservations.end() && record->second.tid == jl_atomic_load_relaxed(&ct->tid); @@ -139,7 +139,7 @@ void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) { jl_task_t *ct = jl_current_task; std::unique_lock lock(engine_lock); - auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), ci->owner}); + auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), jl_pinned_ref_create(jl_value_t, ci->owner)}); if (record == Reservations.end() || record->second.ci != ci) return; assert(jl_atomic_load_relaxed(&ct->tid) == record->second.tid); From 9e5b64355da360264cabb518007d98c32c6afd42 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 14 Apr 2025 04:47:15 +0000 Subject: [PATCH 634/662] Wrapping types from cfunc_decl_t with jl_pinned_ref --- src/aotcompile.cpp | 6 +++--- src/codegen.cpp | 2 +- src/julia_internal.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index d68c5e36d7e6d..b6c0c604031ef 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -578,9 +578,9 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : params.cfuncs) { Module *M = cfunc.cfuncdata->getParent(); - jl_value_t *sigt = cfunc.abi.sigt; + jl_value_t *sigt = jl_pinned_ref_get(cfunc.abi.sigt); JL_GC_PROMISE_ROOTED(sigt); - jl_value_t *declrt = cfunc.abi.rt; + jl_value_t *declrt = jl_pinned_ref_get(cfunc.abi.rt); JL_GC_PROMISE_ROOTED(declrt); Function *unspec = aot_abi_converter(params, M, cfunc.abi, nullptr, nullptr, "", "", false); jl_code_instance_t *codeinst = nullptr; @@ -2493,7 +2493,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ jl_compiled_functions_t compiled_functions; size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : output.cfuncs) { - jl_value_t *sigt = cfunc.abi.sigt; + jl_value_t *sigt = jl_pinned_ref_get(cfunc.abi.sigt); JL_GC_PROMISE_ROOTED(sigt); jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); if (mi == nullptr) diff --git a/src/codegen.cpp b/src/codegen.cpp index c8c9ff5f97606..a2b1dde6807a4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7223,7 +7223,7 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ cw->setAttributes(getcaller->getAttributes()); return cw; }); - jl_abi_t cfuncabi = {sigt, declrt, nargs, specsig, is_opaque_closure}; + jl_abi_t cfuncabi = {jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, declrt), nargs, specsig, is_opaque_closure}; ctx.emission_context.cfuncs.push_back({cfuncabi, cfuncdata}); if (specsig) { // TODO: could we force this to guarantee passing a box for `f` here (since we diff --git a/src/julia_internal.h b/src/julia_internal.h index d3eca3f2a5029..de88418b9b0fe 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -392,8 +392,8 @@ static inline void memassign_safe(int hasptr, char *dst, const jl_value_t *src, // data structures for runtime codegen typedef struct _jl_abi_t { - jl_value_t *sigt; - jl_value_t *rt; + jl_pinned_ref(jl_value_t) sigt; + jl_pinned_ref(jl_value_t) rt; size_t nargs; int specsig; // bool // OpaqueClosure Methods override the first argument of their signature From 43b983eb40c41d7a478b052b142e0b3750cc3395 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 17 Apr 2025 04:03:40 +0000 Subject: [PATCH 635/662] Adding support for setting MMTK_MOVING_STRESS --- deps/mmtk_julia.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index b443f9b6a25da..4bfd62b9dd8a9 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -4,7 +4,8 @@ # FIXME: By default we do a non-moving build. We should change the default to 1 # once we support moving plans. MMTK_MOVING ?= 0 -MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) +MMTK_MOVING_STRESS ?= 0 +MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) MMTK_MOVING_STRESS=$(MMTK_MOVING_STRESS) ifneq ($(USE_BINARYBUILDER_MMTK_JULIA),1) $(eval $(call git-external,mmtk_julia,MMTK_JULIA,,,$(BUILDDIR))) From 8199d5d1a2058f33bba50af7b4fc541dc1cd5963 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 17 Apr 2025 04:04:44 +0000 Subject: [PATCH 636/662] Adding more globally rooted functions and using less transitive pinning (only global roots now) --- src/gc-mmtk.c | 50 ++++++++++++++++++++++------------ src/interpreter.c | 4 ++- src/julia.h | 8 ++++-- src/llvm-final-gc-lowering.cpp | 2 +- src/threading.c | 2 +- 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 1af93fcc61a9e..1e3181e7ccdef 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -21,6 +21,8 @@ extern "C" { extern jl_value_t *cmpswap_names JL_GLOBALLY_ROOTED; extern const unsigned pool_sizes[]; extern jl_mutex_t finalizers_lock; +extern jl_task_t *wait_empty JL_GLOBALLY_ROOTED; +extern _Atomic(jl_function_t*) init_task_lock_func JL_GLOBALLY_ROOTED; // FIXME: Should the values below be shared between both GC's? // Note that MMTk uses a hard max heap limit, which is set by default @@ -562,6 +564,9 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, jl_typemap_entry_t *v = jl_atomic_load_relaxed(&call_cache[i]); TRACE_GLOBALLY_ROOTED(v); } + + TRACE_GLOBALLY_ROOTED(wait_empty); + TRACE_GLOBALLY_ROOTED(jl_precompile_toplevel_module); TRACE_GLOBALLY_ROOTED(jl_global_roots_list); TRACE_GLOBALLY_ROOTED(jl_global_roots_keyset); @@ -628,7 +633,9 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(jl_fielderror_type); TRACE_GLOBALLY_ROOTED(jl_atomicerror_type); TRACE_GLOBALLY_ROOTED(jl_missingcodeerror_type); + TRACE_GLOBALLY_ROOTED(jl_trimfailure_type); TRACE_GLOBALLY_ROOTED(jl_lineinfonode_type); + TRACE_GLOBALLY_ROOTED(jl_abioverride_type); TRACE_GLOBALLY_ROOTED(jl_stackovf_exception); TRACE_GLOBALLY_ROOTED(jl_memory_exception); TRACE_GLOBALLY_ROOTED(jl_readonlymemory_exception); @@ -654,6 +661,7 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(jl_float16_type); TRACE_GLOBALLY_ROOTED(jl_float32_type); TRACE_GLOBALLY_ROOTED(jl_float64_type); + TRACE_GLOBALLY_ROOTED(jl_bfloat16_type); TRACE_GLOBALLY_ROOTED(jl_floatingpoint_type); TRACE_GLOBALLY_ROOTED(jl_number_type); TRACE_GLOBALLY_ROOTED(jl_void_type); // deprecated @@ -726,6 +734,7 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(task_done_hook_func); // threading.c // TRACE_GLOBALLY_ROOTED(jl_all_tls_states); -- we don't need to pin these. Julia TLS are allocated with calloc. + TRACE_GLOBALLY_ROOTED(init_task_lock_func); } // These are from gc_mark_roots -- this is not enough for a moving GC. We need to make sure @@ -771,21 +780,30 @@ JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) trace_full_globally_rooted(closure, &buf, &len); // Simply pin things in global roots table - // size_t i; - // for (i = 0; i < jl_array_len(jl_global_roots_table); i++) { - // jl_value_t* root = jl_array_ptr_ref(jl_global_roots_table, i); - // add_node_to_roots_buffer(closure, &buf, &len, root); - // } - // for (i = 0; i < jl_global_roots_list->length; i++) { - // jl_value_t* root = jl_genericmemory_ptr_ref(jl_global_roots_list, i); - // add_node_to_roots_buffer(closure, &buf, &len, root); - // } - // for (i = 0; i < jl_global_roots_keyset->length; i++) { - // jl_value_t* root = jl_genericmemory_ptr_ref(jl_global_roots_keyset, i); - // add_node_to_roots_buffer(closure, &buf, &len, root); - // } - // add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_list); - // add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_keyset); + size_t i; + for (i = 0; i < jl_global_roots_list->length; i++) { + jl_value_t* root = jl_genericmemory_ptr_ref(jl_global_roots_list, i); + add_node_to_roots_buffer(closure, &buf, &len, root); + } + + if (precompile_field_replace) { + jl_array_t *vals = (jl_array_t*)jl_svecref(precompile_field_replace, 0); + jl_array_t *fields = (jl_array_t*)jl_svecref(precompile_field_replace, 1); + jl_array_t *newvals = (jl_array_t*)jl_svecref(precompile_field_replace, 2); + add_node_to_roots_buffer(closure, &buf, &len, vals); + add_node_to_roots_buffer(closure, &buf, &len, fields); + add_node_to_roots_buffer(closure, &buf, &len, newvals); + + size_t l = jl_array_nrows(vals); + for (i = 0; i < l; i++) { + jl_value_t *val = jl_array_ptr_ref(vals, i); + jl_value_t *field = jl_array_ptr_ref(fields, i); + jl_value_t *newval = jl_array_ptr_ref(newvals, i); + add_node_to_roots_buffer(closure, &buf, &len, val); + add_node_to_roots_buffer(closure, &buf, &len, field); + add_node_to_roots_buffer(closure, &buf, &len, newval); + } + } // // add module // add_node_to_roots_buffer(closure, &buf, &len, jl_main_module); @@ -816,8 +834,6 @@ JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) size_t tpinned_len = 0; add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_list); add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_keyset); - // FIXME: transivitely pinning for now, should be removed after we add moving Immix - add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, precompile_field_replace); // Push the result of the work. (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false); (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false); diff --git a/src/interpreter.c b/src/interpreter.c index 38ae990f93540..c7b2da6d388cb 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -54,9 +54,11 @@ extern void JL_GC_ENABLEFRAME(interpreter_state*) JL_NOTSAFEPOINT; #ifdef WITH_THIRD_PARTY_HEAP #if WITH_THIRD_PARTY_HEAP == 1 // MMTk -#define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<3)|2) +// #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<3)|2) // For roots that are not transitively pinned #define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) ((((size_t)(n))<<3)|6) +// do not transitively pin shadow stack objects +#define JL_GC_ENCODE_PUSHFRAME(n) JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) #endif // endif MMTk #else #define JL_GC_ENCODE_PUSHFRAME(n) ((((size_t)(n))<<2)|2) diff --git a/src/julia.h b/src/julia.h index 7f059a7cd46f2..c20068f800184 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1261,13 +1261,17 @@ struct _jl_gcframe_t { // #define JL_GC_ENCODE_PUSHFRAME_NO_TPIN(n) ((((size_t)(n))<<3)|6) // these are transitively pinning -#define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<3) -#define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<3)|1) +// #define JL_GC_ENCODE_PUSHARGS(n) (((size_t)(n))<<3) +// #define JL_GC_ENCODE_PUSH(n) ((((size_t)(n))<<3)|1) #define JL_GC_DECODE_NROOTS(n) (n >> 3) // these only pin the root object itself #define JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) (((size_t)(n))<<3|4) #define JL_GC_ENCODE_PUSH_NO_TPIN(n) ((((size_t)(n))<<3)|5) + +#define JL_GC_ENCODE_PUSHARGS(n) JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) +#define JL_GC_ENCODE_PUSH(n) JL_GC_ENCODE_PUSH_NO_TPIN(n) + #endif #endif diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index b98b3dc741ebe..11c7e1bf9f76f 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -44,7 +44,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F) StoreInst *inst = builder.CreateAlignedStore( // FIXME: We should use JL_GC_ENCODE_PUSHARGS_NO_TPIN here. // We need to make sure things are properly pinned before turning this into a non TPIN push. - ConstantInt::get(T_size, JL_GC_ENCODE_PUSHARGS(nRoots)), + ConstantInt::get(T_size, JL_GC_ENCODE_PUSHARGS_NO_TPIN(nRoots)), builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0, "frame.nroots"),// GEP of 0 becomes a noop and eats the name Align(sizeof(void*))); inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); diff --git a/src/threading.c b/src/threading.c index 15763978b4611..06c2475ff0869 100644 --- a/src/threading.c +++ b/src/threading.c @@ -409,7 +409,7 @@ jl_ptls_t jl_init_threadtls(int16_t tid) return ptls; } -static _Atomic(jl_function_t*) init_task_lock_func JL_GLOBALLY_ROOTED = NULL; +_Atomic(jl_function_t*) init_task_lock_func JL_GLOBALLY_ROOTED = NULL; static void jl_init_task_lock(jl_task_t *ct) { From 8fbc0a6637635248d4e4753edbf837f99c52dd03 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 17 Apr 2025 04:07:28 +0000 Subject: [PATCH 637/662] Adding more OBJHASH_PIN and PTRHASH_PIN and fix jl_pinned_ref_create --- src/engine.cpp | 4 ++-- src/genericmemory.c | 1 + src/precompile_utils.c | 8 ++++++-- src/signals-mach.c | 2 +- src/staticdata.c | 7 +++++++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/engine.cpp b/src/engine.cpp index 4cd423d36a301..08a9c300c4668 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -27,12 +27,12 @@ template<> struct llvm::DenseMapInfo { static inline InferKey getEmptyKey() { return InferKey{FirstInfo::getEmptyKey(), - jl_pinned_ref_create(jl_value_t, SecondInfo::getEmptyKey())}; + jl_pinned_ref_assume(jl_value_t, SecondInfo::getEmptyKey())}; } static inline InferKey getTombstoneKey() { return InferKey{FirstInfo::getTombstoneKey(), - jl_pinned_ref_create(jl_value_t, SecondInfo::getTombstoneKey())}; + jl_pinned_ref_assume(jl_value_t, SecondInfo::getTombstoneKey())}; } static unsigned getHashValue(const InferKey& PairVal) { diff --git a/src/genericmemory.c b/src/genericmemory.c index 2c3116b9d3467..97b53b181cba7 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -47,6 +47,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory_unchecked(jl_ptls_t ptls else { int isaligned = 1; // jl_gc_managed_malloc is always aligned jl_gc_track_malloced_genericmemory(ptls, m, isaligned); + OBJ_PIN(m); jl_genericmemory_data_owner_field(m) = (jl_value_t*)m; } // length set by codegen diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 491f111ac4746..a34a3838c6ddc 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -425,8 +425,12 @@ static void jl_rebuild_methtables(arraylist_t *MIs, htable_t *mtables) JL_GC_DIS jl_methtable_t *old_mt = jl_method_get_table(m); if ((jl_value_t *)old_mt == jl_nothing) continue; - if (!ptrhash_has(mtables, old_mt)) - ptrhash_put(mtables, old_mt, jl_new_method_table(old_mt->name, old_mt->module)); + if (!ptrhash_has(mtables, old_mt)){ + jl_methtable_t *new_mt = jl_new_method_table(old_mt->name, old_mt->module); + OBJHASH_PIN(old_mt) + OBJHASH_PIN(new_mt) + ptrhash_put(mtables, old_mt, new_mt); + } jl_methtable_t *mt = (jl_methtable_t*)ptrhash_get(mtables, old_mt); //TODO: should this be a function like unsafe_insert_method, since all that is wanted is the jl_typemap_insert on a copy of the existing entry size_t min_world = jl_atomic_load_relaxed(&m->primary_world); diff --git a/src/signals-mach.c b/src/signals-mach.c index b7057416ad407..39a46729deb22 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -780,7 +780,7 @@ void jl_profile_thread_mach(int tid) profile_bt_data_prof[profile_bt_size_cur++].uintptr = ptls->tid + 1; // store task id (never null) - profile_bt_data_prof[profile_bt_size_cur++].jlvalue = (jl_value_t*)jl_atomic_load_relaxed(&ptls->current_task); + jl_pinned_ref_set(profile_bt_data_prof[profile_bt_size_cur++].jlvalue, (jl_value_t*)jl_atomic_load_relaxed(&ptls->current_task)); // store cpu cycle clock profile_bt_data_prof[profile_bt_size_cur++].uintptr = cycleclock(); diff --git a/src/staticdata.c b/src/staticdata.c index d70c8c04620da..61da1d3460962 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2676,12 +2676,16 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED void *idx = ptrhash_get(&serialization_order, bindings); assert(idx != HT_NOTFOUND && idx != (void*)(uintptr_t)-1); assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == bindings); + OBJHASH_PIN(bindings2) + OBJHASH_PIN(idx) ptrhash_put(&serialization_order, bindings2, idx); serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = bindings2; idx = ptrhash_get(&serialization_order, bindingkeyset); assert(idx != HT_NOTFOUND && idx != (void*)(uintptr_t)-1); assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == bindingkeyset); + OBJHASH_PIN(jl_atomic_load_relaxed(&bindingkeyset2)) + OBJHASH_PIN(idx) ptrhash_put(&serialization_order, jl_atomic_load_relaxed(&bindingkeyset2), idx); serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = jl_atomic_load_relaxed(&bindingkeyset2); jl_atomic_store_relaxed(&m->bindings, bindings2); @@ -3087,6 +3091,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, } else if (jl_field_size(st, field) > 0) { // replace the bits + OBJHASH_PIN(fldaddr) + OBJHASH_PIN(newval) ptrhash_put(&bits_replace, (void*)fldaddr, newval); // and any pointers inside jl_datatype_t *rty = (jl_datatype_t*)jl_typeof(newval); @@ -3126,6 +3132,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, htable_new(&fptr_to_id, jl_n_builtins); uintptr_t i; for (i = 0; i < jl_n_builtins; i++) { + PTRHASH_PIN(jl_builtin_f_addrs[i]) ptrhash_put(&fptr_to_id, (void*)(uintptr_t)jl_builtin_f_addrs[i], (void*)(i + 2)); } htable_new(&serialization_order, 25000); From aded1c891a481b7e6ce840650db1c7012415f236 Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Wed, 30 Apr 2025 05:16:15 +0000 Subject: [PATCH 638/662] Pinning buffers; removing transitive pinning of global_roots_table; Pinning wr --- src/gc-mmtk.c | 21 +-------------------- src/julia_internal.h | 6 +++++- src/llvm-late-gc-lowering-stock.cpp | 2 +- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 1e3181e7ccdef..52d8651082518 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -514,19 +514,6 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* } } -static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) { - if (root == NULL) - return; - - buf->ptr[*buf_len] = root; - *buf_len += 1; - if (*buf_len >= buf->cap) { - RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true); - *buf = new_buf; - *buf_len = 0; - } -} - // staticdata_utils.c extern jl_array_t *internal_methods; extern jl_array_t *newly_inferred; @@ -828,15 +815,8 @@ JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) // add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names); // add_node_to_roots_buffer(closure, &buf, &len, precompile_field_replace); - // jl_global_roots_table must be transitively pinned - // FIXME: We need to remove transitive pinning of global roots. Otherwise they may pin most of the objects in the heap. - RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true); - size_t tpinned_len = 0; - add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_list); - add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_keyset); // Push the result of the work. (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false); - (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false); } JL_DLLEXPORT void jl_gc_scan_julia_exc_obj(void* obj_raw, void* closure, ProcessSlotFn process_slot) { @@ -1090,6 +1070,7 @@ JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls, jl_value_t *valu { jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*), jl_weakref_type); wr->value = value; // NOTE: wb not needed here + OBJ_PIN(wr) // Note: we are using MMTk's weak ref processing. If we switch to Julia's weak ref processing, // we need to make sure the value and the weak ref won't be moved (e.g. pin them) mmtk_add_weak_candidate(wr); diff --git a/src/julia_internal.h b/src/julia_internal.h index de88418b9b0fe..10c9d190b33a8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -602,7 +602,11 @@ JL_DLLEXPORT uintptr_t jl_get_buff_tag(void) JL_NOTSAFEPOINT; typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { - return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag); + jl_gc_tracked_buffer_t *buf = jl_gc_alloc(ptls, sz, (void*)jl_buff_tag); + // FIXME: we might not need to pin buffers since the introduction of jl_genericmemory_t objects + // but removing the pin below will currently fail an assertion in the binding + OBJ_PIN(buf); + return buf; } jl_value_t *jl_permbox8(jl_datatype_t *t, uintptr_t tag, uint8_t x); diff --git a/src/llvm-late-gc-lowering-stock.cpp b/src/llvm-late-gc-lowering-stock.cpp index b6e517c52f0a3..838300043768d 100644 --- a/src/llvm-late-gc-lowering-stock.cpp +++ b/src/llvm-late-gc-lowering-stock.cpp @@ -10,4 +10,4 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) void LateLowerGCFrame::CleanupGCPreserve(Function &F, CallInst *CI, Value *callee, Type *T_size) { // Do nothing for the stock GC -} \ No newline at end of file +} From 7fa91638457be073304ba61af917d2d7d50a4659 Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Thu, 1 May 2025 03:59:01 +0000 Subject: [PATCH 639/662] Adding pin in jl_string_to_genericmemory --- src/genericmemory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/genericmemory.c b/src/genericmemory.c index 97b53b181cba7..78f96e496c7a5 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -115,6 +115,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str) m->ptr = jl_string_data(str); jl_genericmemory_data_owner_field(m) = str; OBJ_PIN(str); + OBJ_PIN(m); // FIXME: without this pin, make test-Tar fail with stress copying return m; } From 3a414655cf427284efda89300038dd9ce2ef10c2 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 5 May 2025 03:38:05 +0000 Subject: [PATCH 640/662] Transitively pinning objects from gc_preserve --- src/gc-mmtk.c | 2 +- src/julia.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 52d8651082518..dc282f0344805 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -1533,7 +1533,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p) #define JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(rts_var,n) \ rts_var = ((jl_value_t**)malloc(((n)+2)*sizeof(jl_value_t*)))+2; \ - ((void**)rts_var)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(n); \ + ((void**)rts_var)[-2] = (void*)JL_GC_ENCODE_PUSHARGS_TPIN(n); \ ((void**)rts_var)[-1] = jl_p_gcpreserve_stack; \ memset((void*)rts_var, 0, (n)*sizeof(jl_value_t*)); \ jl_p_gcpreserve_stack = (jl_gcframe_t*)&(((void**)rts_var)[-2]); \ diff --git a/src/julia.h b/src/julia.h index c20068f800184..e387aec7c8645 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1272,6 +1272,8 @@ struct _jl_gcframe_t { #define JL_GC_ENCODE_PUSHARGS(n) JL_GC_ENCODE_PUSHARGS_NO_TPIN(n) #define JL_GC_ENCODE_PUSH(n) JL_GC_ENCODE_PUSH_NO_TPIN(n) +#define JL_GC_ENCODE_PUSHARGS_TPIN(n) (((size_t)(n))<<3) + #endif #endif From a6d1c267ee44e44fe36f4aee1744ca368256cbee Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 26 Mar 2025 22:38:33 +0000 Subject: [PATCH 641/662] Export pool stats --- src/gc-stock.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gc-stock.c b/src/gc-stock.c index b9c0827f60e65..b9af37a931483 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -826,6 +826,7 @@ int jl_gc_classify_pools(size_t sz, int *osize) JL_DLLEXPORT gc_fragmentation_stat_t jl_gc_page_fragmentation_stats[JL_GC_N_POOLS]; JL_DLLEXPORT double jl_gc_page_utilization_stats[JL_GC_N_MAX_POOLS]; +JL_DLLEXPORT gc_fragmentation_stat_t jl_gc_page_fragmentation_stats_export[JL_GC_N_POOLS]; STATIC_INLINE void gc_update_fragmentation_data_for_size_class(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { @@ -853,6 +854,10 @@ STATIC_INLINE void gc_compute_utilization_data_for_size_classes(void) JL_NOTSAFE utilization -= ((double)n_freed_objs * (double)jl_gc_sizeclasses[i]) / (double)n_pages_allocd / (double)GC_PAGE_SZ; } jl_gc_page_utilization_stats[i] = utilization; + jl_gc_page_fragmentation_stats_export[i].n_freed_objs = n_freed_objs; + jl_gc_page_fragmentation_stats_export[i].n_pages_allocd = n_pages_allocd; + jl_atomic_store_relaxed(&stats->n_freed_objs, 0); + jl_atomic_store_relaxed(&stats->n_pages_allocd, 0); } } From 534c6a127f81c1a6acda38df7a731fd7ab4dc32c Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 6 Feb 2025 00:07:35 +0000 Subject: [PATCH 642/662] Invoke make for the binding everytime we make --- deps/mmtk_julia.mk | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index 4bfd62b9dd8a9..9e3f8e46d606d 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -49,13 +49,10 @@ compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so version-check-mmtk_julia: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so -# NB: This will NOT run `cargo build` if there are changes in the Rust source files -# inside the binding repo. However the target below should remake the symlink if there -# are changes in the libmmtk_julia.so from the custom MMTK_JULIA_DIR folder -$(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so +$(BUILDROOT)/usr/lib/libmmtk_julia.so: make-binding @ln -sf $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $@ -$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so: +make-binding: @$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD) MMTK_JULIA_VER := mmtk_julia_custom From b51e8f2e86efa7afceaf9615f754f7574907de63 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 8 May 2025 00:20:19 +0000 Subject: [PATCH 643/662] Use Immix as non moving space --- src/gc-mmtk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index dc282f0344805..5ced787d3c282 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -1099,6 +1099,7 @@ JL_DLLEXPORT int* jl_gc_get_have_pending_finalizers(void) { // ========================================================================= // #define MMTK_DEFAULT_IMMIX_ALLOCATOR (0) +#define MMTK_NONMOVING_ALLOCATOR (1) #define MMTK_IMMORTAL_BUMP_ALLOCATOR (0) int jl_gc_classify_pools(size_t sz, int *osize) @@ -1153,6 +1154,17 @@ STATIC_INLINE void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* } } +STATIC_INLINE void* mmtk_nonmoving_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { + ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_NONMOVING_ALLOCATOR]; + return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 6); +} + +STATIC_INLINE void mmtk_nonmoving_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); + } +} + STATIC_INLINE void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { BumpAllocator* allocator = &mutator->allocators.bump_pointer[MMTK_IMMORTAL_BUMP_ALLOCATOR]; return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (uintptr_t)allocator->limit, size, align, offset, 1); @@ -1190,6 +1202,32 @@ JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int osize, siz return v; } +JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_nonmoving(jl_ptls_t ptls, int osize, size_t align, void *ty) +{ + // safepoint + jl_gc_safepoint_(ptls); + + jl_value_t *v; + if ((uintptr_t)ty != jl_buff_tag) { + // v needs to be 16 byte aligned, therefore v_tagged needs to be offset accordingly to consider the size of header + jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_nonmoving_alloc_fast(&ptls->gc_tls.mmtk_mutator, LLT_ALIGN(osize, align), align, sizeof(jl_taggedvalue_t)); + v = jl_valueof(v_tagged); + mmtk_nonmoving_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, v, LLT_ALIGN(osize, align)); + } else { + // allocating an extra word to store the size of buffer objects + jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_nonmoving_alloc_fast(&ptls->gc_tls.mmtk_mutator, LLT_ALIGN(osize+sizeof(jl_taggedvalue_t), align), align, 0); + jl_value_t* v_tagged_aligned = ((jl_value_t*)((char*)(v_tagged) + sizeof(jl_taggedvalue_t))); + v = jl_valueof(v_tagged_aligned); + mmtk_store_obj_size_c(v, LLT_ALIGN(osize+sizeof(jl_taggedvalue_t), align)); + mmtk_nonmoving_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, v, LLT_ALIGN(osize+sizeof(jl_taggedvalue_t), align)); + } + + ptls->gc_tls_common.gc_num.allocd += osize; + ptls->gc_tls_common.gc_num.poolalloc++; + + return v; +} + JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t sz) { // safepoint @@ -1262,10 +1300,18 @@ inline jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) inline jl_value_t *jl_gc_alloc_nonmoving_(jl_ptls_t ptls, size_t sz, void *ty) { - // TODO: Currently we just alloc and pin the object. We may use a - // different non moving allocator instead. - jl_value_t *v = jl_gc_alloc_(ptls, sz, ty); - OBJ_PIN(v); + jl_value_t *v; + const size_t allocsz = sz + sizeof(jl_taggedvalue_t); + if (sz <= GC_MAX_SZCLASS) { + v = jl_mmtk_gc_alloc_nonmoving(ptls, allocsz, 16, ty); + } + else { + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + v = jl_mmtk_gc_alloc_big(ptls, allocsz); + } + jl_set_typeof(v, ty); + maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty); return v; } From 9d8c245fa2e3264f53f530733bb044826c532995 Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Mon, 19 May 2025 06:14:44 +0000 Subject: [PATCH 644/662] Allocating weak references, genericmemory and buffers into a non-moving space --- src/gc-mmtk.c | 5 +++-- src/genericmemory.c | 8 ++------ src/julia_internal.h | 3 +-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 5ced787d3c282..62be0826c629d 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -244,6 +244,7 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { return; } mmtk_handle_user_collection_request(ptls, collection); + // print_fragmentation(); } @@ -334,6 +335,7 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) SetLastError(last_error); #endif errno = last_errno; + // print_fragmentation(); } JL_DLLEXPORT unsigned char jl_gc_pin_object(void* obj) { @@ -1068,9 +1070,8 @@ JL_DLLEXPORT size_t jl_gc_genericmemory_how(void *arg) JL_NOTSAFEPOINT JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls, jl_value_t *value) { - jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*), jl_weakref_type); + jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc_nonmoving(ptls, sizeof(void*), jl_weakref_type); wr->value = value; // NOTE: wb not needed here - OBJ_PIN(wr) // Note: we are using MMTk's weak ref processing. If we switch to Julia's weak ref processing, // we need to make sure the value and the weak ref won't be moved (e.g. pin them) mmtk_add_weak_candidate(wr); diff --git a/src/genericmemory.c b/src/genericmemory.c index 78f96e496c7a5..52f52b8d978cc 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -38,16 +38,13 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory_unchecked(jl_ptls_t ptls data = (char*)jl_gc_managed_malloc(nbytes); tot = sizeof(jl_genericmemory_t) + sizeof(void*); } - m = (jl_genericmemory_t*)jl_gc_alloc(ptls, tot, mtype); + m = (jl_genericmemory_t*)jl_gc_alloc_nonmoving(ptls, tot, mtype); if (pooled) { data = (char*)m + JL_SMALL_BYTE_ALIGNMENT; - // Data is inlined and ptr is an internal pointer. We pin the object so the ptr will not be invalid. - OBJ_PIN(m); } else { int isaligned = 1; // jl_gc_managed_malloc is always aligned jl_gc_track_malloced_genericmemory(ptls, m, isaligned); - OBJ_PIN(m); jl_genericmemory_data_owner_field(m) = (jl_value_t*)m; } // length set by codegen @@ -110,12 +107,11 @@ JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str) return (jl_genericmemory_t*)((jl_datatype_t*)jl_memory_uint8_type)->instance; jl_task_t *ct = jl_current_task; int tsz = sizeof(jl_genericmemory_t) + sizeof(void*); - jl_genericmemory_t *m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tsz, jl_memory_uint8_type); + jl_genericmemory_t *m = (jl_genericmemory_t*)jl_gc_alloc_nonmoving(ct->ptls, tsz, jl_memory_uint8_type); m->length = jl_string_len(str); m->ptr = jl_string_data(str); jl_genericmemory_data_owner_field(m) = str; OBJ_PIN(str); - OBJ_PIN(m); // FIXME: without this pin, make test-Tar fail with stress copying return m; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 10c9d190b33a8..928e7b65b08da 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -602,10 +602,9 @@ JL_DLLEXPORT uintptr_t jl_get_buff_tag(void) JL_NOTSAFEPOINT; typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { - jl_gc_tracked_buffer_t *buf = jl_gc_alloc(ptls, sz, (void*)jl_buff_tag); + jl_gc_tracked_buffer_t *buf = jl_gc_alloc_nonmoving(ptls, sz, (void*)jl_buff_tag); // FIXME: we might not need to pin buffers since the introduction of jl_genericmemory_t objects // but removing the pin below will currently fail an assertion in the binding - OBJ_PIN(buf); return buf; } From 7dff6ad3f63124d220b606f0de55fe2ce06058df Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Mon, 19 May 2025 06:54:14 +0000 Subject: [PATCH 645/662] Print utilization stats for stock GC --- src/gc-stock.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/gc-stock.c b/src/gc-stock.c index b9af37a931483..491c7c06fca74 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3538,6 +3538,12 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) SetLastError(last_error); #endif errno = last_errno; + + int64_t pool_live_bytes = jl_gc_pool_live_bytes(); + int64_t pool_bytes_in_pages = jl_atomic_load_relaxed(&gc_heap_stats.bytes_resident); + jl_safe_printf("Utilization in pool allocator: %f, %lld live bytes and %lld bytes in pages\n", + (double)pool_live_bytes / (double)pool_bytes_in_pages, + (long long)pool_live_bytes, (long long)pool_bytes_in_pages); } void gc_mark_queue_all_roots(jl_ptls_t ptls, jl_gc_markqueue_t *mq) From 7904cc543cd4f0e23664f2adc081c3f5b1a93c00 Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Tue, 20 May 2025 05:59:23 +0000 Subject: [PATCH 646/662] Adding options to have every GC as defrag (MMTK_ALWAYS_MOVING) and copy every object when possible (MMTK_MAX_MOVING) --- deps/mmtk_julia.mk | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index 9e3f8e46d606d..573cee41176da 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -1,11 +1,13 @@ ## MMTK ## # Both MMTK_MOVING and MMTK_PLAN should be specified in the Make.user file. -# FIXME: By default we do a non-moving build. We should change the default to 1 -# once we support moving plans. +# Whether to move objects at all (standard immix) MMTK_MOVING ?= 0 -MMTK_MOVING_STRESS ?= 0 -MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) MMTK_MOVING_STRESS=$(MMTK_MOVING_STRESS) +# Every GC becomes a defrag GC +MMTK_ALWAYS_MOVING ?= 0 +# Copies every object when possible +MMTK_MAX_MOVING ?= 0 +MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) MMTK_ALWAYS_MOVING=$(MMTK_ALWAYS_MOVING) MMTK_MAX_MOVING=$(MMTK_MAX_MOVING) ifneq ($(USE_BINARYBUILDER_MMTK_JULIA),1) $(eval $(call git-external,mmtk_julia,MMTK_JULIA,,,$(BUILDDIR))) From 7b9b1231c3801fb44090d599068093c1eb482804 Mon Sep 17 00:00:00 2001 From: Luis Eduardo de Souza Amorim Date: Fri, 23 May 2025 01:13:28 +0000 Subject: [PATCH 647/662] Moving jl_genericmemory_t objects to the immix space --- src/gc-mmtk.c | 12 ++++++++++++ src/genericmemory.c | 2 +- src/julia_internal.h | 2 -- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 62be0826c629d..3086ce353f485 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -881,6 +881,8 @@ static void jl_gc_free_memory(jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPO gc_num.freecall++; } +extern void* mmtk_get_possibly_forwarded(void* object); + JL_DLLEXPORT void jl_gc_mmtk_sweep_malloced_memory(void) JL_NOTSAFEPOINT { void* iter = mmtk_new_mutator_iterator(); @@ -893,6 +895,11 @@ JL_DLLEXPORT void jl_gc_mmtk_sweep_malloced_memory(void) JL_NOTSAFEPOINT while (n < l) { jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[n] & ~1); if (mmtk_is_live_object(m)) { + jl_genericmemory_t *maybe_forwarded = (jl_genericmemory_t*)mmtk_get_possibly_forwarded(m); + if(maybe_forwarded != m) { + int isaligned = (uintptr_t)lst[n] & 1; + lst[n] = (void*)((uintptr_t)maybe_forwarded | !!isaligned); + } n++; } else { @@ -929,6 +936,11 @@ JL_DLLEXPORT void jl_gc_update_inlined_array(void* from, void* to) { b->ptr = (void*)((size_t) b + offset_of_data); } } + + // make sure we update the owner pointer when we're copying the generic memory + if (how == 1) { + jl_genericmemory_data_owner_field(b) = (jl_value_t*)b; + } } } diff --git a/src/genericmemory.c b/src/genericmemory.c index 52f52b8d978cc..9ec6dd8c6a4f5 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -38,7 +38,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory_unchecked(jl_ptls_t ptls data = (char*)jl_gc_managed_malloc(nbytes); tot = sizeof(jl_genericmemory_t) + sizeof(void*); } - m = (jl_genericmemory_t*)jl_gc_alloc_nonmoving(ptls, tot, mtype); + m = (jl_genericmemory_t*)jl_gc_alloc(ptls, tot, mtype); if (pooled) { data = (char*)m + JL_SMALL_BYTE_ALIGNMENT; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 928e7b65b08da..c0b18888588bd 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -603,8 +603,6 @@ typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { jl_gc_tracked_buffer_t *buf = jl_gc_alloc_nonmoving(ptls, sz, (void*)jl_buff_tag); - // FIXME: we might not need to pin buffers since the introduction of jl_genericmemory_t objects - // but removing the pin below will currently fail an assertion in the binding return buf; } From 2e8682d88eb0625805c955acfe315856c040f206 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 27 May 2025 01:10:38 +0000 Subject: [PATCH 648/662] Add functions in the GC interface to get object hash --- src/builtins.c | 17 +++-------------- src/gc-interface.h | 3 +++ src/gc-mmtk.c | 8 ++++++++ src/gc-stock.c | 8 ++++++++ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index e972a1839a5e0..c980caa39faa7 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -367,13 +367,10 @@ static uintptr_t type_object_id_(jl_value_t *v, jl_varidx_t *env) JL_NOTSAFEPOIN i++; pe = pe->prev; } - // FIXME: Pinning objects that get hashed - // until we implement address space hashing. - OBJ_PIN(v); uintptr_t bits = jl_astaggedvalue(v)->header; if (bits & GC_IN_IMAGE) return ((uintptr_t*)v)[-2]; - return inthash((uintptr_t)v); + return inthash(jl_gc_get_obj_hash(v)); } if (tv == jl_uniontype_type) { return bitmix(bitmix(jl_object_id((jl_value_t*)tv), @@ -426,12 +423,7 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT // a few select pointers (notably symbol) also have special hash values // which may affect the stability of the objectid hash, even though // they don't affect egal comparison - - // FIXME: Pinning objects that get hashed - // until we implement address space hashing. - PTR_PIN(v); // This has to be a pointer pin -- v could be an internal pointer - - return bits_hash(v, sz) ^ h; + return bits_hash((const void*)jl_gc_get_ptr_hash(v), sz) ^ h; } if (dt == jl_unionall_type) return type_object_id_(v, NULL); @@ -492,10 +484,7 @@ static uintptr_t NOINLINE jl_object_id__cold(uintptr_t tv, jl_value_t *v) JL_NOT if (bits & GC_IN_IMAGE) return ((uintptr_t*)v)[-2]; - // FIXME: Pinning objects that get hashed - // until we implement address space hashing. - OBJ_PIN(v); - return inthash((uintptr_t)v); + return inthash(jl_gc_get_obj_hash(v)); } return immut_id_(dt, v, dt->hash); } diff --git a/src/gc-interface.h b/src/gc-interface.h index 446d35f45d336..0b1804afb1ab3 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -123,6 +123,9 @@ JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT; // Runtime hook for gc preserve end. The GC needs to make sure that the preserved objects and its children stay alive and won't move. JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT; +JL_DLLEXPORT uintptr_t jl_gc_get_obj_hash(void* obj) JL_NOTSAFEPOINT; +JL_DLLEXPORT uintptr_t jl_gc_get_ptr_hash(void* ptr) JL_NOTSAFEPOINT; + // ========================================================================= // // Metrics // ========================================================================= // diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 3086ce353f485..481b2bcc9c82b 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -1627,6 +1627,14 @@ JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT JL_GC_POP_PRESERVE_ROOT_OBJS(); } +JL_DLLEXPORT uintptr_t jl_gc_get_obj_hash(void* obj) JL_NOTSAFEPOINT { + return mmtk_get_object_hash(obj); +} + +JL_DLLEXPORT uintptr_t jl_gc_get_ptr_hash(void* ptr) JL_NOTSAFEPOINT { + return mmtk_get_ptr_hash(ptr); +} + #ifdef __cplusplus } #endif diff --git a/src/gc-stock.c b/src/gc-stock.c index 491c7c06fca74..c8fc8194dce14 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -4162,6 +4162,14 @@ JL_DLLEXPORT void jl_gc_notify_thread_yield(jl_ptls_t ptls, void* ctx) { // Do nothing before a thread yields } +JL_DLLEXPORT uintptr_t jl_gc_get_obj_hash(void* obj) JL_NOTSAFEPOINT { + return (uintptr_t)obj; +} + +JL_DLLEXPORT uintptr_t jl_gc_get_ptr_hash(void* ptr) JL_NOTSAFEPOINT { + return (uintptr_t)ptr; +} + #ifdef __cplusplus } #endif From 1e548d460cc22ef776342633e681f3bceb1efccb Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 28 May 2025 06:25:27 +0000 Subject: [PATCH 649/662] Copying libmmtk_julia.so to the build dir --- deps/mmtk_julia.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index 573cee41176da..3eb470cbe6b3b 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -25,7 +25,8 @@ $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/build-compiled: $(BUILDROOT)/usr/lib/libmmtk_j # NB: use the absolute dir when creating the symlink $(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so - @ln -sf $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so $@ + echo "Copying MMTK Julia to $(BUILDROOT)/usr/lib/libmmtk_julia.so" + @cp $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so $@ $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so: $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/source-extracted @$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD) @@ -52,7 +53,8 @@ compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so version-check-mmtk_julia: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $(BUILDROOT)/usr/lib/libmmtk_julia.so: make-binding - @ln -sf $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $@ + echo "Copying MMTK Julia to $(BUILDROOT)/usr/lib/libmmtk_julia.so" + @cp $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $@ make-binding: @$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD) From dd99627a51b152b30bb35a80ee81cf622adb7183 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 29 May 2025 05:51:42 +0000 Subject: [PATCH 650/662] bits_hash uses the value of the pointers, not the pointers. --- src/builtins.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/builtins.c b/src/builtins.c index c980caa39faa7..684ca3a691f7d 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -423,7 +423,8 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT // a few select pointers (notably symbol) also have special hash values // which may affect the stability of the objectid hash, even though // they don't affect egal comparison - return bits_hash((const void*)jl_gc_get_ptr_hash(v), sz) ^ h; + // return bits_hash((const void*)jl_gc_get_ptr_hash(v), sz) ^ h; + return bits_hash(v, sz) ^ h; } if (dt == jl_unionall_type) return type_object_id_(v, NULL); From 7f02556a2c17fb642a0825d32ce000d262d5719e Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 10 Jun 2025 03:52:52 +0000 Subject: [PATCH 651/662] Allow dump some heap stats for MMTk --- deps/mmtk_julia.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk index 3eb470cbe6b3b..65223d87580d5 100644 --- a/deps/mmtk_julia.mk +++ b/deps/mmtk_julia.mk @@ -7,7 +7,10 @@ MMTK_MOVING ?= 0 MMTK_ALWAYS_MOVING ?= 0 # Copies every object when possible MMTK_MAX_MOVING ?= 0 -MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) MMTK_ALWAYS_MOVING=$(MMTK_ALWAYS_MOVING) MMTK_MAX_MOVING=$(MMTK_MAX_MOVING) +MMTK_DUMP_FRAGMENTATION ?= 0 +MMTK_DUMP_BLOCK_STATS ?= 0 +MMTK_DUMP_HEAP ?= 0 +MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) MMTK_ALWAYS_MOVING=$(MMTK_ALWAYS_MOVING) MMTK_MAX_MOVING=$(MMTK_MAX_MOVING) MMTK_DUMP_FRAGMENTATION=$(MMTK_DUMP_FRAGMENTATION) MMTK_DUMP_BLOCK_STATS=$(MMTK_DUMP_BLOCK_STATS) MMTK_DUMP_HEAP=$(MMTK_DUMP_HEAP) ifneq ($(USE_BINARYBUILDER_MMTK_JULIA),1) $(eval $(call git-external,mmtk_julia,MMTK_JULIA,,,$(BUILDDIR))) From 1cee40f50f7d2f0e37476f47b37d649f3c85773c Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 12 Jun 2025 01:51:32 +0000 Subject: [PATCH 652/662] Pass hard-heap-limit to MMTk --- src/gc-mmtk.c | 73 +++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 481b2bcc9c82b..ef70a232a9e4f 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -108,36 +108,46 @@ void jl_gc_init(void) { // MMTK_MIN_HSIZE and MMTK_MAX_HSIZE environment variables long long min_heap_size; long long max_heap_size; - char* min_size_def = getenv("MMTK_MIN_HSIZE"); - char* min_size_gb = getenv("MMTK_MIN_HSIZE_G"); - - char* max_size_def = getenv("MMTK_MAX_HSIZE"); - char* max_size_gb = getenv("MMTK_MAX_HSIZE_G"); - - // If min and max values are not specified, set them to 0 here - // and use stock heuristics as defined in the binding - if (min_size_def != NULL) { - char *p; - double min_size = strtod(min_size_def, &p); - min_heap_size = (long) 1024 * 1024 * min_size; - } else if (min_size_gb != NULL) { - char *p; - double min_size = strtod(min_size_gb, &p); - min_heap_size = (long) 1024 * 1024 * 1024 * min_size; - } else { - min_heap_size = 0; - } - if (max_size_def != NULL) { - char *p; - double max_size = strtod(max_size_def, &p); - max_heap_size = (long) 1024 * 1024 * max_size; - } else if (max_size_gb != NULL) { - char *p; - double max_size = strtod(max_size_gb, &p); - max_heap_size = (long) 1024 * 1024 * 1024 * max_size; + if (jl_options.hard_heap_limit != 0) { + min_heap_size = jl_options.hard_heap_limit; + max_heap_size = jl_options.hard_heap_limit; } else { - max_heap_size = 0; + char* min_size_def = getenv("MMTK_MIN_HSIZE"); + char* min_size_gb = getenv("MMTK_MIN_HSIZE_G"); + char* max_size_def = getenv("MMTK_MAX_HSIZE"); + char* max_size_gb = getenv("MMTK_MAX_HSIZE_G"); + + // If min and max values are not specified, set them to 0 here + // and use stock heuristics as defined in the binding + if (min_size_def != NULL) { + char *p; + double min_size = strtod(min_size_def, &p); + min_heap_size = (long) 1024 * 1024 * min_size; + } else if (min_size_gb != NULL) { + char *p; + double min_size = strtod(min_size_gb, &p); + min_heap_size = (long) 1024 * 1024 * 1024 * min_size; + } else { + min_heap_size = 0; + } + + if (max_size_def != NULL) { + char *p; + double max_size = strtod(max_size_def, &p); + max_heap_size = (long) 1024 * 1024 * max_size; + } else if (max_size_gb != NULL) { + char *p; + double max_size = strtod(max_size_gb, &p); + max_heap_size = (long) 1024 * 1024 * 1024 * max_size; + } else { + max_heap_size = 0; + } + + // if only max size is specified initialize MMTk with a fixed size heap + if (max_size_def != NULL || (max_size_gb != NULL && (min_size_def == NULL && min_size_gb == NULL))) { + min_heap_size = 0; + } } // Assert that the number of stock GC threads is 0; MMTK uses the number of threads in jl_options.ngcthreads @@ -154,15 +164,10 @@ void jl_gc_init(void) { mmtk_julia_copy_stack_check(copy_stacks); - // if only max size is specified initialize MMTk with a fixed size heap // TODO: We just assume mark threads means GC threads, and ignore the number of concurrent sweep threads. // If the two values are the same, we can use either. Otherwise, we need to be careful. uintptr_t gcthreads = jl_options.nmarkthreads; - if (max_size_def != NULL || (max_size_gb != NULL && (min_size_def == NULL && min_size_gb == NULL))) { - mmtk_gc_init(0, max_heap_size, gcthreads, (sizeof(jl_taggedvalue_t)), jl_buff_tag); - } else { - mmtk_gc_init(min_heap_size, max_heap_size, gcthreads, (sizeof(jl_taggedvalue_t)), jl_buff_tag); - } + mmtk_gc_init(min_heap_size, max_heap_size, gcthreads, (sizeof(jl_taggedvalue_t)), jl_buff_tag); } void jl_start_gc_threads(void) { From fa73fa52195c3171fff898147516856c2b82aa79 Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Fri, 6 Jun 2025 13:22:19 -0300 Subject: [PATCH 653/662] pinning log --- src/Makefile | 4 +- src/gc-mmtk.c | 7 ++ src/gc-pinning-log.cpp | 192 +++++++++++++++++++++++++++++++++++++++++ src/julia.h | 132 ++++++++++++++++------------ 4 files changed, 280 insertions(+), 55 deletions(-) create mode 100644 src/gc-pinning-log.cpp diff --git a/src/Makefile b/src/Makefile index 37e58f0620760..de33986abca58 100644 --- a/src/Makefile +++ b/src/Makefile @@ -48,7 +48,7 @@ FLAGS += -I$(LOCALBASE)/include endif # GC source code. It depends on which GC implementation to use. -GC_SRCS := gc-common gc-stacks gc-alloc-profiler gc-heap-snapshot +GC_SRCS := gc-common gc-stacks gc-alloc-profiler gc-heap-snapshot gc-pinning-log ifeq (${USE_THIRD_PARTY_GC},mmtk) GC_SRCS += gc-mmtk else @@ -69,7 +69,7 @@ CG_LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) # Currently these files are used by both GCs. But we should make the list specific to stock, and MMTk should have its own implementation. -GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier +GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier gc-pinning-log ifeq (${USE_THIRD_PARTY_GC},mmtk) FLAGS += -I$(MMTK_API_INC) GC_CODEGEN_SRCS += llvm-late-gc-lowering-mmtk diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index ef70a232a9e4f..030e89fa7e4ff 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -59,6 +59,8 @@ extern void mmtk_post_alloc(void* mutator, void* refer, size_t bytes, int alloca extern void mmtk_store_obj_size_c(void* obj, size_t size); extern bool mmtk_is_pinned(void* obj); extern unsigned char mmtk_pin_object(void* obj); +extern bool mmtk_is_reachable_object(void* obj); +extern bool mmtk_is_live_object(void* obj); extern bool mmtk_is_object_pinned(void* obj); extern unsigned char mmtk_pin_pointer(void* ptr); extern bool mmtk_is_pointer_pinned(void* ptr); @@ -73,6 +75,8 @@ void jl_gc_init(void) { // TODO: use jl_options.heap_size_hint to set MMTk's fixed heap size? (see issue: https://github.com/mmtk/mmtk-julia/issues/167) JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock"); + jl_set_check_alive_fn(mmtk_is_reachable_object); + arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); gc_num.interval = default_collect_interval; @@ -341,6 +345,9 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) #endif errno = last_errno; // print_fragmentation(); +#ifdef ENABLE_PINNING_LOGGING + jl_print_pinning_log(); +#endif } JL_DLLEXPORT unsigned char jl_gc_pin_object(void* obj) { diff --git a/src/gc-pinning-log.cpp b/src/gc-pinning-log.cpp new file mode 100644 index 0000000000000..15349c31beeae --- /dev/null +++ b/src/gc-pinning-log.cpp @@ -0,0 +1,192 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include +#include +#include +#include + +#include "julia.h" + +struct pinning_site_t { + int lineno; + const char *filename; + pinning_site_t() : lineno(0), filename(nullptr) {} + pinning_site_t(int line, const char *file) : lineno(line), filename(file) {} + bool operator<(const pinning_site_t &other) const { + if (lineno != other.lineno) { + return lineno < other.lineno; + } + return filename < other.filename; + } + bool operator==(const pinning_site_t &other) const { + return lineno == other.lineno && filename == other.filename; + } +}; + +struct pinning_log_entry_t { + void *pinned_object; + pinning_site_t site; + pinning_log_entry_t() : pinned_object(nullptr), site(pinning_site_t{}) {} +}; + +struct linear_pinning_log_t { +#define BUFFER_CAPACITY (1ULL << 20) + size_t idx; + pinning_log_entry_t buffer[BUFFER_CAPACITY]; + linear_pinning_log_t() : idx(0) { + for (size_t i = 0; i < BUFFER_CAPACITY; ++i) { + buffer[i] = pinning_log_entry_t{}; + } + } + pinning_log_entry_t *bump_alloc_log_entry() { + if (idx >= BUFFER_CAPACITY) { + assert(0 && "Exceeded buffer capacity"); + } + auto e = &buffer[idx++]; + return e; + } + void reset_log_buffer() { + idx = 0; + for (size_t i = 0; i < BUFFER_CAPACITY; ++i) { + buffer[i] = pinning_log_entry_t{}; + } + } +}; + +struct coalesced_pinning_log_t { + std::map> objects_to_pinning_sites; + coalesced_pinning_log_t() = default; + void add_pinning_event(void *pinned_object, const pinning_site_t &site) { + if (objects_to_pinning_sites.find(pinned_object) == objects_to_pinning_sites.end()) { + std::map site_map{}; + objects_to_pinning_sites[pinned_object] = std::map{}; + } + auto &site_map = objects_to_pinning_sites[pinned_object]; + if (site_map.find(site) == site_map.end()) { + site_map[site] = 0; + } + site_map[site]++; + } +}; + +struct pinning_log_t { + linear_pinning_log_t linear_log; + coalesced_pinning_log_t coalesced_log; + check_alive_fn is_alive; + std::mutex mu; + pinning_log_entry_t *alloc_pinning_log_entry(void *pinned_object) { + pinning_log_entry_t *e; + mu.lock(); + e = linear_log.bump_alloc_log_entry(); + mu.unlock(); + return e; + } + void coalesce_linear_pinning_log() { + mu.lock(); + for (size_t i = 0; i < linear_log.idx; ++i) { + auto &entry = linear_log.buffer[i]; + if (entry.pinned_object != nullptr) { + coalesced_log.add_pinning_event(entry.pinned_object, entry.site); + } + } + linear_log.reset_log_buffer(); + mu.unlock(); + } + void set_check_alive_fn(check_alive_fn fn) { + mu.lock(); + is_alive = fn; + mu.unlock(); + } + void gc_log(void) { + coalesce_linear_pinning_log(); + mu.lock(); + for (auto it = coalesced_log.objects_to_pinning_sites.begin(); it != coalesced_log.objects_to_pinning_sites.end();) { + if (!is_alive(reinterpret_cast(it->first))) { + it = coalesced_log.objects_to_pinning_sites.erase(it); + } else { + ++it; + } + } + mu.unlock(); + } + void print_pinning_log_as_json() { + mu.lock(); + std::cerr << "[\n"; + bool first = true; + for (const auto &object_entry : coalesced_log.objects_to_pinning_sites) { + if (!first) { + std::cerr << ",\n"; + } + first = false; + std::cerr << " {\n"; + std::cerr << " \"pinned_object\": \"" << object_entry.first << "\",\n"; + const char *type; + if (is_alive(reinterpret_cast(object_entry.first))) { + type = jl_typeof_str(reinterpret_cast(object_entry.first)); + } else { + type = "unknown"; + } + std::cerr << " \"type\": \"" << type << "\",\n"; + std::cerr << " \"pinning_sites\": [\n"; + bool first_site = true; + for (const auto &site_entry : object_entry.second) { + if (!first_site) { + std::cerr << ",\n"; + } + first_site = false; + std::cerr << " {\n"; + auto filename = site_entry.first.filename ? site_entry.first.filename : "unknown"; + std::cerr << " \"filename\": \"" << filename << "\",\n"; + std::cerr << " \"lineno\": " << site_entry.first.lineno << ",\n"; + std::cerr << " \"count\": " << site_entry.second << "\n"; + std::cerr << " }"; + } + std::cerr << "\n ]\n"; + std::cerr << " }"; + } + std::cerr << "\n]\n"; + mu.unlock(); + } +}; + +pinning_log_t pinning_log; + +#ifdef __cplusplus +extern "C" { +#endif + +int pinning_log_enabled; + +JL_DLLEXPORT void jl_set_check_alive_fn(check_alive_fn fn) { + pinning_log.set_check_alive_fn(fn); +} +JL_DLLEXPORT void jl_enable_pinning_log(void) { + pinning_log_enabled = 1; +} +JL_DLLEXPORT void jl_gc_log(void) { + if (!pinning_log_enabled) { + return; + } + pinning_log.gc_log(); +} +JL_DLLEXPORT void jl_log_pinning_event(void *pinned_object, const char *filename, int lineno) { + if (!pinning_log_enabled) { + return; + } + auto pe = pinning_log.alloc_pinning_log_entry(pinned_object); + pe->pinned_object = pinned_object; + pe->site.lineno = lineno; + pe->site.filename = filename; +} +JL_DLLEXPORT void jl_print_pinning_log(void) { + if (!pinning_log_enabled) { + return; + } + pinning_log.coalesce_linear_pinning_log(); + pinning_log.print_pinning_log_as_json(); + jl_safe_printf("=========================\n"); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/julia.h b/src/julia.h index e387aec7c8645..88a06c8fd3bb4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -20,6 +20,7 @@ #include "libsupport.h" #include #include +#include #include "htable.h" #include "arraylist.h" @@ -91,59 +92,6 @@ typedef struct _jl_value_t jl_value_t; extern "C" { #endif -// object pinning ------------------------------------------------------------ - -// FIXME: Pinning objects that get hashed in the ptrhash table -// until we implement address space hashing. -#define OBJHASH_PIN(key) if (key) jl_gc_pin_object(key); -#define PTRHASH_PIN(key) if (key) jl_gc_pin_pointer(key); - -// Called when pinning objects that would cause an error if moved -// The difference: the argument for pin_object needs to pointer to an object (jl_value_t*), -// but the argument for pin_pointer can be an internal pointer. -#define OBJ_PIN(key) if (key) jl_gc_pin_object(key); -#define PTR_PIN(key) if (key) jl_gc_pin_pointer(key); - -#ifdef __cplusplus -} // extern "C" -#endif - -#ifdef __cplusplus - -// C++ template version -template -class pinned_ref { - T* ptr; -public: - explicit pinned_ref() : ptr(static_cast(assume(NULL))) {} - explicit pinned_ref(void* p) : ptr(static_cast(p)) {} - operator void*() const { return ptr; } - T* get() const { return ptr; } - static pinned_ref create(void* p) { OBJ_PIN(p); return pinned_ref(p); } - static pinned_ref assume(void* p) { return pinned_ref(p); } -}; - -// Redefine macros for C++ to use the template version -#define jl_pinned_ref(T) pinned_ref -#define jl_pinned_ref_assume(T, ptr) pinned_ref::assume(ptr) -#define jl_pinned_ref_create(T, ptr) pinned_ref::create(ptr) -#define jl_pinned_ref_get(ref) (ref).get() - -#else - -// Primary type definition -#define jl_pinned_ref(T) union { T* t; void* unused; } -#define jl_pinned_ref_assume(T, ptr) ((jl_pinned_ref(T)){ .t = (ptr) }) -// Assignment macro -#define jl_pinned_ref_set(lhs, ptr) OBJ_PIN(ptr); jl_pinned_ref_get(lhs) = ptr; -// Getter macro -#define jl_pinned_ref_get(ref) ((ref).t) - -#endif - -#ifdef __cplusplus -extern "C" { -#endif // core data types ------------------------------------------------------------ struct _jl_taggedvalue_bits { @@ -1427,6 +1375,84 @@ STATIC_INLINE jl_value_t *jl_svecset( JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; #define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) +// object pinning ------------------------------------------------------------ + +typedef bool (*check_alive_fn)(void *); +JL_DLLEXPORT void jl_set_check_alive_fn(check_alive_fn fn); +JL_DLLEXPORT void jl_log_pinning_event(void *pinned_object, const char *filename, int lineno); +JL_DLLEXPORT void jl_print_pinning_log(void); + +#define ENABLE_PINNING_LOGGING +#ifdef ENABLE_PINNING_LOGGING +#define LOG_PINNING_EVENT(key) do { \ + jl_log_pinning_event(key, __FILE__, __LINE__); \ +} while (0); +#else +#define LOG_PINNING_EVENT(key) ; +#endif + +// FIXME: Pinning objects that get hashed in the ptrhash table +// until we implement addrePTR_PINss space hashing. +#define OBJHASH_PIN(key) do { \ + if (key) { \ + LOG_PINNING_EVENT(key); \ + jl_gc_pin_object(key); \ + } \ +} while (0); +#define PTRHASH_PIN(key) if (key) jl_gc_pin_pointer(key); + +// Called when pinning objects that would cause an error if moved +// The difference: the argument for pin_object needs to pointer to an object (jl_value_t*), +// but the argument for pin_pointer can be an internal pointer. +#define OBJ_PIN(key) do { \ + if (key) { \ + LOG_PINNING_EVENT(key); \ + jl_gc_pin_object(key); \ + } \ +} while (0); +#define PTR_PIN(key) if (key) jl_gc_pin_pointer(key); + +#ifdef __cplusplus +} // extern "C" +#endif + +#ifdef __cplusplus + +// C++ template version +template +class pinned_ref { + T* ptr; +public: + explicit pinned_ref() : ptr(static_cast(assume(NULL))) {} + explicit pinned_ref(void* p) : ptr(static_cast(p)) {} + operator void*() const { return ptr; } + T* get() const { return ptr; } + static pinned_ref create(void* p) { OBJ_PIN(p); return pinned_ref(p); } + static pinned_ref assume(void* p) { return pinned_ref(p); } +}; + +// Redefine macros for C++ to use the template version +#define jl_pinned_ref(T) pinned_ref +#define jl_pinned_ref_assume(T, ptr) pinned_ref::assume(ptr) +#define jl_pinned_ref_create(T, ptr) pinned_ref::create(ptr) +#define jl_pinned_ref_get(ref) (ref).get() + +#else + +// Primary type definition +#define jl_pinned_ref(T) union { T* t; void* unused; } +#define jl_pinned_ref_assume(T, ptr) ((jl_pinned_ref(T)){ .t = (ptr) }) +// Assignment macro +#define jl_pinned_ref_set(lhs, ptr) OBJ_PIN(ptr); jl_pinned_ref_get(lhs) = ptr; +// Getter macro +#define jl_pinned_ref_get(ref) ((ref).t) + +#endif + +#ifdef __cplusplus +extern "C" { +#endif + STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; // write barriers From d10de94c219d84384eb8447ac38391091b6caa5c Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Mon, 16 Jun 2025 17:49:57 -0300 Subject: [PATCH 654/662] Don't pin objects permanently in engine.cpp --- src/engine.cpp | 23 +++++++++++++---------- src/gc-common.c | 2 ++ src/gc-mmtk.c | 27 ++++++++++++++++++++++++++- src/gc-pinning-log.cpp | 8 ++++---- src/julia.h | 9 +++++---- 5 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/engine.cpp b/src/engine.cpp index 08a9c300c4668..e6a8d2c452fed 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -18,7 +18,7 @@ struct ReservationInfo { struct InferKey { jl_method_instance_t *mi = nullptr; - jl_pinned_ref(jl_value_t) owner = jl_pinned_ref_assume(jl_value_t, nullptr); + jl_value_t *owner = nullptr; }; template<> struct llvm::DenseMapInfo { @@ -26,18 +26,16 @@ template<> struct llvm::DenseMapInfo { using SecondInfo = DenseMapInfo; static inline InferKey getEmptyKey() { - return InferKey{FirstInfo::getEmptyKey(), - jl_pinned_ref_assume(jl_value_t, SecondInfo::getEmptyKey())}; + return InferKey{FirstInfo::getEmptyKey(), SecondInfo::getEmptyKey()}; } static inline InferKey getTombstoneKey() { - return InferKey{FirstInfo::getTombstoneKey(), - jl_pinned_ref_assume(jl_value_t, SecondInfo::getTombstoneKey())}; + return InferKey{FirstInfo::getTombstoneKey(), SecondInfo::getTombstoneKey()}; } static unsigned getHashValue(const InferKey& PairVal) { return detail::combineHashValue(FirstInfo::getHashValue(PairVal.mi), - SecondInfo::getHashValue(jl_pinned_ref_get(PairVal.owner))); + SecondInfo::getHashValue(PairVal.owner)); } static bool isEqual(const InferKey &LHS, const InferKey &RHS) { @@ -66,13 +64,15 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto tid = jl_atomic_load_relaxed(&ct->tid); if (([tid, m, owner, ci] () -> bool { // necessary scope block / lambda for unique_lock jl_unique_gcsafe_lock lock(engine_lock); - InferKey key{m, jl_pinned_ref_create(jl_value_t, owner)}; + arraylist_push(&objects_pinned_by_inference_engine, owner); + InferKey key{m, owner}; if ((signed)Awaiting.size() < tid + 1) Awaiting.resize(tid + 1); while (1) { auto record = Reservations.find(key); if (record == Reservations.end()) { Reservations[key] = ReservationInfo{tid, ci}; + arraylist_pop(&objects_pinned_by_inference_engine); return false; } // before waiting, need to run deadlock/cycle detection @@ -80,8 +80,10 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner // and waiting for (transitively) any lease that is held by this thread auto wait_tid = record->second.tid; while (1) { - if (wait_tid == tid) + if (wait_tid == tid) { + arraylist_pop(&objects_pinned_by_inference_engine); return true; + } if ((signed)Awaiting.size() <= wait_tid) break; // no cycle, since it is running (and this should be unreachable) auto key2 = Awaiting[wait_tid]; @@ -97,6 +99,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner lock.wait(engine_wait); Awaiting[tid] = InferKey{}; } + arraylist_pop(&objects_pinned_by_inference_engine); })()) ct->ptls->engine_nqueued--; JL_GC_POP(); @@ -106,7 +109,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner int jl_engine_hasreserved(jl_method_instance_t *m, jl_value_t *owner) { jl_task_t *ct = jl_current_task; - InferKey key = {m, jl_pinned_ref_create(jl_value_t, owner)}; + InferKey key = {m, owner}; std::unique_lock lock(engine_lock); auto record = Reservations.find(key); return record != Reservations.end() && record->second.tid == jl_atomic_load_relaxed(&ct->tid); @@ -139,7 +142,7 @@ void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) { jl_task_t *ct = jl_current_task; std::unique_lock lock(engine_lock); - auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), jl_pinned_ref_create(jl_value_t, ci->owner)}); + auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), ci->owner}); if (record == Reservations.end() || record->second.ci != ci) return; assert(jl_atomic_load_relaxed(&ct->tid) == record->second.tid); diff --git a/src/gc-common.c b/src/gc-common.c index a5067bbf125d9..f2d782834f212 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -687,6 +687,8 @@ JL_DLLEXPORT int jl_gc_enable(int on) // MISC // =========================================================================== // +arraylist_t objects_pinned_by_inference_engine; + JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { jl_ptls_t ptls = jl_current_task->ptls; diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 030e89fa7e4ff..2b0d777168e71 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -59,6 +59,7 @@ extern void mmtk_post_alloc(void* mutator, void* refer, size_t bytes, int alloca extern void mmtk_store_obj_size_c(void* obj, size_t size); extern bool mmtk_is_pinned(void* obj); extern unsigned char mmtk_pin_object(void* obj); +extern unsigned char mmtk_unpin_object(void* obj); extern bool mmtk_is_reachable_object(void* obj); extern bool mmtk_is_live_object(void* obj); extern bool mmtk_is_object_pinned(void* obj); @@ -75,8 +76,9 @@ void jl_gc_init(void) { // TODO: use jl_options.heap_size_hint to set MMTk's fixed heap size? (see issue: https://github.com/mmtk/mmtk-julia/issues/167) JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock"); - jl_set_check_alive_fn(mmtk_is_reachable_object); + jl_set_check_alive_type(mmtk_is_reachable_object); + arraylist_new(&objects_pinned_by_inference_engine, 0); arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); gc_num.interval = default_collect_interval; @@ -256,6 +258,24 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { // print_fragmentation(); } +void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) +{ + for (size_t i = 0; i < objects_pinned_by_inference_engine.len; i++) { + void *obj = objects_pinned_by_inference_engine.items[i]; + unsigned char got_pinned = mmtk_pin_object(obj); + if (got_pinned) { + arraylist_push(objects_pinned_by_call, obj); + } + } +} + +void gc_unpin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) +{ + for (size_t i = 0; i < objects_pinned_by_call->len; i++) { + void *obj = objects_pinned_by_call->items[i]; + mmtk_unpin_object(obj); + } +} // Based on jl_gc_collect from gc-stock.c // called when stopping the thread in `mmtk_block_for_gc` @@ -319,7 +339,12 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) jl_gc_notify_thread_yield(ptls, NULL); JL_LOCK_NOGC(&finalizers_lock); // all the other threads are stopped, so this does not make sense, right? otherwise, failing that, this seems like plausibly a deadlock #ifndef __clang_gcanalyzer__ + arraylist_t objects_pinned_by_call; + arraylist_new(&objects_pinned_by_call, 0); + gc_pin_objects_from_inference_engine(&objects_pinned_by_call); mmtk_block_thread_for_gc(); + gc_unpin_objects_from_inference_engine(&objects_pinned_by_call); + arraylist_free(&objects_pinned_by_call); #endif JL_UNLOCK_NOGC(&finalizers_lock); } diff --git a/src/gc-pinning-log.cpp b/src/gc-pinning-log.cpp index 15349c31beeae..dc6f478ef18d2 100644 --- a/src/gc-pinning-log.cpp +++ b/src/gc-pinning-log.cpp @@ -72,7 +72,7 @@ struct coalesced_pinning_log_t { struct pinning_log_t { linear_pinning_log_t linear_log; coalesced_pinning_log_t coalesced_log; - check_alive_fn is_alive; + check_alive_fn_type is_alive; std::mutex mu; pinning_log_entry_t *alloc_pinning_log_entry(void *pinned_object) { pinning_log_entry_t *e; @@ -92,7 +92,7 @@ struct pinning_log_t { linear_log.reset_log_buffer(); mu.unlock(); } - void set_check_alive_fn(check_alive_fn fn) { + void set_check_alive_fn_type(check_alive_fn_type fn) { mu.lock(); is_alive = fn; mu.unlock(); @@ -157,8 +157,8 @@ extern "C" { int pinning_log_enabled; -JL_DLLEXPORT void jl_set_check_alive_fn(check_alive_fn fn) { - pinning_log.set_check_alive_fn(fn); +JL_DLLEXPORT void jl_set_check_alive_type(check_alive_fn_type fn) { + pinning_log.set_check_alive_fn_type(fn); } JL_DLLEXPORT void jl_enable_pinning_log(void) { pinning_log_enabled = 1; diff --git a/src/julia.h b/src/julia.h index 88a06c8fd3bb4..a0dc85c7542c2 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1377,8 +1377,9 @@ JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_RO // object pinning ------------------------------------------------------------ -typedef bool (*check_alive_fn)(void *); -JL_DLLEXPORT void jl_set_check_alive_fn(check_alive_fn fn); +extern arraylist_t objects_pinned_by_inference_engine; +typedef bool (*check_alive_fn_type)(void *); +JL_DLLEXPORT void jl_set_check_alive_type(check_alive_fn_type fn); JL_DLLEXPORT void jl_log_pinning_event(void *pinned_object, const char *filename, int lineno); JL_DLLEXPORT void jl_print_pinning_log(void); @@ -1427,14 +1428,14 @@ class pinned_ref { explicit pinned_ref(void* p) : ptr(static_cast(p)) {} operator void*() const { return ptr; } T* get() const { return ptr; } - static pinned_ref create(void* p) { OBJ_PIN(p); return pinned_ref(p); } + static pinned_ref create(void* p, const char *file, int line) { jl_log_pinning_event(p, file, line); jl_gc_pin_object(p); return pinned_ref(p); } static pinned_ref assume(void* p) { return pinned_ref(p); } }; // Redefine macros for C++ to use the template version #define jl_pinned_ref(T) pinned_ref #define jl_pinned_ref_assume(T, ptr) pinned_ref::assume(ptr) -#define jl_pinned_ref_create(T, ptr) pinned_ref::create(ptr) +#define jl_pinned_ref_create(T, ptr) pinned_ref::create(ptr, __FILE__, __LINE__) #define jl_pinned_ref_get(ref) (ref).get() #else From 4a22f2c7f5404576eb61b3502875eb7121066afe Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Tue, 17 Jun 2025 22:43:15 -0300 Subject: [PATCH 655/662] suggestions from Yi' review --- src/engine.cpp | 10 +++++----- src/gc-common.c | 2 +- src/gc-mmtk.c | 6 +++--- src/julia.h | 2 +- src/support/analyzer_annotations.h | 2 ++ 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/engine.cpp b/src/engine.cpp index e6a8d2c452fed..324dc9f12347c 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -18,7 +18,7 @@ struct ReservationInfo { struct InferKey { jl_method_instance_t *mi = nullptr; - jl_value_t *owner = nullptr; + JL_ROOT jl_value_t *owner = nullptr; }; template<> struct llvm::DenseMapInfo { @@ -64,7 +64,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto tid = jl_atomic_load_relaxed(&ct->tid); if (([tid, m, owner, ci] () -> bool { // necessary scope block / lambda for unique_lock jl_unique_gcsafe_lock lock(engine_lock); - arraylist_push(&objects_pinned_by_inference_engine, owner); + arraylist_push(&gc_pinned_objects, owner); InferKey key{m, owner}; if ((signed)Awaiting.size() < tid + 1) Awaiting.resize(tid + 1); @@ -72,7 +72,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto record = Reservations.find(key); if (record == Reservations.end()) { Reservations[key] = ReservationInfo{tid, ci}; - arraylist_pop(&objects_pinned_by_inference_engine); + arraylist_pop(&gc_pinned_objects); return false; } // before waiting, need to run deadlock/cycle detection @@ -81,7 +81,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto wait_tid = record->second.tid; while (1) { if (wait_tid == tid) { - arraylist_pop(&objects_pinned_by_inference_engine); + arraylist_pop(&gc_pinned_objects); return true; } if ((signed)Awaiting.size() <= wait_tid) @@ -99,7 +99,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner lock.wait(engine_wait); Awaiting[tid] = InferKey{}; } - arraylist_pop(&objects_pinned_by_inference_engine); + arraylist_pop(&gc_pinned_objects); })()) ct->ptls->engine_nqueued--; JL_GC_POP(); diff --git a/src/gc-common.c b/src/gc-common.c index f2d782834f212..c901315bd9d3c 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -687,7 +687,7 @@ JL_DLLEXPORT int jl_gc_enable(int on) // MISC // =========================================================================== // -arraylist_t objects_pinned_by_inference_engine; +arraylist_t gc_pinned_objects; JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 2b0d777168e71..dd2885e2cbd9c 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -78,7 +78,7 @@ void jl_gc_init(void) { jl_set_check_alive_type(mmtk_is_reachable_object); - arraylist_new(&objects_pinned_by_inference_engine, 0); + arraylist_new(&gc_pinned_objects, 0); arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); gc_num.interval = default_collect_interval; @@ -260,8 +260,8 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) { - for (size_t i = 0; i < objects_pinned_by_inference_engine.len; i++) { - void *obj = objects_pinned_by_inference_engine.items[i]; + for (size_t i = 0; i < gc_pinned_objects.len; i++) { + void *obj = gc_pinned_objects.items[i]; unsigned char got_pinned = mmtk_pin_object(obj); if (got_pinned) { arraylist_push(objects_pinned_by_call, obj); diff --git a/src/julia.h b/src/julia.h index a0dc85c7542c2..6abe3ea6499c6 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1377,7 +1377,7 @@ JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_RO // object pinning ------------------------------------------------------------ -extern arraylist_t objects_pinned_by_inference_engine; +extern arraylist_t gc_pinned_objects; typedef bool (*check_alive_fn_type)(void *); JL_DLLEXPORT void jl_set_check_alive_type(check_alive_fn_type fn); JL_DLLEXPORT void jl_log_pinning_event(void *pinned_object, const char *filename, int lineno); diff --git a/src/support/analyzer_annotations.h b/src/support/analyzer_annotations.h index 69827e4d77f37..5f8b9b70c1762 100644 --- a/src/support/analyzer_annotations.h +++ b/src/support/analyzer_annotations.h @@ -15,6 +15,7 @@ #define JL_NOTSAFEPOINT_ENTER __attribute__((annotate("julia_notsafepoint_enter"))) #define JL_NOTSAFEPOINT_LEAVE __attribute__((annotate("julia_notsafepoint_leave"))) #define JL_MAYBE_UNROOTED __attribute__((annotate("julia_maybe_unrooted"))) +#define JL_ROOT __attribute__((annotate("julia_rooted"))) #define JL_GLOBALLY_ROOTED __attribute__((annotate("julia_globally_rooted"))) #define JL_ROOTING_ARGUMENT __attribute__((annotate("julia_rooting_argument"))) #define JL_ROOTED_ARGUMENT __attribute__((annotate("julia_rooted_argument"))) @@ -38,6 +39,7 @@ extern "C" { #define JL_NOTSAFEPOINT_ENTER #define JL_NOTSAFEPOINT_LEAVE #define JL_MAYBE_UNROOTED +#define JL_ROOT #define JL_GLOBALLY_ROOTED #define JL_ROOTING_ARGUMENT #define JL_ROOTED_ARGUMENT From 9499a8a7effc7960be8b75acebe50dbaad5eb844 Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Thu, 19 Jun 2025 19:24:39 -0300 Subject: [PATCH 656/662] optimize pinning in ast.c --- src/ast.c | 26 ++++++++++++++++---------- src/gc-mmtk.c | 8 ++++---- src/julia.h | 1 + 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/ast.c b/src/ast.c index d7679f4a6ad51..fac98a3cf6826 100644 --- a/src/ast.c +++ b/src/ast.c @@ -134,7 +134,7 @@ typedef struct _jl_ast_context_t { value_t ssavalue_sym; value_t slot_sym; jl_module_t *module; // context module for `current-julia-module-counter` - struct _jl_ast_context_t *next; // invasive list pointer for getting free contexts + arraylist_t pinned_objects; } jl_ast_context_t; static jl_ast_context_t jl_ast_main_ctx; @@ -277,27 +277,29 @@ static void jl_init_ast_ctx(jl_ast_context_t *ctx) JL_NOTSAFEPOINT ctx->slot_sym = symbol(fl_ctx, "slot"); ctx->module = NULL; set(symbol(fl_ctx, "*scopewarn-opt*"), fixnum(jl_options.warn_scope)); + arraylist_new(&ctx->pinned_objects, 0); } // There should be no GC allocation while holding this lock static uv_mutex_t flisp_lock; -static jl_ast_context_t *jl_ast_ctx_freed = NULL; +int flisp_initialized = 0; +arraylist_t jl_ast_ctx_freed; +arraylist_t jl_ast_ctx_used; static jl_ast_context_t *jl_ast_ctx_enter(jl_module_t *m) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT { JL_SIGATOMIC_BEGIN(); uv_mutex_lock(&flisp_lock); - jl_ast_context_t *ctx = jl_ast_ctx_freed; - if (ctx != NULL) { - jl_ast_ctx_freed = ctx->next; - ctx->next = NULL; - } + jl_ast_context_t *ctx = (jl_ast_context_t*)arraylist_pop(&jl_ast_ctx_freed); uv_mutex_unlock(&flisp_lock); if (ctx == NULL) { // Construct a new one if we can't find any ctx = (jl_ast_context_t*)calloc(1, sizeof(jl_ast_context_t)); jl_init_ast_ctx(ctx); } + uv_mutex_lock(&flisp_lock); + arraylist_push(&jl_ast_ctx_used, ctx); + uv_mutex_unlock(&flisp_lock); ctx->module = m; return ctx; } @@ -306,16 +308,20 @@ static void jl_ast_ctx_leave(jl_ast_context_t *ctx) { uv_mutex_lock(&flisp_lock); ctx->module = NULL; - ctx->next = jl_ast_ctx_freed; - jl_ast_ctx_freed = ctx; + ctx->pinned_objects.len = 0; // clear pinned objects + arraylist_pop(&jl_ast_ctx_used); + arraylist_push(&jl_ast_ctx_freed, ctx); uv_mutex_unlock(&flisp_lock); JL_SIGATOMIC_END(); } void jl_init_flisp(void) { - if (jl_ast_ctx_freed) + if (flisp_initialized) return; + flisp_initialized = 1; + arraylist_new(&jl_ast_ctx_freed, 0); + arraylist_new(&jl_ast_ctx_used, 0); uv_mutex_init(&flisp_lock); jl_init_ast_ctx(&jl_ast_main_ctx); // To match the one in jl_ast_ctx_leave diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index dd2885e2cbd9c..18f689e4d3fa3 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -258,7 +258,7 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { // print_fragmentation(); } -void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) +void gc_pin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) { for (size_t i = 0; i < gc_pinned_objects.len; i++) { void *obj = gc_pinned_objects.items[i]; @@ -269,7 +269,7 @@ void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) } } -void gc_unpin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call) +void gc_unpin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) { for (size_t i = 0; i < objects_pinned_by_call->len; i++) { void *obj = objects_pinned_by_call->items[i]; @@ -341,9 +341,9 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) #ifndef __clang_gcanalyzer__ arraylist_t objects_pinned_by_call; arraylist_new(&objects_pinned_by_call, 0); - gc_pin_objects_from_inference_engine(&objects_pinned_by_call); + gc_pin_objects_from_compiler_frontend(&objects_pinned_by_call); mmtk_block_thread_for_gc(); - gc_unpin_objects_from_inference_engine(&objects_pinned_by_call); + gc_unpin_objects_from_compiler_frontend(&objects_pinned_by_call); arraylist_free(&objects_pinned_by_call); #endif JL_UNLOCK_NOGC(&finalizers_lock); diff --git a/src/julia.h b/src/julia.h index 6abe3ea6499c6..4314519980fdd 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2460,6 +2460,7 @@ JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *trace // AST access JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr JL_MAYBE_UNROOTED); +extern arraylist_t jl_ast_ctx_used; // IR representation JL_DLLEXPORT jl_value_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code); From b8e2c601907072a86c14ceaaf9ef5ebbb75284fe Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Thu, 19 Jun 2025 19:43:15 -0300 Subject: [PATCH 657/662] pass ast context to julia_to_scm --- src/ast.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/ast.c b/src/ast.c index fac98a3cf6826..b1b0a63956782 100644 --- a/src/ast.c +++ b/src/ast.c @@ -151,7 +151,7 @@ struct macroctx_stack { }; static jl_value_t *scm_to_julia(fl_context_t *fl_ctx, value_t e, jl_module_t *mod); -static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v); +static value_t julia_to_scm(jl_ast_context_t *ctx, jl_value_t *v); static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, struct macroctx_stack *macroctx, int onelevel, size_t world, int throw_load_error); static jl_sym_t *scmsym_to_julia(fl_context_t *fl_ctx, value_t s) @@ -676,14 +676,15 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m jl_error("malformed tree"); } -static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid); +static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_valid); -static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v) +static value_t julia_to_scm(jl_ast_context_t *ctx, jl_value_t *v) { value_t temp; // need try/catch to reset GC handle stack in case of error + fl_context_t *fl_ctx = &ctx->fl; FL_TRY_EXTERN(fl_ctx) { - temp = julia_to_scm_(fl_ctx, v, 1); + temp = julia_to_scm_(ctx, v, 1); } FL_CATCH_EXTERN(fl_ctx) { temp = fl_ctx->lasterror; @@ -691,22 +692,24 @@ static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v) return temp; } -static void array_to_list(fl_context_t *fl_ctx, jl_array_t *a, value_t *pv, int check_valid) +static void array_to_list(jl_ast_context_t *ctx, jl_array_t *a, value_t *pv, int check_valid) { value_t temp; + fl_context_t *fl_ctx = &ctx->fl; for (long i = jl_array_nrows(a) - 1; i >= 0; i--) { *pv = fl_cons(fl_ctx, fl_ctx->NIL, *pv); - temp = julia_to_scm_(fl_ctx, jl_array_ptr_ref(a, i), check_valid); + temp = julia_to_scm_(ctx, jl_array_ptr_ref(a, i), check_valid); // note: must be separate statement car_(*pv) = temp; } } -static value_t julia_to_list2(fl_context_t *fl_ctx, jl_value_t *a, jl_value_t *b, int check_valid) +static value_t julia_to_list2(jl_ast_context_t *ctx, jl_value_t *a, jl_value_t *b, int check_valid) { - value_t sa = julia_to_scm_(fl_ctx, a, check_valid); + fl_context_t *fl_ctx = &ctx->fl; + value_t sa = julia_to_scm_(ctx, a, check_valid); fl_gc_handle(fl_ctx, &sa); - value_t sb = julia_to_scm_(fl_ctx, b, check_valid); + value_t sb = julia_to_scm_(ctx, b, check_valid); value_t l = fl_list2(fl_ctx, sa, sb); fl_free_gc_handles(fl_ctx, 1); return l; @@ -780,12 +783,13 @@ static value_t julia_to_list2_noalloc(fl_context_t *fl_ctx, jl_value_t *a, jl_va return l; } -static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid) +static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_valid) { // The following code will take internal pointers to v's fields. We need to make sure // that v will not be moved by GC. OBJ_PIN(v); value_t retval; + fl_context_t *fl_ctx = &ctx->fl; if (julia_to_scm_noalloc1(fl_ctx, v, &retval)) return retval; if (jl_is_expr(v)) { @@ -794,12 +798,12 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali fl_gc_handle(fl_ctx, &args); if (jl_expr_nargs(ex) > 520000 && ex->head != jl_block_sym) lerror(fl_ctx, symbol(fl_ctx, "error"), "expression too large"); - array_to_list(fl_ctx, ex->args, &args, check_valid); - value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)ex->head, check_valid); + array_to_list(ctx, ex->args, &args, check_valid); + value_t hd = julia_to_scm_(ctx, (jl_value_t*)ex->head, check_valid); if (ex->head == jl_lambda_sym && jl_expr_nargs(ex)>0 && jl_is_array(jl_exprarg(ex,0))) { value_t llist = fl_ctx->NIL; fl_gc_handle(fl_ctx, &llist); - array_to_list(fl_ctx, (jl_array_t*)jl_exprarg(ex,0), &llist, check_valid); + array_to_list(ctx, (jl_array_t*)jl_exprarg(ex,0), &llist, check_valid); car_(args) = llist; fl_free_gc_handles(fl_ctx, 1); } @@ -815,7 +819,7 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali jl_value_t *line = jl_fieldref(v,0); value_t args = julia_to_list2_noalloc(fl_ctx, line, file, check_valid); fl_gc_handle(fl_ctx, &args); - value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)jl_line_sym, check_valid); + value_t hd = julia_to_scm_(ctx, (jl_value_t*)jl_line_sym, check_valid); value_t scmv = fl_cons(fl_ctx, hd, args); fl_free_gc_handles(fl_ctx, 1); return scmv; @@ -823,18 +827,18 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali if (jl_typetagis(v, jl_gotonode_type)) return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_goto_sym, jl_fieldref(v,0), check_valid); if (jl_typetagis(v, jl_quotenode_type)) - return julia_to_list2(fl_ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0); + return julia_to_list2(ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0); if (jl_typetagis(v, jl_newvarnode_type)) return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_newvar_sym, jl_fieldref(v,0), check_valid); if (jl_typetagis(v, jl_globalref_type)) { jl_module_t *m = jl_globalref_mod(v); jl_sym_t *sym = jl_globalref_name(v); if (m == jl_core_module) - return julia_to_list2(fl_ctx, (jl_value_t*)jl_core_sym, + return julia_to_list2(ctx, (jl_value_t*)jl_core_sym, (jl_value_t*)sym, check_valid); - value_t args = julia_to_list2(fl_ctx, (jl_value_t*)m, (jl_value_t*)sym, check_valid); + value_t args = julia_to_list2(ctx, (jl_value_t*)m, (jl_value_t*)sym, check_valid); fl_gc_handle(fl_ctx, &args); - value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)jl_globalref_sym, check_valid); + value_t hd = julia_to_scm_(ctx, (jl_value_t*)jl_globalref_sym, check_valid); value_t scmv = fl_cons(fl_ctx, hd, args); fl_free_gc_handles(fl_ctx, 1); return scmv; @@ -902,8 +906,8 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char *text, size_t text_len, static jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule) { jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); + value_t arg = julia_to_scm(ctx, expr); fl_context_t *fl_ctx = &ctx->fl; - value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg); jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule); JL_GC_PUSH1(&result); @@ -916,8 +920,8 @@ jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr, jl_module_t *inmodule, const char *file, int line) { jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); + value_t arg = julia_to_scm(ctx, expr); fl_context_t *fl_ctx = &ctx->fl; - value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, funcname)), arg, symbol(fl_ctx, file), fixnum(line)); jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule); @@ -1294,6 +1298,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, expr = jl_copy_ast(expr); expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1); jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); + value_t arg = julia_to_scm(ctx, expr); fl_context_t *fl_ctx = &ctx->fl; value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, From c75596a259b60804fd132608cc72a15516173d3b Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Thu, 19 Jun 2025 20:10:34 -0300 Subject: [PATCH 658/662] call pin function on AST ctx --- src/ast.c | 11 +++++++++++ src/gc-mmtk.c | 11 +++++++++++ src/julia.h | 1 + 3 files changed, 23 insertions(+) diff --git a/src/ast.c b/src/ast.c index b1b0a63956782..622d65a715d73 100644 --- a/src/ast.c +++ b/src/ast.c @@ -137,6 +137,17 @@ typedef struct _jl_ast_context_t { arraylist_t pinned_objects; } jl_ast_context_t; +// FIXME: Ugly hack to get a pointer to the pinned objects +arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx) +{ + // This is used to extract pinned objects from the context + // for the purpose of pinning them in MMTk. + if (ctx == NULL) + return NULL; + jl_ast_context_t *jl_ctx = (jl_ast_context_t*)ctx; + return &jl_ctx->pinned_objects; +} + static jl_ast_context_t jl_ast_main_ctx; #ifdef __clang_gcanalyzer__ diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 18f689e4d3fa3..952bdfdc6d2e3 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -267,6 +267,17 @@ void gc_pin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) arraylist_push(objects_pinned_by_call, obj); } } + for (size_t i = 0; i < jl_ast_ctx_used.len; i++) { + void *ctx = jl_ast_ctx_used.items[i]; + arraylist_t *pinned_objects = extract_pinned_objects_from_ast_ctx(ctx); + for (size_t j = 0; j < pinned_objects->len; j++) { + void *obj = pinned_objects->items[j]; + unsigned char got_pinned = mmtk_pin_object(obj); + if (got_pinned) { + arraylist_push(objects_pinned_by_call, obj); + } + } + } } void gc_unpin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) diff --git a/src/julia.h b/src/julia.h index 4314519980fdd..57792d4891d70 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2460,6 +2460,7 @@ JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *trace // AST access JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr JL_MAYBE_UNROOTED); +arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx); extern arraylist_t jl_ast_ctx_used; // IR representation From b378651873880baa3246dc229a8e87821a944989 Mon Sep 17 00:00:00 2001 From: Diogo Correia Netto Date: Thu, 19 Jun 2025 20:16:44 -0300 Subject: [PATCH 659/662] push into pinned_objects --- src/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast.c b/src/ast.c index 622d65a715d73..4c1d43b7b9060 100644 --- a/src/ast.c +++ b/src/ast.c @@ -798,7 +798,7 @@ static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_val { // The following code will take internal pointers to v's fields. We need to make sure // that v will not be moved by GC. - OBJ_PIN(v); + arraylist_push(&ctx->pinned_objects, v); value_t retval; fl_context_t *fl_ctx = &ctx->fl; if (julia_to_scm_noalloc1(fl_ctx, v, &retval)) From bf552592ce17e5cb8ce2871981a110c7b9f3e9a5 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 27 Jun 2025 09:19:12 +0800 Subject: [PATCH 660/662] Trace pinned objects as roots (#91) Following the suggestion in https://github.com/mmtk/julia/pull/89#issuecomment-2982261402, trace recently added pinned objects as roots. When we pass those objects as 'nodes' to MMTk, MMTk does not know the slots so MMTk cannot update the reference, thus MMTk will not move those objects. This is essentially the same as pinning those objects before a GC, and unpinning after a GC. --- src/ast.c | 13 ++++++------ src/engine.cpp | 8 ++++---- src/gc-common.c | 2 +- src/gc-mmtk.c | 53 ++++++++++++++++--------------------------------- src/julia.h | 6 ++++-- 5 files changed, 33 insertions(+), 49 deletions(-) diff --git a/src/ast.c b/src/ast.c index 4c1d43b7b9060..5d91c68ed0539 100644 --- a/src/ast.c +++ b/src/ast.c @@ -134,18 +134,19 @@ typedef struct _jl_ast_context_t { value_t ssavalue_sym; value_t slot_sym; jl_module_t *module; // context module for `current-julia-module-counter` - arraylist_t pinned_objects; + // These are essentially roots for ast context. + arraylist_t ast_roots; } jl_ast_context_t; // FIXME: Ugly hack to get a pointer to the pinned objects -arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx) +arraylist_t *extract_ast_roots_from_ast_ctx(void *ctx) { // This is used to extract pinned objects from the context // for the purpose of pinning them in MMTk. if (ctx == NULL) return NULL; jl_ast_context_t *jl_ctx = (jl_ast_context_t*)ctx; - return &jl_ctx->pinned_objects; + return &jl_ctx->ast_roots; } static jl_ast_context_t jl_ast_main_ctx; @@ -288,7 +289,7 @@ static void jl_init_ast_ctx(jl_ast_context_t *ctx) JL_NOTSAFEPOINT ctx->slot_sym = symbol(fl_ctx, "slot"); ctx->module = NULL; set(symbol(fl_ctx, "*scopewarn-opt*"), fixnum(jl_options.warn_scope)); - arraylist_new(&ctx->pinned_objects, 0); + arraylist_new(&ctx->ast_roots, 0); } // There should be no GC allocation while holding this lock @@ -319,7 +320,7 @@ static void jl_ast_ctx_leave(jl_ast_context_t *ctx) { uv_mutex_lock(&flisp_lock); ctx->module = NULL; - ctx->pinned_objects.len = 0; // clear pinned objects + ctx->ast_roots.len = 0; // clear root objects arraylist_pop(&jl_ast_ctx_used); arraylist_push(&jl_ast_ctx_freed, ctx); uv_mutex_unlock(&flisp_lock); @@ -798,7 +799,7 @@ static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_val { // The following code will take internal pointers to v's fields. We need to make sure // that v will not be moved by GC. - arraylist_push(&ctx->pinned_objects, v); + arraylist_push(&ctx->ast_roots, v); value_t retval; fl_context_t *fl_ctx = &ctx->fl; if (julia_to_scm_noalloc1(fl_ctx, v, &retval)) diff --git a/src/engine.cpp b/src/engine.cpp index 324dc9f12347c..a9f2fda8692d5 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -64,7 +64,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto tid = jl_atomic_load_relaxed(&ct->tid); if (([tid, m, owner, ci] () -> bool { // necessary scope block / lambda for unique_lock jl_unique_gcsafe_lock lock(engine_lock); - arraylist_push(&gc_pinned_objects, owner); + arraylist_push(&extra_gc_roots, owner); InferKey key{m, owner}; if ((signed)Awaiting.size() < tid + 1) Awaiting.resize(tid + 1); @@ -72,7 +72,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto record = Reservations.find(key); if (record == Reservations.end()) { Reservations[key] = ReservationInfo{tid, ci}; - arraylist_pop(&gc_pinned_objects); + arraylist_pop(&extra_gc_roots); return false; } // before waiting, need to run deadlock/cycle detection @@ -81,7 +81,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner auto wait_tid = record->second.tid; while (1) { if (wait_tid == tid) { - arraylist_pop(&gc_pinned_objects); + arraylist_pop(&extra_gc_roots); return true; } if ((signed)Awaiting.size() <= wait_tid) @@ -99,7 +99,7 @@ jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner lock.wait(engine_wait); Awaiting[tid] = InferKey{}; } - arraylist_pop(&gc_pinned_objects); + arraylist_pop(&extra_gc_roots); })()) ct->ptls->engine_nqueued--; JL_GC_POP(); diff --git a/src/gc-common.c b/src/gc-common.c index c901315bd9d3c..6f1e85d5db691 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -687,7 +687,7 @@ JL_DLLEXPORT int jl_gc_enable(int on) // MISC // =========================================================================== // -arraylist_t gc_pinned_objects; +arraylist_t extra_gc_roots; JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 952bdfdc6d2e3..e0a3773cb2f86 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -78,7 +78,7 @@ void jl_gc_init(void) { jl_set_check_alive_type(mmtk_is_reachable_object); - arraylist_new(&gc_pinned_objects, 0); + arraylist_new(&extra_gc_roots, 0); arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); gc_num.interval = default_collect_interval; @@ -258,36 +258,6 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) { // print_fragmentation(); } -void gc_pin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) -{ - for (size_t i = 0; i < gc_pinned_objects.len; i++) { - void *obj = gc_pinned_objects.items[i]; - unsigned char got_pinned = mmtk_pin_object(obj); - if (got_pinned) { - arraylist_push(objects_pinned_by_call, obj); - } - } - for (size_t i = 0; i < jl_ast_ctx_used.len; i++) { - void *ctx = jl_ast_ctx_used.items[i]; - arraylist_t *pinned_objects = extract_pinned_objects_from_ast_ctx(ctx); - for (size_t j = 0; j < pinned_objects->len; j++) { - void *obj = pinned_objects->items[j]; - unsigned char got_pinned = mmtk_pin_object(obj); - if (got_pinned) { - arraylist_push(objects_pinned_by_call, obj); - } - } - } -} - -void gc_unpin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call) -{ - for (size_t i = 0; i < objects_pinned_by_call->len; i++) { - void *obj = objects_pinned_by_call->items[i]; - mmtk_unpin_object(obj); - } -} - // Based on jl_gc_collect from gc-stock.c // called when stopping the thread in `mmtk_block_for_gc` JL_DLLEXPORT void jl_gc_prepare_to_collect(void) @@ -350,12 +320,7 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void) jl_gc_notify_thread_yield(ptls, NULL); JL_LOCK_NOGC(&finalizers_lock); // all the other threads are stopped, so this does not make sense, right? otherwise, failing that, this seems like plausibly a deadlock #ifndef __clang_gcanalyzer__ - arraylist_t objects_pinned_by_call; - arraylist_new(&objects_pinned_by_call, 0); - gc_pin_objects_from_compiler_frontend(&objects_pinned_by_call); mmtk_block_thread_for_gc(); - gc_unpin_objects_from_compiler_frontend(&objects_pinned_by_call); - arraylist_free(&objects_pinned_by_call); #endif JL_UNLOCK_NOGC(&finalizers_lock); } @@ -842,6 +807,22 @@ JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure) } } + // Trace objects in extra_gc_roots + for (size_t i = 0; i < extra_gc_roots.len; i++) { + void* obj = extra_gc_roots.items[i]; + add_node_to_roots_buffer(closure, &buf, &len, obj); + } + + // Trace objects in jl_ast_ctx_used + for (size_t i = 0; i < jl_ast_ctx_used.len; i++) { + void *ctx = jl_ast_ctx_used.items[i]; + arraylist_t *ast_roots = extract_ast_roots_from_ast_ctx(ctx); + for (size_t j = 0; j < ast_roots->len; j++) { + void *obj = ast_roots->items[j]; + add_node_to_roots_buffer(closure, &buf, &len, obj); + } + } + // // add module // add_node_to_roots_buffer(closure, &buf, &len, jl_main_module); diff --git a/src/julia.h b/src/julia.h index 57792d4891d70..4fd9fb7823dc7 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1377,7 +1377,9 @@ JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_RO // object pinning ------------------------------------------------------------ -extern arraylist_t gc_pinned_objects; +// These 'new roots' are added for moving GCs. +// We could consider merging this list with global roots list if we can push and pop from global roots list in the same way. +extern arraylist_t extra_gc_roots; typedef bool (*check_alive_fn_type)(void *); JL_DLLEXPORT void jl_set_check_alive_type(check_alive_fn_type fn); JL_DLLEXPORT void jl_log_pinning_event(void *pinned_object, const char *filename, int lineno); @@ -2460,7 +2462,7 @@ JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *trace // AST access JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr JL_MAYBE_UNROOTED); -arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx); +arraylist_t *extract_ast_roots_from_ast_ctx(void *ctx); extern arraylist_t jl_ast_ctx_used; // IR representation From 22661cd6182da55f228f28df32fc38e71bb4fbae Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Aug 2025 02:25:32 +0000 Subject: [PATCH 661/662] Fixing errors after updating --- src/Makefile | 2 +- src/ast.c | 1 - src/cgutils.cpp | 6 +++--- src/codegen.cpp | 38 +++++++++++++++++++------------------- src/gc-mmtk.c | 12 +++++++----- src/jitlayers.cpp | 4 ++-- src/runtime_ccall.cpp | 2 +- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/Makefile b/src/Makefile index de33986abca58..f31802d1b78d9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -69,7 +69,7 @@ CG_LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) # Currently these files are used by both GCs. But we should make the list specific to stock, and MMTk should have its own implementation. -GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier gc-pinning-log +GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier ifeq (${USE_THIRD_PARTY_GC},mmtk) FLAGS += -I$(MMTK_API_INC) GC_CODEGEN_SRCS += llvm-late-gc-lowering-mmtk diff --git a/src/ast.c b/src/ast.c index 5d91c68ed0539..a0458657651bd 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1312,7 +1312,6 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); value_t arg = julia_to_scm(ctx, expr); fl_context_t *fl_ctx = &ctx->fl; - value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, symbol(fl_ctx, filename), fixnum(line)); value_t lwr = car_(e); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 3617341b46694..52316e4652012 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2418,11 +2418,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, // emit this only if we have a possibility of optimizing it if (Order == AtomicOrdering::Unordered) Order = AtomicOrdering::Monotonic; - if (jl_is_pointerfree(rhs.typ) && !rhs.isghost && (rhs.constant || rhs.isboxed || rhs.ispointer())) { + if (jl_is_pointerfree(jl_pinned_ref_get(rhs.typ)) && !rhs.isghost && (rhs.constant || rhs.isboxed || rhs.ispointer())) { // if this value can be loaded from memory, do that now so that it is sequenced before the atomicmodify // and the IR is less dependent on what was emitted before now to create this rhs. // Inlining should do okay to clean this up later if there are parts we don't need. - rhs = jl_cgval_t(emit_unbox(ctx, julia_type_to_llvm(ctx, rhs.typ), rhs, rhs.typ), rhs.typ, NULL); + rhs = jl_cgval_t(emit_unbox(ctx, julia_type_to_llvm(ctx, jl_pinned_ref_get(rhs.typ)), rhs, jl_pinned_ref_get(rhs.typ)), jl_pinned_ref_get(rhs.typ), NULL); } bool gcstack_arg = JL_FEAT_TEST(ctx,gcstack_arg); Function *op = emit_modifyhelper(ctx, cmpop, *modifyop, jltype, elty, rhs, fname, gcstack_arg); @@ -4718,7 +4718,7 @@ static jl_cgval_t emit_memoryref_direct(jl_codectx_t &ctx, const jl_cgval_t &mem } else { data = emit_genericmemoryptr(ctx, boxmem, layout, 0); - idx0 = ctx.builder.CreateMul(idx0, emit_genericmemoryelsize(ctx, boxmem, mem.typ, false), "", true, true); + idx0 = ctx.builder.CreateMul(idx0, emit_genericmemoryelsize(ctx, boxmem, jl_pinned_ref_get(mem.typ), false), "", true, true); data = ctx.builder.CreatePtrAdd(data, idx0); } diff --git a/src/codegen.cpp b/src/codegen.cpp index a2b1dde6807a4..ac3cda587880a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4157,7 +4157,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { const jl_datatype_layout_t *layout = mty_dt->layout; - jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; + jl_value_t *boundscheck = nargs == 3 ? jl_pinned_ref_get(argv[3].constant) : nullptr; if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(mty_dt->parameters), jl_svec_len(mty_dt->parameters)); @@ -5342,19 +5342,19 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ if (f.constant == BUILTIN(modifyfield)) { if (emit_f_opfield(ctx, &ret, BUILTIN(modifyfield), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(f.constant); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } else if (f.constant == BUILTIN(modifyglobal)) { if (emit_f_opglobal(ctx, &ret, BUILTIN(modifyglobal), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(f.constant); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } else if (f.constant == BUILTIN(memoryrefmodify)) { if (emit_f_opmemory(ctx, &ret, BUILTIN(memoryrefmodify), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(f.constant); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } else if (jl_typetagis(jl_pinned_ref_get(f.constant), jl_intrinsic_type)) { @@ -5435,12 +5435,12 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); } - else if (f.constant && jl_isa(f.constant, (jl_value_t*)jl_builtin_type)) { + else if (f.constant && jl_isa(jl_pinned_ref_get(f.constant), (jl_value_t*)jl_builtin_type)) { jl_cgval_t result; - bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); + bool handled = emit_builtin_call(ctx, &result, jl_pinned_ref_get(f.constant), argv, nargs - 1, rt, ex, is_promotable); if (handled) return result; - auto it = builtin_func_map().find(f.constant); + auto it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); if (it != builtin_func_map().end()) { Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); setName(ctx.emission_context, ret, it->second->name + "_ret"); @@ -6811,7 +6811,7 @@ static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, con } assert(AI == w->arg_end()); ctx.f = w; - ctx.rettype = jltype; + ctx.rettype = jl_pinned_ref_create(jl_value_t, jltype); BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); ctx.builder.SetInsertPoint(b0); DebugLoc noDbg; @@ -7102,17 +7102,17 @@ std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_abi_t jl_value_t *abi = get_ci_abi(codeinst); jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, target_is_opaque_closure); if (from_abi.specsig) - emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, &targetspec, nullptr); else - gen_invoke_wrapper(mi, abi, codeinst->rettype, from_abi.rt, targetspec, from_abi.nargs, -1, from_abi.is_opaque_closure, gf_thunk_name, M, params); + gen_invoke_wrapper(mi, abi, codeinst->rettype, jl_pinned_ref_get(from_abi.rt), targetspec, from_abi.nargs, -1, from_abi.is_opaque_closure, gf_thunk_name, M, params); } else { if (from_abi.specsig) - emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, from_abi.rt, codeinst->rettype, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, jl_pinned_ref_get(from_abi.rt), codeinst->rettype, params); } return gf_thunk_name; } @@ -7135,10 +7135,10 @@ std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_abi_t raw_string_ostream(gf_thunk_name) << "j_"; raw_string_ostream(gf_thunk_name) << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1) << "_gfthunk"; if (from_abi.specsig) - emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, - target, from_abi.sigt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, + target, jl_pinned_ref_get(from_abi.sigt), codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, from_abi.rt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, jl_pinned_ref_get(from_abi.rt), codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); return gf_thunk_name; } @@ -7147,11 +7147,11 @@ std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_abi_ std::string gf_thunk_name; raw_string_ostream(gf_thunk_name) << "jconst_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); if (from_abi.specsig) { - emit_specsig_to_specsig(M, gf_thunk_name, from_abi.sigt, from_abi.rt, from_abi.is_opaque_closure, from_abi.nargs, params, - nullptr, from_abi.sigt, jl_typeof(rettype_const), nullptr, rettype_const); + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, + nullptr, jl_pinned_ref_get(from_abi.sigt), jl_typeof(rettype_const), nullptr, rettype_const); } else { - emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, from_abi.rt, jl_typeof(rettype_const), params); + emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, jl_pinned_ref_get(from_abi.rt), jl_typeof(rettype_const), params); } return gf_thunk_name; } @@ -7165,7 +7165,7 @@ std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool sp bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; size_t nargs = specsig ? jl_nparams(sigt) : 0; - jl_abi_t abi = {sigt, rt, nargs, specsig, is_opaque_closure}; + jl_abi_t abi = {jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, rt), nargs, specsig, is_opaque_closure}; return emit_abi_constreturn(M, params, abi, codeinst->rettype_const); } diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index e0a3773cb2f86..cbe80ca490167 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -549,9 +549,8 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(call_cache[i]); } // julia_internal.h - TRACE_GLOBALLY_ROOTED(jl_type_type_mt); - TRACE_GLOBALLY_ROOTED(jl_nonfunction_mt); - TRACE_GLOBALLY_ROOTED(jl_kwcall_mt); + TRACE_GLOBALLY_ROOTED(jl_typeinf_func); + TRACE_GLOBALLY_ROOTED(jl_method_table); TRACE_GLOBALLY_ROOTED(jl_opaque_closure_method); TRACE_GLOBALLY_ROOTED(jl_nulldebuginfo); TRACE_GLOBALLY_ROOTED(_jl_debug_method_invalidation); @@ -711,15 +710,17 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(jl_newvarnode_type); TRACE_GLOBALLY_ROOTED(jl_intrinsic_type); TRACE_GLOBALLY_ROOTED(jl_methtable_type); + TRACE_GLOBALLY_ROOTED(jl_methcache_type); TRACE_GLOBALLY_ROOTED(jl_typemap_level_type); TRACE_GLOBALLY_ROOTED(jl_typemap_entry_type); + TRACE_GLOBALLY_ROOTED(jl_kwcall_type); + TRACE_GLOBALLY_ROOTED(jl_emptysvec); TRACE_GLOBALLY_ROOTED(jl_emptytuple); TRACE_GLOBALLY_ROOTED(jl_true); TRACE_GLOBALLY_ROOTED(jl_false); TRACE_GLOBALLY_ROOTED(jl_nothing); - TRACE_GLOBALLY_ROOTED(jl_kwcall_func); TRACE_GLOBALLY_ROOTED(jl_libdl_dlopen_func); @@ -747,7 +748,8 @@ void trace_partial_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* b // add module TRACE_GLOBALLY_ROOTED(jl_main_module); - // buildin values + // invisible builtin values + TRACE_GLOBALLY_ROOTED(jl_method_table); TRACE_GLOBALLY_ROOTED(jl_an_empty_vec_any); TRACE_GLOBALLY_ROOTED(jl_module_init_order); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 1ade8581be2f0..da26d210ea6f2 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -280,7 +280,7 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, } else if (invoke == jl_fptr_args_addr) { assert(specptr != nullptr); - if (!from_abi.specsig && jl_subtype(codeinst->rettype, from_abi.rt)) + if (!from_abi.specsig && jl_subtype(codeinst->rettype, jl_pinned_ref_get(from_abi.rt))) return specptr; // no adapter required target = specptr; @@ -288,7 +288,7 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, } else if (specsigflags & 0b1) { assert(specptr != nullptr); - if (from_abi.specsig && jl_egal(mi->specTypes, from_abi.sigt) && jl_egal(codeinst->rettype, from_abi.rt)) + if (from_abi.specsig && jl_egal(mi->specTypes, jl_pinned_ref_get(from_abi.sigt)) && jl_egal(codeinst->rettype, jl_pinned_ref_get(from_abi.rt))) return specptr; // no adapter required target = specptr; diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 2efa8bb001903..7d69b8a2f260f 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -425,7 +425,7 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) return f; }; bool is_opaque_closure = false; - jl_abi_t from_abi = { sigt, declrt, nargs, specsig, is_opaque_closure }; + jl_abi_t from_abi = { jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, declrt), nargs, specsig, is_opaque_closure }; if (codeinst == nullptr) { // Generate an adapter to a dynamic dispatch if (cfuncdata->unspecialized == nullptr) From 0c6d0d4a93b86f47e80662446ef1c1f75a62b65b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 5 Aug 2025 22:02:51 +1200 Subject: [PATCH 662/662] Record hidden ptr in layout (#92) * Add fields about hidden pointers in jl_datatype_layout_t * Record hidden pointer in layout * Move the new field to the end of the struct --- base/runtime_internals.jl | 2 + src/datatype.c | 137 ++++++++++++++++++++++++++++++++++---- src/jltypes.c | 18 +++-- src/julia.h | 66 +++++++++++++++++- src/staticdata.c | 2 + 5 files changed, 202 insertions(+), 23 deletions(-) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index ba224acf897d4..6fc4f139f4a89 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -558,7 +558,9 @@ struct DataTypeLayout size::UInt32 nfields::UInt32 npointers::UInt32 + nhidden_pointers::UInt32 firstptr::Int32 + firsthiddenptr::Int32 alignment::UInt16 flags::UInt16 # haspadding : 1; diff --git a/src/datatype.c b/src/datatype.c index 1dbabbfe96688..71673aa693fe0 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -89,6 +89,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; + tn->hiddenptrfields = NULL; tn->max_methods = 0; jl_atomic_store_relaxed(&tn->max_args, 0); jl_atomic_store_relaxed(&tn->cache_entry_count, 0); @@ -137,6 +138,7 @@ static uint32_t _hash_djb2(uint32_t hash, const char *mem, size_t s) JL_NOTSAFEP return hash; } + static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOINT { (void)unused; @@ -149,11 +151,20 @@ static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOI const char *pointers = jl_dt_layout_ptrs(layout); assert(pointers); size_t pointers_size = layout->first_ptr < 0 ? 0 : (layout->npointers << layout->flags.fielddesc_type); + const char *hidden_pointers = NULL; + size_t hidden_ptrs_size = 0; + if (layout->first_hidden_ptr >= 0) { + hidden_pointers = jl_dt_layout_hidden_ptrs(layout); + hidden_ptrs_size = layout->nhidden_pointers << layout->flags.fielddesc_type; + } uint_t hash = 5381; hash = _hash_djb2(hash, (char *)layout, own_size); hash = _hash_djb2(hash, fields, fields_size); hash = _hash_djb2(hash, pointers, pointers_size); + if (hidden_ptrs_size > 0) { + hash = _hash_djb2(hash, hidden_pointers, hidden_ptrs_size); + } return hash; } @@ -174,6 +185,13 @@ static int layout_eq(void *_l1, void *_l2, void *unused) JL_NOTSAFEPOINT size_t pointers_size = l1->first_ptr < 0 ? 0 : (l1->npointers << l1->flags.fielddesc_type); if (memcmp(p1, p2, pointers_size)) return 0; + if (l1->first_hidden_ptr >= 0 && l2->first_hidden_ptr >= 0) { + const char *h1 = jl_dt_layout_hidden_ptrs(l1); + const char *h2 = jl_dt_layout_hidden_ptrs(l2); + size_t hidden_ptrs_size = l1->nhidden_pointers << l1->flags.fielddesc_type; + if (memcmp(h1, h2, hidden_ptrs_size)) + return 0; + } return 1; } @@ -185,15 +203,18 @@ HTIMPL_R(layoutcache, _hash_layout_djb2, layout_eq) static htable_t layoutcache; static int layoutcache_initialized = 0; + static jl_datatype_layout_t *jl_get_layout(uint32_t sz, uint32_t nfields, uint32_t npointers, + uint32_t nhidden_pointers, uint32_t alignment, int haspadding, int isbitsegal, int arrayelem, jl_fielddesc32_t desc[], - uint32_t pointers[]) JL_NOTSAFEPOINT + uint32_t pointers[], + uint32_t hidden_pointers[]) JL_NOTSAFEPOINT { assert(alignment); // should have been verified by caller @@ -223,11 +244,13 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, } } int32_t first_ptr = (npointers > 0 ? (int32_t)pointers[0] : -1); + int32_t first_hidden_ptr = (nhidden_pointers > 0 ? (int32_t)hidden_pointers[0] : -1); // allocate a new descriptor, on the stack if possible. size_t fields_size = nfields * jl_fielddesc_size(fielddesc_type); size_t pointers_size = first_ptr < 0 ? 0 : (npointers << fielddesc_type); - size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size; + size_t hidden_ptrs_size = first_hidden_ptr < 0 ? 0 : (nhidden_pointers << fielddesc_type); + size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size + hidden_ptrs_size; int should_malloc = flddesc_sz >= jl_page_size; jl_datatype_layout_t *mallocmem = (jl_datatype_layout_t *)(should_malloc ? malloc(flddesc_sz) : NULL); jl_datatype_layout_t *allocamem = (jl_datatype_layout_t *)(should_malloc ? NULL : alloca(flddesc_sz)); @@ -246,6 +269,8 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, flddesc->flags.padding = 0; flddesc->npointers = npointers; flddesc->first_ptr = first_ptr; + flddesc->nhidden_pointers = nhidden_pointers; + flddesc->first_hidden_ptr = first_hidden_ptr; // fill out the fields of the new descriptor jl_fielddesc8_t *desc8 = (jl_fielddesc8_t *)jl_dt_layout_fields(flddesc); @@ -285,6 +310,25 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, } } + // fill out hidden pointer descriptors + if (first_hidden_ptr >= 0 && hidden_pointers) { + uint8_t *hptrs8 = (uint8_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint16_t *hptrs16 = (uint16_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint32_t *hptrs32 = (uint32_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint32_t *src_descs = (uint32_t *)hidden_pointers; + for (size_t i = 0; i < nhidden_pointers; i++) { + if (fielddesc_type == 0) { + hptrs8[i] = src_descs[i]; + } + else if (fielddesc_type == 1) { + hptrs16[i] = src_descs[i]; + } + else { + hptrs32[i] = src_descs[i]; + } + } + } + if (__unlikely(!layoutcache_initialized)) { htable_new(&layoutcache, 4096); layoutcache_initialized = 1; @@ -312,6 +356,7 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, return ret; } + // Determine if homogeneous tuple with fields of type t will have // a special alignment and vector-ABI beyond normal rules for aggregates. // Return special alignment if one exists, 0 if normal alignment rules hold. @@ -521,7 +566,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) } if (!jl_is_type(eltype)) { // this is expected to have a layout, but since it is not constructable, we don't care too much what it is - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), {0}}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), {0}}; st->layout = &opaque_ptr_layout; st->has_concrete_subtype = 0; return; @@ -607,7 +652,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) arrayelem |= 8; // arrayelem_islocked } assert(!st->layout); - st->layout = jl_get_layout(elsz, nfields, npointers, al, haspadding, isbitsegal, arrayelem, NULL, pointers); + st->layout = jl_get_layout(elsz, nfields, npointers, 0, al, haspadding, isbitsegal, arrayelem, NULL, pointers, NULL); st->zeroinit = zi; //st->has_concrete_subtype = 1; //st->isbitstype = 0; @@ -666,17 +711,17 @@ void jl_compute_field_offsets(jl_datatype_t *st) // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob - static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, -1, 1, { .isbitsegal=1 }}; + static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, 0, -1, -1, 1, { .isbitsegal=1 }}; st->layout = &opaque_byte_layout; return; } else if (st == jl_simplevector_type || st == jl_module_type) { - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), { .isbitsegal=1 }}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), { .isbitsegal=1 }}; st->layout = &opaque_ptr_layout; return; } else { - static const jl_datatype_layout_t singleton_layout = {0, 0, 0, -1, 1, { .isbitsegal=1 }}; + static const jl_datatype_layout_t singleton_layout = {0, 0, 0, 0, -1, -1, 1, { .isbitsegal=1 }}; st->layout = &singleton_layout; } } @@ -708,6 +753,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) size_t descsz = nfields * sizeof(jl_fielddesc32_t); jl_fielddesc32_t *desc; uint32_t *pointers; + uint32_t *hidden_pointers; int should_malloc = descsz >= jl_page_size; if (should_malloc) desc = (jl_fielddesc32_t*)malloc_s(descsz); @@ -721,12 +767,22 @@ void jl_compute_field_offsets(jl_datatype_t *st) int homogeneous = 1; int needlock = 0; uint32_t npointers = 0; + uint32_t nhidden_pointers = 0; jl_value_t *firstty = jl_field_type(st, 0); for (i = 0; i < nfields; i++) { jl_value_t *fld = jl_field_type(st, i); int isatomic = jl_field_isatomic(st, i); + int ishiddenptr = jl_field_ishiddenptr(st, i); size_t fsz = 0, al = 1; - if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc + if (ishiddenptr) { + fsz = sizeof(void*); + al = fsz; + if (al > MAX_ALIGN) + al = MAX_ALIGN; + desc[i].isptr = 0; // Hidden pointers are stored as non-pointer fields + nhidden_pointers++; + } + else if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc if (__unlikely(fsz > max_size)) // Should never happen throw_ovf(should_malloc, desc, st, fsz); @@ -766,6 +822,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (!zeroinit) zeroinit = ((jl_datatype_t*)fld)->zeroinit; npointers += fld_npointers; + nhidden_pointers += ((jl_datatype_t*)fld)->layout->nhidden_pointers; } } else { @@ -824,25 +881,61 @@ void jl_compute_field_offsets(jl_datatype_t *st) pointers = (uint32_t*)malloc_s(npointers * sizeof(uint32_t)); else pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); + if (should_malloc && nhidden_pointers) { + hidden_pointers = (uint32_t*)malloc_s(nhidden_pointers * sizeof(uint32_t)); + } else { + hidden_pointers = (uint32_t*)alloca(nhidden_pointers * sizeof(uint32_t)); + } size_t ptr_i = 0; + size_t hptr_i = 0; for (i = 0; i < nfields; i++) { jl_value_t *fld = jl_field_type(st, i); uint32_t offset = desc[i].offset / sizeof(jl_value_t**); - if (desc[i].isptr) + int ishiddenptr = jl_field_ishiddenptr(st, i); + if (ishiddenptr) { + // Direct hidden pointer field + hidden_pointers[hptr_i] = offset; + hptr_i++; + } + else if (desc[i].isptr) pointers[ptr_i++] = offset; else if (jl_is_datatype(fld)) { + // Handle nested datatype with regular/hidden pointers int j, npointers = ((jl_datatype_t*)fld)->layout->npointers; for (j = 0; j < npointers; j++) { pointers[ptr_i++] = offset + jl_ptr_offset((jl_datatype_t*)fld, j); } + // Copy hidden pointers from nested field + int nhidden = ((jl_datatype_t*)fld)->layout->nhidden_pointers; + for (j = 0; j < nhidden; j++) { + hidden_pointers[hptr_i] = offset + jl_hidden_ptr_offset((jl_datatype_t*)fld, j); + hptr_i++; + } } } assert(ptr_i == npointers); - st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, isbitsegal, 0, desc, pointers); + assert(hptr_i == nhidden_pointers); + st->layout = jl_get_layout(sz, nfields, npointers, nhidden_pointers, alignm, haspadding, isbitsegal, 0, desc, pointers, hidden_pointers); + + // Validation: Ensure no overlap between pointer and hidden pointer offsets + if (npointers > 0 && nhidden_pointers > 0) { + for (size_t p = 0; p < npointers; p++) { + for (size_t hp = 0; hp < nhidden_pointers; hp++) { + if (pointers[p] == hidden_pointers[hp]) { + jl_errorf("Field offset conflict: field at offset %u appears in both regular pointer offsets and hidden pointer offsets", + pointers[p]); + } + } + } + } + + // All hidden pointers are assumed to be PtrOrOffset type if (should_malloc) { free(desc); if (npointers) free(pointers); + if (nhidden_pointers) + free(hidden_pointers); } st->zeroinit = zeroinit; } @@ -855,7 +948,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) return; } -JL_DLLEXPORT jl_datatype_t *jl_new_datatype( +JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs( jl_sym_t *name, jl_module_t *module, jl_datatype_t *super, @@ -864,7 +957,8 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( jl_svec_t *ftypes, jl_svec_t *fattrs, int abstract, int mutabl, - int ninitialized) + int ninitialized, + uint32_t* hiddenptrfields) { jl_datatype_t *t = NULL; jl_typename_t *tn = NULL; @@ -942,6 +1036,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( } tn->atomicfields = atomicfields; tn->constfields = constfields; + tn->hiddenptrfields = hiddenptrfields; if (t->name->wrapper == NULL) { t->name->wrapper = (jl_value_t*)t; @@ -963,6 +1058,22 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( return t; } +JL_DLLEXPORT jl_datatype_t *jl_new_datatype( + jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_svec_t *parameters, + jl_svec_t *fnames, + jl_svec_t *ftypes, + jl_svec_t *fattrs, + int abstract, int mutabl, + int ninitialized) +{ + return jl_new_datatype_with_hiddenptrs( + name, module, super, parameters, fnames, ftypes, fattrs, + abstract, mutabl, ninitialized, NULL); +} + JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *module, jl_datatype_t *super, jl_svec_t *parameters, size_t nbits) @@ -992,7 +1103,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * bt->ismutationfree = 1; bt->isidentityfree = 1; bt->isbitstype = (parameters == jl_emptysvec); - bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, 1, 0, NULL, NULL); + bt->layout = jl_get_layout(nbytes, 0, 0, 0, alignm, 0, 1, 0, NULL, NULL, NULL); bt->instance = NULL; return bt; } diff --git a/src/jltypes.c b/src/jltypes.c index d33c4050c8e20..c19fd5cc2ebc9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3004,19 +3004,20 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name->wrapper = (jl_value_t*)jl_typename_type; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 18 - 2; - jl_typename_type->name->names = jl_perm_symsvec(18, "name", "module", "singletonname", + jl_typename_type->name->n_uninitialized = 19 - 2; + jl_typename_type->name->names = jl_perm_symsvec(19, "name", "module", "singletonname", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", "partial", "hash", "max_args", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", - "cache_entry_count", "max_methods", "constprop_heuristic"); + "cache_entry_count", "max_methods", "constprop_heuristic", + "hiddenptrfields"); const static uint32_t typename_constfields[1] = { 0b000110100001001011 }; // TODO: put back atomicfields and constfields in this list const static uint32_t typename_atomicfields[1] = { 0b001001001110000000 }; jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(18, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, + jl_typename_type->types = jl_svec(19, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, jl_type_type, jl_simplevector_type, jl_simplevector_type, @@ -3027,7 +3028,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, - jl_any_type /*jl_uint8_type*/); + jl_any_type /*jl_uint8_type*/, + jl_any_type/*jl_voidpointer_type*/); jl_methcache_type->name = jl_new_typename_in(jl_symbol("MethodCache"), core, 0, 1); jl_methcache_type->name->wrapper = (jl_value_t*)jl_methcache_type; @@ -3336,11 +3338,12 @@ void jl_init_types(void) JL_GC_DISABLED memory_datatype->ismutationfree = 0; jl_datatype_t *jl_memoryref_supertype = (jl_datatype_t*)jl_apply_type1((jl_value_t*)jl_ref_type, jl_svecref(tv, 1)); + const static uint32_t memoryref_hiddenptrfields[1] = { 0x00000001 }; // (1<<0) - field 0 is a hidden pointer jl_datatype_t *memoryref_datatype = - jl_new_datatype(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv, + jl_new_datatype_with_hiddenptrs(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv, jl_perm_symsvec(2, "ptr_or_offset", "mem"), jl_svec(2, pointer_void, memory_datatype), - jl_emptysvec, 0, 0, 2); + jl_emptysvec, 0, 0, 2, memoryref_hiddenptrfields); jl_genericmemoryref_typename = memoryref_datatype->name; jl_genericmemoryref_type = (jl_unionall_t*)jl_genericmemoryref_typename->wrapper; memoryref_datatype->ismutationfree = 0; @@ -3853,6 +3856,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_typename_type->types, 15, jl_uint8_type); jl_svecset(jl_typename_type->types, 16, jl_uint8_type); jl_svecset(jl_typename_type->types, 17, jl_uint8_type); + jl_svecset(jl_typename_type->types, 18, jl_voidpointer_type); // hiddenptrfields jl_svecset(jl_methcache_type->types, 2, jl_long_type); // voidpointer jl_svecset(jl_methcache_type->types, 3, jl_long_type); // uint32_t plus alignment jl_svecset(jl_methtable_type->types, 3, jl_module_type); diff --git a/src/julia.h b/src/julia.h index 4fd9fb7823dc7..abe9b08c677d9 100644 --- a/src/julia.h +++ b/src/julia.h @@ -188,9 +188,12 @@ JL_EXTENSION typedef struct _jl_genericmemory_t { #endif } jl_genericmemory_t; +// Special type annotation for fields containing hidden pointers +typedef void * jl_hidden_ptr_ptr_or_offset_t; + JL_EXTENSION typedef struct { JL_DATA_TYPE - void *ptr_or_offset; + jl_hidden_ptr_ptr_or_offset_t ptr_or_offset; jl_genericmemory_t *mem; } jl_genericmemoryref_t; @@ -563,6 +566,7 @@ typedef struct { _Atomic(uint8_t) cache_entry_count; // (approximate counter of TypeMapEntry for heuristics) uint8_t max_methods; // override for inference's max_methods setting (0 = no additional limit or relaxation) uint8_t constprop_heustic; // override for inference's constprop heuristic + const uint32_t *hiddenptrfields; // if any fields are hidden pointers, we record them here } jl_typename_t; typedef struct { @@ -593,8 +597,10 @@ typedef struct { typedef struct { uint32_t size; uint32_t nfields; - uint32_t npointers; // number of pointers embedded inside - int32_t first_ptr; // index of the first pointer (or -1) + uint32_t npointers; // number of direct object pointers + uint32_t nhidden_pointers; // number of hidden/internal pointers + int32_t first_ptr; // index of the first direct pointer (or -1) + int32_t first_hidden_ptr; // index of the first hidden pointer (or -1) uint16_t alignment; // strictest alignment over all fields struct { // combine these fields into a struct so that we can take addressof them uint16_t haspadding : 1; // has internal undefined bytes @@ -619,6 +625,11 @@ typedef struct { // uint16_t ptr16[npointers]; // uint32_t ptr32[npointers]; // }; + // union { // offsets relative to data start in words + // uint8_t ptr8[nhidden_pointers]; + // uint16_t ptr16[nhidden_pointers]; + // uint32_t ptr32[nhidden_pointers]; + // }; } jl_datatype_layout_t; typedef struct _jl_datatype_t { @@ -1743,6 +1754,34 @@ static inline uint32_t jl_ptr_offset(jl_datatype_t *st, int i) JL_NOTSAFEPOINT } } +// Access functions for hidden pointer descriptors +static inline const char *jl_dt_layout_hidden_ptrs(const jl_datatype_layout_t *l) JL_NOTSAFEPOINT +{ + const char *ptrs = jl_dt_layout_ptrs(l); + size_t direct_ptrs_size = l->first_ptr < 0 ? 0 : (l->npointers << l->flags.fielddesc_type); + return ptrs + direct_ptrs_size; +} + +static inline uint32_t jl_hidden_ptr_offset(jl_datatype_t *st, int i) JL_NOTSAFEPOINT +{ + const jl_datatype_layout_t *ly = st->layout; + assert(i >= 0 && (size_t)i < ly->nhidden_pointers); + const void *hidden_ptrs = jl_dt_layout_hidden_ptrs(ly); + + if (ly->flags.fielddesc_type == 0) { + return ((const uint8_t*)hidden_ptrs)[i]; + } + else if (ly->flags.fielddesc_type == 1) { + return ((const uint16_t*)hidden_ptrs)[i]; + } + else { + assert(ly->flags.fielddesc_type == 2); + return ((const uint32_t*)hidden_ptrs)[i]; + } +} + + + static inline int jl_field_isatomic(jl_datatype_t *st, int i) JL_NOTSAFEPOINT { const uint32_t *atomicfields = st->name->atomicfields; @@ -1766,6 +1805,17 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT return 0; } +static inline int jl_field_ishiddenptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT +{ + const uint32_t *hiddenptrfields = st->name->hiddenptrfields; + if (hiddenptrfields != NULL) { + if (hiddenptrfields[i / 32] & (1 << (i % 32))) + return 1; + } + return 0; +} + + // basic predicates ----------------------------------------------------------- #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) @@ -2071,6 +2121,16 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_svec_t *fattrs, int abstract, int mutabl, int ninitialized); +JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs(jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_svec_t *parameters, + jl_svec_t *fnames, + jl_svec_t *ftypes, + jl_svec_t *fattrs, + int abstract, int mutabl, + int ninitialized, + uint32_t* hiddenptrfields); JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *module, jl_datatype_t *super, diff --git a/src/staticdata.c b/src/staticdata.c index 61da1d3460962..eb4c17504ff57 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1983,6 +1983,8 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize; if (!is_foreign_type && dt->layout->first_ptr != -1) fldsize += np << dt->layout->flags.fielddesc_type; + if (!is_foreign_type && dt->layout->first_hidden_ptr != -1) + fldsize += dt->layout->nhidden_pointers << dt->layout->flags.fielddesc_type; uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*)); write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream newdt->layout = NULL; // relocation offset