DO NOT MERGE: Comparison of v1.10.2+RAI to v1.10.2

.github/labeler.yml

@@ -0,0 +1,3 @@
+# See https://github.com/actions/labeler
+port-to-master: '**'
+port-to-v1.10: '**'

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+<!---
+PRs to RelationalAI/julia must be opened to the correct branch (see
+https://github.com/RelationalAI/raicode/blob/master/nix/julia-version.json).
+-->
+## PR Description
+
+_What does this PR do?_
+
+## Checklist
+
+Requirements for merging:
+- [ ] I have opened an issue or PR upstream on JuliaLang/julia: <link to JuliaLang/julia>
+- [ ] I have removed the `port-to-*` labels that don't apply.
+- [ ] I have opened a PR on raicode to test these changes: <link to raicode>

.github/workflows/labeler.yml

@@ -0,0 +1,17 @@
+# See https://github.com/actions/labeler
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+    types:
+      - opened
+
+jobs:
+  triage:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v4
+      with:
+        dot: true

.github/workflows/stale.yml

@@ -0,0 +1,16 @@
+name: "Close stale PRs"
+on:
+  schedule:
+  - cron: "0 0 * * *" # every night at midnight
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v8
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Comment or remove stale label, or this PR will be closed in 5 days.'
+        days-before-stale: 30
+        days-before-close: 5
+        stale-pr-label: 'stale'

.github/workflows/update-upstream-branches.yml

@@ -0,0 +1,28 @@
+name: "Update upstream branches"
+on:
+  schedule:
+    - cron: "0 0 * * *" # every night at midnight
+  workflow_dispatch:
+
+jobs:
+  PullUpstream:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false  # run all jobs in the matrix even if one fails
+      matrix:
+        branch:
+          - "master"
+          - "backports-release-1.10"
+    steps:
+      - name: Checkout RAI/julia
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+      - name: Update ${{ matrix.branch }}
+        run: |
+          git config --global user.email "julia-engineering@relational.ai"
+          git config --global user.name "RAI CI (GitHub Action Automation)"
+
+          git remote add upstream https://github.com/JuliaLang/julia
+          git pull upstream ${{ matrix.branch }}
+          git push origin ${{ matrix.branch }}

NEWS.md

@@ -133,6 +133,8 @@ Deprecated or removed
 
 * The `@pure` macro is now deprecated. Use `Base.@assume_effects :foldable` instead ([#48682]).
 
+- A wall-time profiler is now available for users who need a sampling profiler that captures tasks regardless of their scheduling or running state. This type of profiler enables profiling of I/O-heavy tasks and helps detect areas of heavy contention in the system ([#55889]).
+
 <!--- generated by NEWS-update.jl: -->
 [#31836]: https://github.com/JuliaLang/julia/issues/31836
 [#40105]: https://github.com/JuliaLang/julia/issues/40105

VERSION

@@ -1 +1 @@
-1.10.2
+1.10.2+RAI

base/Base.jl

@@ -595,6 +595,7 @@ function __init__()
     init_load_path()
     init_active_project()
     append!(empty!(_sysimage_modules), keys(loaded_modules))
+    empty!(explicit_loaded_modules)
     if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
         MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
     end

base/boot.jl

@@ -156,15 +156,33 @@
 #end
 
 #mutable struct Task
-#    parent::Task
+#    next::Any
+#    queue::Any
 #    storage::Any
-#    state::Symbol
 #    donenotify::Any
 #    result::Any
-#    exception::Any
-#    backtrace::Any
-#    logstate::Any
+#    scope::Any
 #    code::Any
+#    @atomic _state::UInt8
+#    sticky::UInt8
+#    priority::UInt16
+#    @atomic _isexception::UInt8
+#    pad00::UInt8
+#    pad01::UInt8
+#    pad02::UInt8
+#    rngState0::UInt64
+#    rngState1::UInt64
+#    rngState2::UInt64
+#    rngState3::UInt64
+#    rngState4::UInt64
+#    const metrics_enabled::Bool
+#    pad10::UInt8
+#    pad11::UInt8
+#    pad12::UInt8
+#    @atomic first_enqueued_at::UInt64
+#    @atomic last_started_running_at::UInt64
+#    @atomic running_time_ns::UInt64
+#    @atomic finished_at::UInt64
 #end
 
 export

base/checked.jl

@@ -13,7 +13,7 @@ return both the unchecked results and a boolean value denoting the presence of a
 module Checked
 
 export checked_neg, checked_abs, checked_add, checked_sub, checked_mul,
-       checked_div, checked_rem, checked_fld, checked_mod, checked_cld,
+       checked_div, checked_rem, checked_fld, checked_mod, checked_cld, checked_pow,
        checked_length, add_with_overflow, sub_with_overflow, mul_with_overflow
 
 import Core.Intrinsics:
@@ -358,6 +358,19 @@ The overflow protection may impose a perceptible performance penalty.
 """
 checked_cld(x::T, y::T) where {T<:Integer} = cld(x, y) # Base.cld already checks
 
+"""
+    Base.checked_pow(x, y)
+
+Calculates `^(x,y)`, checking for overflow errors where applicable.
+
+The overflow protection may impose a perceptible performance penalty.
+"""
+checked_pow(x::Integer, y::Integer) = checked_power_by_squaring(x, y)
+
+checked_power_by_squaring(x_, p::Integer) = Base.power_by_squaring(x_, p; mul = checked_mul)
+# For Booleans, the default implementation covers all cases.
+checked_power_by_squaring(x::Bool, p::Integer) = Base.power_by_squaring(x, p)
+
 """
     Base.checked_length(r)
 

base/compiler/abstractinterpretation.jl

@@ -243,6 +243,7 @@ function find_matching_methods(𝕃::AbstractLattice,
         for i in 1:length(split_argtypes)
             arg_n = split_argtypes[i]::Vector{Any}
             sig_n = argtypes_to_type(arg_n)
+            sig_n === Bottom && continue
             mt = ccall(:jl_method_table_for, Any, (Any,), sig_n)
             mt === nothing && return FailedMethodMatch("Could not identify method table for call")
             mt = mt::MethodTable
@@ -506,7 +507,10 @@ function abstract_call_method(interp::AbstractInterpreter,
         return MethodCallResult(Any, false, false, nothing, Effects())
     end
     sigtuple = unwrap_unionall(sig)
-    sigtuple isa DataType || return MethodCallResult(Any, false, false, nothing, Effects())
+    sigtuple isa DataType ||
+        return MethodCallResult(Any, false, false, nothing, Effects())
+    all(@nospecialize(x) -> isvarargtype(x) || valid_as_lattice(x, true), sigtuple.parameters) ||
+        return MethodCallResult(Union{}, false, false, nothing, EFFECTS_THROWS) # catch bad type intersections early
 
     if is_nospecializeinfer(method)
         sig = get_nospecializeinfer_sig(method, sig, sparams)
@@ -1385,25 +1389,35 @@ function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft)
     end
     if isa(tti, Union)
         utis = uniontypes(tti)
-        if any(@nospecialize(t) -> !isa(t, DataType) || !(t <: Tuple) || !isknownlength(t), utis)
-            return AbstractIterationResult(Any[Vararg{Any}], nothing, Effects())
-        end
-        ltp = length((utis[1]::DataType).parameters)
-        for t in utis
-            if length((t::DataType).parameters) != ltp
-                return AbstractIterationResult(Any[Vararg{Any}], nothing)
+        # refine the Union to remove elements that are not valid tags for objects
+        filter!(@nospecialize(x) -> valid_as_lattice(x, true), utis)
+        if length(utis) == 0
+            return AbstractIterationResult(Any[], nothing) # oops, this statement was actually unreachable
+        elseif length(utis) == 1
+            tti = utis[1]
+            tti0 = rewrap_unionall(tti, tti0)
+        else
+            if any(@nospecialize(t) -> !isa(t, DataType) || !(t <: Tuple) || !isknownlength(t), utis)
+                return AbstractIterationResult(Any[Vararg{Any}], nothing, Effects())
             end
-        end
-        result = Any[ Union{} for _ in 1:ltp ]
-        for t in utis
-            tps = (t::DataType).parameters
-            _all(valid_as_lattice, tps) || continue
-            for j in 1:ltp
-                result[j] = tmerge(result[j], rewrap_unionall(tps[j], tti0))
+            ltp = length((utis[1]::DataType).parameters)
+            for t in utis
+                if length((t::DataType).parameters) != ltp
+                    return AbstractIterationResult(Any[Vararg{Any}], nothing)
+                end
+            end
+            result = Any[ Union{} for _ in 1:ltp ]
+            for t in utis
+                tps = (t::DataType).parameters
+                for j in 1:ltp
+                    @assert valid_as_lattice(tps[j], true)
+                    result[j] = tmerge(result[j], rewrap_unionall(tps[j], tti0))
+                end
             end
+            return AbstractIterationResult(result, nothing)
         end
-        return AbstractIterationResult(result, nothing)
-    elseif tti0 <: Tuple
+    end
+    if tti0 <: Tuple
         if isa(tti0, DataType)
             return AbstractIterationResult(Any[ p for p in tti0.parameters ], nothing)
         elseif !isa(tti, DataType)
@@ -1667,7 +1681,7 @@ end
     return isa_condition(xt, ty, max_union_splitting)
 end
 @inline function isa_condition(@nospecialize(xt), @nospecialize(ty), max_union_splitting::Int)
-    tty_ub, isexact_tty = instanceof_tfunc(ty)
+    tty_ub, isexact_tty = instanceof_tfunc(ty, true)
     tty = widenconst(xt)
     if isexact_tty && !isa(tty_ub, TypeVar)
         tty_lb = tty_ub # TODO: this would be wrong if !isexact_tty, but instanceof_tfunc doesn't preserve this info
@@ -1677,7 +1691,7 @@ end
                 # `typeintersect` may be unable narrow down `Type`-type
                 thentype = tty_ub
             end
-            valid_as_lattice(thentype) || (thentype = Bottom)
+            valid_as_lattice(thentype, true) || (thentype = Bottom)
             elsetype = typesubtract(tty, tty_lb, max_union_splitting)
             return ConditionalTypes(thentype, elsetype)
         end
@@ -1923,7 +1937,7 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn
     ft′ = argtype_by_index(argtypes, 2)
     ft = widenconst(ft′)
     ft === Bottom && return CallMeta(Bottom, EFFECTS_THROWS, NoCallInfo())
-    (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, 3))
+    (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, 3), false)
     isexact || return CallMeta(Any, Effects(), NoCallInfo())
     unwrapped = unwrap_unionall(types)
     if types === Bottom || !(unwrapped isa DataType) || unwrapped.name !== Tuple.name
@@ -2153,6 +2167,7 @@ function abstract_call_unknown(interp::AbstractInterpreter, @nospecialize(ft),
     end
     # non-constant function, but the number of arguments is known and the `f` is not a builtin or intrinsic
     atype = argtypes_to_type(arginfo.argtypes)
+    atype === Bottom && return CallMeta(Union{}, Union{}, EFFECTS_THROWS, NoCallInfo()) # accidentally unreachable
     return abstract_call_gf_by_type(interp, nothing, arginfo, si, atype, sv, max_methods)
 end
 
@@ -2380,7 +2395,7 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, vtyp
         (; rt, effects) = abstract_eval_call(interp, e, vtypes, sv)
         t = rt
     elseif ehead === :new
-        t, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv))
+        t, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv), true)
         ut = unwrap_unionall(t)
         consistent = ALWAYS_FALSE
         nothrow = false
@@ -2444,7 +2459,7 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, vtyp
         end
         effects = Effects(EFFECTS_TOTAL; consistent, nothrow)
     elseif ehead === :splatnew
-        t, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv))
+        t, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv), true)
         nothrow = false # TODO: More precision
         if length(e.args) == 2 && isconcretedispatch(t) && !ismutabletype(t)
             at = abstract_eval_value(interp, e.args[2], vtypes, sv)

base/compiler/abstractlattice.jl

@@ -98,16 +98,18 @@ is_valid_lattice_norec(::InferenceLattice, @nospecialize(elem)) = isa(elem, Limi
 """
     tmeet(𝕃::AbstractLattice, a, b::Type)
 
-Compute the lattice meet of lattice elements `a` and `b` over the lattice `𝕃`.
-If `𝕃` is `JLTypeLattice`, this is equivalent to type intersection.
+Compute the lattice meet of lattice elements `a` and `b` over the lattice `𝕃`,
+dropping any results that will not be inhabited at runtime.
+If `𝕃` is `JLTypeLattice`, this is equivalent to type intersection plus the
+elimination of results that have no concrete subtypes.
 Note that currently `b` is restricted to being a type
 (interpreted as a lattice element in the `JLTypeLattice` sub-lattice of `𝕃`).
 """
 function tmeet end
 
 function tmeet(::JLTypeLattice, @nospecialize(a::Type), @nospecialize(b::Type))
     ti = typeintersect(a, b)
-    valid_as_lattice(ti) || return Bottom
+    valid_as_lattice(ti, true) || return Bottom
     return ti
 end
 

base/compiler/optimize.jl

@@ -305,7 +305,7 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe
         elseif head === :new_opaque_closure
             length(args) < 4 && return (false, false, false)
             typ = argextype(args[1], src)
-            typ, isexact = instanceof_tfunc(typ)
+            typ, isexact = instanceof_tfunc(typ, true)
             isexact || return (false, false, false)
             ⊑(𝕃ₒ, typ, Tuple) || return (false, false, false)
             rt_lb = argextype(args[2], src)
@@ -561,7 +561,7 @@ function convert_to_ircode(ci::CodeInfo, sv::OptimizationState)
             idx += 1
             prevloc = codeloc
         end
-        if code[idx] isa Expr && ssavaluetypes[idx] === Union{}
+        if ssavaluetypes[idx] === Union{} && code[idx] !== nothing && !(code[idx] isa Core.Const)
             if !(idx < length(code) && isa(code[idx + 1], ReturnNode) && !isdefined((code[idx + 1]::ReturnNode), :val))
                 # insert unreachable in the same basic block after the current instruction (splitting it)
                 insert!(code, idx + 1, ReturnNode())

base/compiler/ssair/inlining.jl

@@ -1202,7 +1202,7 @@ function handle_invoke_call!(todo::Vector{Pair{Int,Any}},
 end
 
 function invoke_signature(argtypes::Vector{Any})
-    ft, argtyps = widenconst(argtypes[2]), instanceof_tfunc(widenconst(argtypes[3]))[1]
+    ft, argtyps = widenconst(argtypes[2]), instanceof_tfunc(widenconst(argtypes[3]), false)[1]
     return rewrap_unionall(Tuple{ft, unwrap_unionall(argtyps).parameters...}, argtyps)
 end
 
@@ -1450,8 +1450,9 @@ function handle_call!(todo::Vector{Pair{Int,Any}},
     cases = compute_inlining_cases(info, flag, sig, state)
     cases === nothing && return nothing
     cases, all_covered, joint_effects = cases
-    handle_cases!(todo, ir, idx, stmt, argtypes_to_type(sig.argtypes), cases,
-        all_covered, joint_effects)
+    atype = argtypes_to_type(sig.argtypes)
+    atype === Union{} && return nothing # accidentally actually unreachable
+    handle_cases!(todo, ir, idx, stmt, atype, cases, all_covered, joint_effects)
 end
 
 function handle_match!(cases::Vector{InliningCase},

base/compiler/ssair/passes.jl

@@ -1704,7 +1704,7 @@ function adce_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing)
         else
             if is_known_call(stmt, typeassert, compact) && length(stmt.args) == 3
                 # nullify safe `typeassert` calls
-                ty, isexact = instanceof_tfunc(argextype(stmt.args[3], compact))
+                ty, isexact = instanceof_tfunc(argextype(stmt.args[3], compact), true)
                 if isexact && ⊑(𝕃ₒ, argextype(stmt.args[2], compact), ty)
                     compact[idx] = nothing
                     continue