Skip to content

Commit

Permalink
add being able to pass a CacheFlags() to isprecompiled to verify …
Browse files Browse the repository at this point in the history 
…a precompile file against a set of custom flags (#53332)
  • Loading branch information
@KristofferC
KristofferC authored Feb 19, 2024
1 parent 59102aa commit 9c0f1dc
Show file tree
Hide file tree
Showing 5 changed files with 168 additions and 142 deletions.
268 changes: 139 additions & 129 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1495,6 +1495,126 @@ end

# End extensions


struct CacheFlags
# OOICCDDP - see jl_cache_flags
use_pkgimages::Bool
debug_level::Int
check_bounds::Int
inline::Bool
opt_level::Int
end
function CacheFlags(f::UInt8)
use_pkgimages = Bool(f & 1)
debug_level = Int((f >> 1) & 3)
check_bounds = Int((f >> 3) & 3)
inline = Bool((f >> 5) & 1)
opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils
CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level)
end
CacheFlags(f::Int) = CacheFlags(UInt8(f))
CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ()))

function _cacheflag_to_uint8(cf::CacheFlags)::UInt8
f = UInt8(0)
f |= cf.use_pkgimages << 0
f |= cf.debug_level << 1
f |= cf.check_bounds << 3
f |= cf.inline << 5
f |= cf.opt_level << 6
return f
end

function show(io::IO, cf::CacheFlags)
print(io, "use_pkgimages = ", cf.use_pkgimages)
print(io, ", debug_level = ", cf.debug_level)
print(io, ", check_bounds = ", cf.check_bounds)
print(io, ", inline = ", cf.inline)
print(io, ", opt_level = ", cf.opt_level)
end

struct ImageTarget
name::String
flags::Int32
ext_features::String
features_en::Vector{UInt8}
features_dis::Vector{UInt8}
end

function parse_image_target(io::IO)
flags = read(io, Int32)
nfeature = read(io, Int32)
feature_en = read(io, 4*nfeature)
feature_dis = read(io, 4*nfeature)
name_len = read(io, Int32)
name = String(read(io, name_len))
ext_features_len = read(io, Int32)
ext_features = String(read(io, ext_features_len))
ImageTarget(name, flags, ext_features, feature_en, feature_dis)
end

function parse_image_targets(targets::Vector{UInt8})
io = IOBuffer(targets)
ntargets = read(io, Int32)
targets = Vector{ImageTarget}(undef, ntargets)
for i in 1:ntargets
targets[i] = parse_image_target(io)
end
return targets
end

function current_image_targets()
targets = @ccall jl_reflect_clone_targets()::Vector{UInt8}
return parse_image_targets(targets)
end

struct FeatureName
name::Cstring
bit::UInt32 # bit index into a `uint32_t` array;
llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support
end

function feature_names()
fnames = Ref{Ptr{FeatureName}}()
nf = Ref{Csize_t}()
@ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid
if fnames[] == C_NULL
@assert nf[] == 0
return Vector{FeatureName}(undef, 0)
end
Base.unsafe_wrap(Array, fnames[], nf[], own=false)
end

function test_feature(features::Vector{UInt8}, feat::FeatureName)
bitidx = feat.bit
u8idx = div(bitidx, 8) + 1
bit = bitidx % 8
return (features[u8idx] & (1 << bit)) != 0
end

function show(io::IO, it::ImageTarget)
print(io, it.name)
if !isempty(it.ext_features)
print(io, ",", it.ext_features)
end
print(io, "; flags=", it.flags)
print(io, "; features_en=(")
first = true
for feat in feature_names()
if test_feature(it.features_en, feat)
name = Base.unsafe_string(feat.name)
if first
first = false
print(io, name)
else
print(io, ", ", name)
end
end
end
print(io, ")")
# Is feature_dis useful?
end

# should sync with the types of arguments of `stale_cachefile`
const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String}

Expand All @@ -1515,11 +1635,11 @@ function isprecompiled(pkg::PkgId;
ignore_loaded::Bool=false,
stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(),
cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg),
sourcepath::Union{String,Nothing}=Base.locate_package(pkg)
)
sourcepath::Union{String,Nothing}=Base.locate_package(pkg),
flags::CacheFlags=CacheFlags())
isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))")
for path_to_try in cachepaths
staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true)
staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags)
if staledeps === true
continue
end
Expand All @@ -1532,7 +1652,7 @@ function isprecompiled(pkg::PkgId;
modpaths = find_all_in_cache_path(modkey)
for modpath_to_try in modpaths::Vector{String}
stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey
if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded) === true,
if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true,
stale_cache, stale_cache_key)
continue
end
Expand Down Expand Up @@ -2535,8 +2655,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::
if output_o !== nothing
@debug "Generating object cache file for $(repr("text/plain", pkg))"
cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing)
opt_level = Base.JLOptions().opt_level
opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes`
opts = `--output-o $(output_o) --output-ji $(output) --output-incremental=yes`
else
@debug "Generating cache file for $(repr("text/plain", pkg))"
cpu_target = nothing
Expand All @@ -2546,10 +2665,11 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::
deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing)
deps = deps_eltype * "[" * join(deps_strs, ",") * "]"
trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : ``
io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts)
io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd)
$(flags)
$(opts)
--startup-file=no --history-file=no --warn-overwrite=yes
--color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no")
$flags
$trace
-`,
"OPENBLAS_NUM_THREADS" => 1,
Expand Down Expand Up @@ -3212,116 +3332,6 @@ function check_clone_targets(clone_targets)
end
end

struct CacheFlags
# OOICCDDP - see jl_cache_flags
use_pkgimages::Bool
debug_level::Int
check_bounds::Int
inline::Bool
opt_level::Int

function CacheFlags(f::UInt8)
use_pkgimages = Bool(f & 1)
debug_level = Int((f >> 1) & 3)
check_bounds = Int((f >> 3) & 3)
inline = Bool((f >> 5) & 1)
opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils
new(use_pkgimages, debug_level, check_bounds, inline, opt_level)
end
end
CacheFlags(f::Int) = CacheFlags(UInt8(f))
CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ()))

function show(io::IO, cf::CacheFlags)
print(io, "use_pkgimages = ", cf.use_pkgimages)
print(io, ", debug_level = ", cf.debug_level)
print(io, ", check_bounds = ", cf.check_bounds)
print(io, ", inline = ", cf.inline)
print(io, ", opt_level = ", cf.opt_level)
end

struct ImageTarget
name::String
flags::Int32
ext_features::String
features_en::Vector{UInt8}
features_dis::Vector{UInt8}
end

function parse_image_target(io::IO)
flags = read(io, Int32)
nfeature = read(io, Int32)
feature_en = read(io, 4*nfeature)
feature_dis = read(io, 4*nfeature)
name_len = read(io, Int32)
name = String(read(io, name_len))
ext_features_len = read(io, Int32)
ext_features = String(read(io, ext_features_len))
ImageTarget(name, flags, ext_features, feature_en, feature_dis)
end

function parse_image_targets(targets::Vector{UInt8})
io = IOBuffer(targets)
ntargets = read(io, Int32)
targets = Vector{ImageTarget}(undef, ntargets)
for i in 1:ntargets
targets[i] = parse_image_target(io)
end
return targets
end

function current_image_targets()
targets = @ccall jl_reflect_clone_targets()::Vector{UInt8}
return parse_image_targets(targets)
end

struct FeatureName
name::Cstring
bit::UInt32 # bit index into a `uint32_t` array;
llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support
end

function feature_names()
fnames = Ref{Ptr{FeatureName}}()
nf = Ref{Csize_t}()
@ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid
if fnames[] == C_NULL
@assert nf[] == 0
return Vector{FeatureName}(undef, 0)
end
Base.unsafe_wrap(Array, fnames[], nf[], own=false)
end

function test_feature(features::Vector{UInt8}, feat::FeatureName)
bitidx = feat.bit
u8idx = div(bitidx, 8) + 1
bit = bitidx % 8
return (features[u8idx] & (1 << bit)) != 0
end

function show(io::IO, it::ImageTarget)
print(io, it.name)
if !isempty(it.ext_features)
print(io, ",", it.ext_features)
end
print(io, "; flags=", it.flags)
print(io, "; features_en=(")
first = true
for feat in feature_names()
if test_feature(it.features_en, feat)
name = Base.unsafe_string(feat.name)
if first
first = false
print(io, name)
else
print(io, ", ", name)
end
end
end
print(io, ")")
# Is feature_dis useful?
end

# Set by FileWatching.__init__()
global mkpidlock_hook
global trymkpidlock_hook
Expand Down Expand Up @@ -3377,11 +3387,11 @@ list_reasons(::Nothing) = ""

# returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey
# otherwise returns the list of dependencies to also check
@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, reasons=nothing)
return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, reasons)
@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing)
return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons)
end
@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String;
ignore_loaded::Bool = false, reasons::Union{Dict{String,Int},Nothing}=nothing)
ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=nothing)
io = open(cachefile, "r")
try
checksum = isvalid_cache_header(io)
Expand All @@ -3390,15 +3400,15 @@ end
record_reason(reasons, "invalid header")
return true # invalid cache file
end
modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io, cachefile)
modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, actual_flags = parse_cache_header(io, cachefile)
if isempty(modules)
return true # ignore empty file
end
if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0
if @ccall(jl_match_cache_flags(_cacheflag_to_uint8(requested_flags)::UInt8, actual_flags::UInt8)::UInt8) == 0
@debug """
Rejecting cache file $cachefile for $modkey since the flags are mismatched
current session: $(CacheFlags())
cache file: $(CacheFlags(flags))
requested flags: $(requested_flags) [$(_cacheflag_to_uint8(requested_flags))]
cache file: $(CacheFlags(actual_flags)) [$actual_flags]
"""
record_reason(reasons, "mismatched flags")
return true
Expand Down Expand Up @@ -3672,7 +3682,7 @@ function precompile(@nospecialize(argt::Type), m::Method)
return precompile(mi)
end

precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing))
precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String))
precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO, Cmd))
precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO, Cmd))
@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing))
@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String))
@assert precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, IO, IO))
@assert precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, IO, IO))
1 change: 1 addition & 0 deletions src/jl_exported_funcs.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@
XX(jl_raise_debugger) \
XX(jl_readuntil) \
XX(jl_cache_flags) \
XX(jl_match_cache_flags_current) \
XX(jl_match_cache_flags) \
XX(jl_read_verify_header) \
XX(jl_realloc) \
Expand Down
2 changes: 1 addition & 1 deletion src/staticdata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3566,7 +3566,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_
"Precompile file header verification checks failed.");
}
uint8_t flags = read_uint8(f);
if (pkgimage && !jl_match_cache_flags(flags)) {
if (pkgimage && !jl_match_cache_flags_current(flags)) {
return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch");
}
if (!pkgimage) {
Expand Down
Loading

0 comments on commit 9c0f1dc

Please sign in to comment.