From d6c0109706ebcf69bbece09d25b8f36c47fb0cff Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 28 Nov 2019 18:17:26 +0100 Subject: [PATCH] only include needed stdlibs in apps (#25) * wip only using needed stdlibs * only include needed stdlibs in apps * make default off --- examples/MyApp/src/MyApp.jl | 5 +- src/PackageCompilerX.jl | 146 ++++++++++++++++++++++++------------ test/runtests.jl | 22 ++++-- 3 files changed, 117 insertions(+), 56 deletions(-) diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index 8e955358..45502c43 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -23,9 +23,12 @@ function real_main() @show LOAD_PATH @show pwd() @show Base.active_project() + @show Threads.nthreads() @show Sys.BINDIR - @info "Running an artifact:" + display(Base.loaded_modules) + println("Running an artifact:") run(`$socrates`) + println() @show unsafe_string(Base.JLOptions().image_file) @show Example.domath(5) @show sin(0.0) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 7b5b85ab..d86d4a06 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -32,20 +32,19 @@ function get_julia_cmd() cmd = `$julia_path --color=yes --startup-file=no --cpu-target=$DEFAULT_TARGET` end -# TODO: Add output file? -function run_precompilation_script(project::String, precompile_file::String) - tracefile = tempname() - julia_code = """Base.__init__(); include($(repr(precompile_file)))""" - cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project - --compile=all --trace-compile=$tracefile -e $julia_code` - @debug "run_precompilation_script: running $cmd" - run(cmd) - return tracefile -end +all_stdlibs() = readdir(Sys.STDLIB) + +function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) + sysimg_source_path = Base.find_source_file("sysimg.jl") + sysimg_content = read(sysimg_source_path, String) + # replaces the hardcoded list of stdlibs in sysimg.jl with + # the stdlibs that is given as argument + return replace(sysimg_content, r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* string.(stdlibs), ",\n"), "]")) +end -function bootstrap_sys() - tmp = mktempdir() +function create_fresh_base_sysimage(stdlibs::Vector{String}) + tmp = mktempdir(cleanup=false) sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji") @@ -60,14 +59,38 @@ function bootstrap_sys() read(cmd) # Use that to create sys.ji - cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji --output-ji=$tmp_sys_ji $sysimg_source_path` - @debug "running $cmd" - read(cmd) + new_sysimage_content = rewrite_sysimg_jl_only_needed_stdlibs(stdlibs) + new_sysimage_source_path = joinpath(base_dir, "sysimage_packagecompiler_x.jl") + write(new_sysimage_source_path, new_sysimage_content) + try + cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` + @debug "running $cmd" + read(cmd) + finally + rm(new_sysimage_source_path; force=true) + end end return tmp_sys_ji end +# TODO: Add output file? +function run_precompilation_script(project::String, precompile_file::Union{String, Nothing}) + # TODO: Audit tempname usage + tracefile = tempname() + if precompile_file == nothing + arg = `-e ''` + else + arg = `$precompile_file` + end + touch(tracefile) + cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project + --compile=all --trace-compile=$tracefile $arg` + @debug "run_precompilation_script: running $cmd" + run(cmd) + return tracefile +end + function create_sysimg_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; project::String, base_sysimg::String, @@ -86,41 +109,37 @@ function create_sysimg_object_file(object_file::String, packages::Union{Symbol, end # handle precompilation - if precompile_execution_file !== nothing || precompile_statements_file !== nothing - precompile_statements = "" - if precompile_execution_file !== nothing - @debug "running precompilation execution script..." - tracefile = run_precompilation_script(project, precompile_execution_file) - precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" - end - if precompile_statements_file != nothing - precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" - end + precompile_statements = "" + @debug "running precompilation execution script..." + tracefile = run_precompilation_script(project, precompile_execution_file) + precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + if precompile_statements_file != nothing + precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" + end - precompile_code = """ - # This @eval prevents symbols from being put into Main - @eval Module() begin - PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) - end + precompile_code = """ + # This @eval prevents symbols from being put into Main + @eval Module() begin + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) end - precompile_statements = String[] - $precompile_statements - for statement in sort(precompile_statements) - # println(statement) - try - Base.include_string(PrecompileStagingArea, statement) - catch - # See julia issue #28808 - @error "failed to execute \$statement" - end + end + precompile_statements = String[] + $precompile_statements + for statement in sort(precompile_statements) + # println(statement) + try + Base.include_string(PrecompileStagingArea, statement) + catch + # See julia issue #28808 + @error "failed to execute \$statement" end - end # module - """ - julia_code *= precompile_code - end + end + end # module + """ + julia_code *= precompile_code # finally, make julia output the resulting object file @debug "creating object file at $object_file" @@ -135,12 +154,28 @@ default_sysimg_name() = basename(default_sysimg_path()) backup_default_sysimg_path() = default_sysimg_path() * ".backup" backup_default_sysimg_name() = basename(backup_default_sysimg_path()) +# TODO: Also check UUIDs for stdlibs, not only names +function gather_stdlibs_project(project::String) + project_toml_path = abspath(Pkg.Types.projectfile_path(project; strict=true)) + ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) + @assert ctx.env.manifest !== nothing + stdlibs = all_stdlibs() + stdlibs_project = String[] + for (uuid, pkg) in ctx.env.manifest + if pkg.name in stdlibs + push!(stdlibs_project, pkg.name) + end + end + return stdlibs_project +end + function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path::Union{String,Nothing}=nothing, project::String=active_project(), precompile_execution_file::Union{String, Nothing}=nothing, precompile_statements_file::Union{String, Nothing}=nothing, - incremental=true, + incremental::Bool=true, + filter_stdlibs=false, replace_default_sysimg::Bool=false) if sysimage_path === nothing && replace_default_sysimg == false error("`sysimage_path` cannot be `nothing` if `replace_default_sysimg` is `false`") @@ -150,8 +185,17 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) end + if filter_stdlibs && incremental + error("must use `incremental=false` to use `filter_stdlibs=true`") + end + if !incremental - base_sysimg = bootstrap_sys() + if filter_stdlibs + stdlibs = gather_stdlibs_project(project) + else + stdlibs= all_stdlibs() + end + base_sysimg = create_fresh_base_sysimage(stdlibs) else base_sysimg = current_process_sysimage_path() end @@ -239,6 +283,7 @@ function create_app(package_dir::String, precompile_execution_file::Union{String,Nothing}=nothing, precompile_statements_file::Union{String,Nothing}=nothing, incremental=false, + filter_stdlibs=false, audit=true, force=false) project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) @@ -277,7 +322,8 @@ function create_app(package_dir::String, # TODO: Create in a temp dir and then move it into place? # TODO: Maybe avoid this cd? cd(app_dir) do - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, incremental=incremental) + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + incremental=incremental, filter_stdlibs=filter_stdlibs) mkpath("bin") create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=joinpath("bin", app_name)) mv(sysimg_file, joinpath("bin", sysimg_file)) diff --git a/test/runtests.jl b/test/runtests.jl index 379a575a..ae12cfca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -17,10 +17,22 @@ using Libdl @test_logs PackageCompilerX.audit_app(app_source_dir) app_exe_dir = joinpath(tmp, "MyApp") for incremental in (true, false) - create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true) - app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) - app_output = read(`$app_path`, String) - @test occursin("Example.domath", app_output) - @test occursin("ἔοικα γοῦν τούτου", app_output) + if incremental == false + filter_stdlibs = (true, false) + else + filter_stdlibs = (false,) + end + for filter in filter_stdlibs + create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true, filter_stdlibs=filter) + app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) + app_output = read(`$app_path`, String) + if filter == true + @test !(occursin("LinearAlgebra", app_output)) + else + @test occursin("LinearAlgebra", app_output) + end + @test occursin("Example.domath", app_output) + @test occursin("ἔοικα γοῦν τούτου", app_output) + end end end