From d1be419a038018aecd3fa803f5b117b334553c7a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 3 Aug 2018 02:10:42 +0200 Subject: [PATCH] emit precompile statements to separate file if the filename to trace--compile is stdout, print to the stdout stream --- base/options.jl | 2 +- contrib/generate_precompile.jl | 98 +++++++++++++++++----------------- contrib/precompile_explicit.jl | 2 +- src/gf.c | 25 +++++++-- src/jloptions.c | 16 ++---- src/julia.h | 2 +- src/options.h | 1 - 7 files changed, 76 insertions(+), 70 deletions(-) diff --git a/base/options.jl b/base/options.jl index 0abf2d55fd866..0706b1c07d72f 100644 --- a/base/options.jl +++ b/base/options.jl @@ -26,7 +26,7 @@ struct JLOptions warn_overwrite::Int8 can_inline::Int8 polly::Int8 - trace_compile::Int8 + trace_compile::Ptr{UInt8} fast_math::Int8 worker::Int8 cookie::Ptr{UInt8} diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index e109d5c77056f..30aaa085d568c 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -82,65 +82,63 @@ function generate_precompile_statements() print("Generating precompile statements...") sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1] - # Run a repl process and replay our script - stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer() - with_fake_pty() do slave, master - with_fake_pty() do slave_err, master_err - done = false - withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing, - "TERM" => "") do - p = run(`$(julia_cmd()) -O0 --trace-compile=yes --sysimage $sysimg - --startup-file=no --color=yes`, - slave, slave, slave_err; wait=false) - readuntil(master, "julia>", keep=true) - for (tty, accumulator) in (master => stdout_accumulator, - master_err => stderr_accumulator) - @async begin - while true - done && break - write(accumulator, readavailable(tty)) + mktemp() do precompile_file, _ + # Run a repl process and replay our script + stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer() + with_fake_pty() do slave, master + with_fake_pty() do slave_err, master_err + done = false + withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing, + "TERM" => "") do + p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg + --startup-file=no --color=yes`, + slave, slave, slave_err; wait=false) + readuntil(master, "julia>", keep=true) + for (tty, accumulator) in (master => stdout_accumulator, + master_err => stderr_accumulator) + @async begin + while true + done && break + write(accumulator, readavailable(tty)) + end end end - end - if have_repl - for l in split(precompile_script, '\n'; keepempty=false) - write(master, l, '\n') + if have_repl + for l in split(precompile_script, '\n'; keepempty=false) + write(master, l, '\n') + end end + write(master, "exit()\n") + wait(p) + done = true end - write(master, "exit()\n") - wait(p) - done = true end end - end - stderr_output = String(take!(stderr_accumulator)) - # println(stderr_output) - # stdout_output = String(take!(stdout_accumulator)) - # println(stdout_output) - - # Extract the precompile statements from stderr - statements = Set{String}() - for statement in split(stderr_output, '\n') - m = match(r"(precompile\(Tuple{.*)", statement) - m === nothing && continue - statement = m.captures[1] - occursin(r"Main.", statement) && continue - push!(statements, statement) - end + # Check what the REPL displayed + # stdout_output = String(take!(stdout_accumulator)) + # println(stdout_output) - # Load the precompile statements - statements_ordered = join(sort(collect(statements)), '\n') - # println(statements_ordered) - if have_repl - # Seems like a reasonable number right now, adjust as needed - @assert length(statements) > 700 - end + # Extract the precompile statements from stderr + statements = Set{String}() + for statement in split(read(precompile_file, String), '\n') + occursin("Main.", statement) && continue + push!(statements, statement) + end + + # Load the precompile statements + statements_ordered = join(sort(collect(statements)), '\n') + # println(statements_ordered) + if have_repl + # Seems like a reasonable number right now, adjust as needed + @assert length(statements) > 700 + end - Base.include_string(PrecompileStagingArea, statements_ordered) - print(" $(length(statements)) generated in ") - Base.time_print((time() - start_time) * 10^9) - println() + Base.include_string(PrecompileStagingArea, statements_ordered) + print(" $(length(statements)) generated in ") + Base.time_print((time() - start_time) * 10^9) + println() + end end # Fall back to explicit list on Windows, might as well include them diff --git a/contrib/precompile_explicit.jl b/contrib/precompile_explicit.jl index b8adb344f2c16..f0d83e491b7eb 100644 --- a/contrib/precompile_explicit.jl +++ b/contrib/precompile_explicit.jl @@ -3,7 +3,7 @@ # Steps to regenerate this file: # 1. Remove all `precompile` calls # 2. Rebuild system image -# 3. Start julia with `--trace-compile=yes and do some stuff +# 3. Start julia with `--trace-compile=precompiles.txt and do some stuff # 5. Run `grep -v '#[0-9]' precompiles.txt >> contrib/precompile_explicit.jl` # (filters out closures, which might have different generated names in different environments) # This list is only used on Windows, otherwise precompile statements are generated dynamically. diff --git a/src/gf.c b/src/gf.c index bb96cd3a721f0..c179563422013 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1071,6 +1071,10 @@ static jl_method_instance_t *cache_method( static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int mt_cache, int allow_exec, size_t world) { + // TODO: Merge with jl_dump_compiles? + static ios_t f_precompile; + static JL_STREAM* s_precompile = NULL; + // caller must hold the mt->writelock jl_typemap_entry_t *entry = NULL; entry = jl_typemap_assoc_by_type(mt->cache, (jl_value_t*)tt, NULL, /*subtype*/1, jl_cachearg_offset(mt), world, /*max_world_mask*/0); @@ -1088,11 +1092,24 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype if (entry != NULL) { jl_method_t *m = entry->func.method; if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) { - if (jl_options.trace_compile) { + if (jl_options.trace_compile != NULL) { + if (s_precompile == NULL) { + const char* t = jl_options.trace_compile; + if (!strncmp(t, "stderr", 6)) + s_precompile = JL_STDERR; + else { + if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL) + jl_errorf("cannot open precompile statement file \"%s\" for writing", t); + s_precompile = (JL_STREAM*) &f_precompile; + } + } if (!jl_has_free_typevars((jl_value_t*)tt)) { - jl_printf(JL_STDERR, "precompile("); - jl_static_show(JL_STDERR, (jl_value_t*)tt); - jl_printf(JL_STDERR, ")\n"); + jl_printf(s_precompile, "precompile("); + jl_static_show(s_precompile, (jl_value_t*)tt); + jl_printf(s_precompile, ")\n"); + + if (s_precompile != JL_STDERR) + ios_flush(&f_precompile); } } if (!mt_cache) { diff --git a/src/jloptions.c b/src/jloptions.c index 9d2a30e2d0fa7..aeefc20c24892 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -55,11 +55,7 @@ jl_options_t jl_options = { 0, // quiet 0, // method overwrite warning 1, // can_inline JL_OPTIONS_POLLY_ON, // polly -#ifdef TRACE_COMPILE - 1, // trace_compile -#else - 0, // trace_compile -#endif + NULL, // trace_compile JL_OPTIONS_FAST_MATH_DEFAULT, 0, // worker NULL, // cookie @@ -575,13 +571,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) } break; case opt_trace_compile: - if (!strcmp(optarg,"yes")) - jl_options.trace_compile = 1; - else if (!strcmp(optarg,"no")) - jl_options.trace_compile = 0; - else { - jl_errorf("julia: invalid argument to --trace-compile (%s)", optarg); - } + jl_options.trace_compile = strdup(optarg); + if (!jl_options.trace_compile) + jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno)); break; case opt_math_mode: if (!strcmp(optarg,"ieee")) diff --git a/src/julia.h b/src/julia.h index a49bd51db1853..bcf1d1050dc21 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1799,7 +1799,7 @@ typedef struct { int8_t warn_overwrite; int8_t can_inline; int8_t polly; - int8_t trace_compile; + const char *trace_compile; int8_t fast_math; int8_t worker; const char *cookie; diff --git a/src/options.h b/src/options.h index a669ade52ce43..b860edecce3dc 100644 --- a/src/options.h +++ b/src/options.h @@ -89,7 +89,6 @@ // print all signatures type inference is invoked on //#define TRACE_INFERENCE -//#define TRACE_COMPILE // print all generic method dispatches (excludes inlined and specialized call // sites). this generally prints too much output to be useful.