emit precompile statements to separate file

base/options.jl

@@ -26,7 +26,7 @@ struct JLOptions
     warn_overwrite::Int8
     can_inline::Int8
     polly::Int8
-    trace_compile::Int8
+    trace_compile::Ptr{UInt8}
     fast_math::Int8
     worker::Int8
     cookie::Ptr{UInt8}

contrib/generate_precompile.jl

@@ -82,65 +82,63 @@ function generate_precompile_statements()
         print("Generating precompile statements...")
         sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1]
 
-        # Run a repl process and replay our script
-        stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
-        with_fake_pty() do slave, master
-            with_fake_pty() do slave_err, master_err
-                done = false
-                withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
-                        "TERM" => "") do
-                    p = run(`$(julia_cmd()) -O0 --trace-compile=yes --sysimage $sysimg
-                                           --startup-file=no --color=yes`,
-                            slave, slave, slave_err; wait=false)
-                    readuntil(master, "julia>", keep=true)
-                    for (tty, accumulator) in (master     => stdout_accumulator,
-                                               master_err => stderr_accumulator)
-                        @async begin
-                            while true
-                                done && break
-                                write(accumulator, readavailable(tty))
+        mktemp() do precompile_file, _
+            # Run a repl process and replay our script
+            stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
+            with_fake_pty() do slave, master
+                with_fake_pty() do slave_err, master_err
+                    done = false
+                    withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
+                            "TERM" => "") do
+                        p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
+                                               --startup-file=no --color=yes`,
+                                slave, slave, slave_err; wait=false)
+                        readuntil(master, "julia>", keep=true)
+                        for (tty, accumulator) in (master     => stdout_accumulator,
+                                                   master_err => stderr_accumulator)
+                            @async begin
+                                while true
+                                    done && break
+                                    write(accumulator, readavailable(tty))
+                                end
                             end
                         end
-                    end
-                    if have_repl
-                        for l in split(precompile_script, '\n'; keepempty=false)
-                            write(master, l, '\n')
+                        if have_repl
+                            for l in split(precompile_script, '\n'; keepempty=false)
+                                write(master, l, '\n')
+                            end
                         end
+                        write(master, "exit()\n")
+                        wait(p)
+                        done = true
                     end
-                    write(master, "exit()\n")
-                    wait(p)
-                    done = true
                 end
             end
-        end
 
-        stderr_output = String(take!(stderr_accumulator))
-        # println(stderr_output)
-        # stdout_output = String(take!(stdout_accumulator))
-        # println(stdout_output)
-
-        # Extract the precompile statements from stderr
-        statements = Set{String}()
-        for statement in split(stderr_output, '\n')
-            m = match(r"(precompile\(Tuple{.*)", statement)
-            m === nothing && continue
-            statement = m.captures[1]
-            occursin(r"Main.", statement) && continue
-            push!(statements, statement)
-        end
+            # Check what the REPL displayed
+            # stdout_output = String(take!(stdout_accumulator))
+            # println(stdout_output)
 
-        # Load the precompile statements
-        statements_ordered = join(sort(collect(statements)), '\n')
-        # println(statements_ordered)
-        if have_repl
-            # Seems like a reasonable number right now, adjust as needed
-            @assert length(statements) > 700
-        end
+            # Extract the precompile statements from stderr
+            statements = Set{String}()
+            for statement in split(read(precompile_file, String), '\n')
+                occursin("Main.", statement) && continue
+                push!(statements, statement)
+            end
+
+            # Load the precompile statements
+            statements_ordered = join(sort(collect(statements)), '\n')
+            # println(statements_ordered)
+            if have_repl
+                # Seems like a reasonable number right now, adjust as needed
+                @assert length(statements) > 700
+            end
 
-        Base.include_string(PrecompileStagingArea, statements_ordered)
-        print(" $(length(statements)) generated in ")
-        Base.time_print((time() - start_time) * 10^9)
-        println()
+            Base.include_string(PrecompileStagingArea, statements_ordered)
+            print(" $(length(statements)) generated in ")
+            Base.time_print((time() - start_time) * 10^9)
+            println()
+        end
     end
 
     # Fall back to explicit list on Windows, might as well include them

contrib/precompile_explicit.jl

@@ -3,7 +3,7 @@
 # Steps to regenerate this file:
 # 1. Remove all `precompile` calls
 # 2. Rebuild system image
-# 3. Start julia with `--trace-compile=yes and do some stuff
+# 3. Start julia with `--trace-compile=precompiles.txt and do some stuff
 # 5. Run `grep -v '#[0-9]' precompiles.txt >> contrib/precompile_explicit.jl`
 # (filters out closures, which might have different generated names in different environments)
 # This list is only used on Windows, otherwise precompile statements are generated dynamically.

src/gf.c

@@ -1071,6 +1071,10 @@ static jl_method_instance_t *cache_method(
 
 static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int mt_cache, int allow_exec, size_t world)
 {
+    // TODO: Merge with jl_dump_compiles?
+    static ios_t f_precompile;
+    static JL_STREAM* s_precompile = NULL;
+
     // caller must hold the mt->writelock
     jl_typemap_entry_t *entry = NULL;
     entry = jl_typemap_assoc_by_type(mt->cache, (jl_value_t*)tt, NULL, /*subtype*/1, jl_cachearg_offset(mt), world, /*max_world_mask*/0);
@@ -1088,11 +1092,24 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype
     if (entry != NULL) {
         jl_method_t *m = entry->func.method;
         if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) {
-            if (jl_options.trace_compile) {
+            if (jl_options.trace_compile != NULL) {
+                if (s_precompile == NULL) {
+                    const char* t = jl_options.trace_compile;
+                    if (!strncmp(t, "stderr", 6))
+                        s_precompile = JL_STDERR;
+                    else {
+                        if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL)
+                            jl_errorf("cannot open precompile statement file \"%s\" for writing", t);
+                        s_precompile = (JL_STREAM*) &f_precompile;
+                    }
+                }
                 if (!jl_has_free_typevars((jl_value_t*)tt)) {
-                    jl_printf(JL_STDERR, "precompile(");
-                    jl_static_show(JL_STDERR, (jl_value_t*)tt);
-                    jl_printf(JL_STDERR, ")\n");
+                    jl_printf(s_precompile, "precompile(");
+                    jl_static_show(s_precompile, (jl_value_t*)tt);
+                    jl_printf(s_precompile, ")\n");
+
+                    if (s_precompile != JL_STDERR)
+                        ios_flush(&f_precompile);
                 }
             }
             if (!mt_cache) {

src/jloptions.c

@@ -55,11 +55,7 @@ jl_options_t jl_options = { 0,    // quiet
                             0,    // method overwrite warning
                             1,    // can_inline
                             JL_OPTIONS_POLLY_ON, // polly
-#ifdef TRACE_COMPILE
-                            1, // trace_compile
-#else
-                            0, // trace_compile
-#endif
+                            NULL, // trace_compile
                             JL_OPTIONS_FAST_MATH_DEFAULT,
                             0,    // worker
                             NULL, // cookie
@@ -575,13 +571,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
             }
             break;
          case opt_trace_compile:
-            if (!strcmp(optarg,"yes"))
-                jl_options.trace_compile = 1;
-            else if (!strcmp(optarg,"no"))
-                jl_options.trace_compile = 0;
-            else {
-                jl_errorf("julia: invalid argument to --trace-compile (%s)", optarg);
-            }
+            jl_options.trace_compile = strdup(optarg);
+            if (!jl_options.trace_compile)
+                jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno));
             break;
         case opt_math_mode:
             if (!strcmp(optarg,"ieee"))

src/julia.h

@@ -1799,7 +1799,7 @@ typedef struct {
     int8_t warn_overwrite;
     int8_t can_inline;
     int8_t polly;
-    int8_t trace_compile;
+    const char *trace_compile;
     int8_t fast_math;
     int8_t worker;
     const char *cookie;

src/options.h

@@ -89,7 +89,6 @@
 
 // print all signatures type inference is invoked on
 //#define TRACE_INFERENCE
-//#define TRACE_COMPILE
 
 // print all generic method dispatches (excludes inlined and specialized call
 // sites). this generally prints too much output to be useful.