compilethis(iscompilable) function for automated opt-in to module com…

…pilation on import (closes #12462)

NEWS.md

@@ -28,11 +28,17 @@ New language features
  * The syntax `function foo end` can be used to introduce a generic function without
  yet adding any methods ([#8283]).
 
- * Incremental compilation of modules: `Base.compile(module::Symbol)` imports the named module,
- but instead of loading it into the current session saves the result of compiling it in
+ * Incremental compilation of modules: call ``compilethis()`` at the top of a
+ module file to automatically compile it when it is imported ([#12491]), or manually
+ run `Base.compile(modulename)`. The resulting compiled `.ji` file is saved in
  `~/.julia/lib/v0.4` ([#8745]).
 
- * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.
+ * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. In particular, to be safely compilable a module may need an `__init__` function to separate code that must be executed at runtime rather than compile-time. Modules that are *not* compilable should call `compilethis(false)`.
+
+ * The compiled `.ji` file includes a list of dependencies (modules and files that
+ were imported/included at compile-time), and the module is automatically recompiled
+ upon `import` when any of its dependencies have changed. Explicit dependencies
+ on other files can be declared with `include_dependency(path)` ([#12458]).
 
  * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line.
 
@@ -1557,3 +1563,5 @@ Too numerous to mention.
 [#12137]: https://github.com/JuliaLang/julia/issues/12137
 [#12162]: https://github.com/JuliaLang/julia/issues/12162
 [#12393]: https://github.com/JuliaLang/julia/issues/12393
+[#12458]: https://github.com/JuliaLang/julia/issues/12458
+[#12491]: https://github.com/JuliaLang/julia/issues/12491

base/docs/helpdb.jl

@@ -14579,6 +14579,26 @@ used via `include`. It has no effect outside of compilation.
 """
 include_dependency
 
+doc"""
+```rst
+::
+ __compile__(iscompilable::Bool=true)
+
+Specify whether the file calling this function is compilable. If
+``iscompilable`` is ``true``, then ``__compile__`` throws an exception
+*unless* the file is being compiled, and in a module file it causes
+the module to be automatically compiled when it is imported.
+Typically, ``__compile__()`` should occur before the ``module``
+declaration in the file, or better yet ``VERSION >= v"0.4-" &&
+__compile__()`` in order to be backward-compatible with Julia 0.3.
+
+If a module or file is *not* safely compilable, it should call
+``__compile__(false)`` in order to throw an error if Julia attempts
+to compile it.
+```
+"""
+__compile__
+
 doc"""
 ```rst
 ::

base/exports.jl

@@ -1101,6 +1101,7 @@ export
  workspace,
 
 # loading source files
+ __compile__,
  evalfile,
  include,
  include_string,

base/loading.jl

@@ -131,6 +131,30 @@ function include_dependency(path::AbstractString)
  return nothing
 end
 
+# We throw CompilableError(true) when a module wants to be compiled but isn't,
+# and CompilableError(false) when a module doesn't want to be compiled but is
+immutable CompilableError <: Exception
+ iscompilable::Bool
+end
+function show(io::IO, ex::CompilableError)
+ if ex.iscompilable
+ print(io, "__compile__(true) is only allowed in module files being imported")
+ else
+ print(io, "__compile__(false) is not allowed in files that are being compiled")
+ end
+end
+compilableerror(ex::CompilableError, c) = ex.iscompilable == c
+compilableerror(ex::LoadError, c) = compilableerror(ex.error, c)
+compilableerror(ex, c) = false
+
+# put at the top of a file to force it to be compiled (true), or
+# to be prevent it from being compiled (false).
+function __compile__(iscompilable::Bool=true)
+ if myid() == 1 && iscompilable != (0 != ccall(:jl_generating_output, Cint, ()))
+ throw(CompilableError(iscompilable))
+ end
+end
+
 # require always works in Main scope and loads files from node 1
 toplevel_load = true
 function require(mod::Symbol)
@@ -166,13 +190,26 @@ function require(mod::Symbol)
  name = string(mod)
  path = find_in_node_path(name, source_dir(), 1)
  path === nothing && throw(ArgumentError("$name not found in path"))
- if last && myid() == 1 && nprocs() > 1
- # broadcast top-level import/using from node 1 (only)
- content = open(readall, path)
- refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ]
- for r in refs; wait(r); end
- else
- eval(Main, :(Base.include_from_node1($path)))
+ try
+ if last && myid() == 1 && nprocs() > 1
+ # include on node 1 first to check for CompilableErrors
+ eval(Main, :(Base.include_from_node1($path)))
+
+ # broadcast top-level import/using from node 1 (only)
+ refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ]
+ for r in refs; wait(r); end
+ else
+ eval(Main, :(Base.include_from_node1($path)))
+ end
+ catch ex
+ if !compilableerror(ex, true)
+ rethrow() # rethrow non-compilable=true errors
+ end
+ isinteractive() && info("Compiling module $mod...")
+ cachefile = compile(mod)
+ if nothing === _require_from_serialized(1, mod, cachefile, last)
+ error("__compile__(true) but require failed to create a precompiled cache file")
+ end
  end
  finally
  toplevel_load = last
@@ -294,7 +331,9 @@ function compile(name::ByteString)
  mkpath(cachepath)
  end
  cachefile = abspath(cachepath, name*".ji")
- create_expr_cache(path, cachefile)
+ if !success(create_expr_cache(path, cachefile))
+ error("Failed to compile $name to $cachefile")
+ end
  return cachefile
 end
 

doc/manual/modules.rst

@@ -263,9 +263,22 @@ incremental compile and custom system image.
 To create a custom system image that can be used to start julia with the -J option,
 recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules.
 
-To create an incremental precompiled module file,
-call ``Base.compile(modulename::Symbol)``.
-The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``.
+To create an incremental precompiled module file, add
+``__compile__()`` at the top of your module file (before the
+``module`` starts). This will cause it to be automatically compiled
+the first time it is imported. Alternatively, you can manually call
+``Base.compile(modulename)``. The resulting cache files will be
+stored in ``Base.LOAD_CACHE_PATH[1]``. Subsequently, the module is
+automatically recompiled upon ``import`` whenever any of its
+dependencies change; dependencies are modules it imports, the Julia
+build, files it includes, or explicit dependencies declared by
+``include_dependency(path)`` in the module file(s). Compiling a
+module also recursively compiles any modules that are imported
+therein. If you know that it is *not* safe to compile your module
+(for the reasons described below), you should put
+``__compile__(false)`` in the module file to cause ``Base.compile`` to
+throw an error (and thereby prevent the module from being imported by
+any other compiled module).
 
 In order to make your module work with precompilation,
 however, you may need to change your module to explicitly separate any

test/compile.jl

@@ -7,10 +7,11 @@ insert!(LOAD_PATH, 1, dir)
 insert!(Base.LOAD_CACHE_PATH, 1, dir)
 Foo_module = :Foo4b3a94a1a081a8cb
 try
- file = joinpath(dir, "$Foo_module.jl")
+ Foo_file = joinpath(dir, "$Foo_module.jl")
 
- open(file, "w") do f
+ open(Foo_file, "w") do f
  print(f, """
+ compilethis(true)
  module $Foo_module
  @doc "foo function" foo(x) = x + 1
  include_dependency("foo.jl")
@@ -22,10 +23,17 @@ try
  """)
  end
 
- cachefile = Base.compile(Foo_module)
+ if myid() == 1
+ @test_throws Base.CompilableError compilethis(true)
+ @test_throws LoadError include(Foo_file) # from compilethis(true)
+ end
+
+ Base.require(Foo_module)
+ cachefile = joinpath(dir, "$Foo_module.ji")
 
  # use _require_from_serialized to ensure that the test fails if
  # the module doesn't load from the image:
+ println(STDERR, "\nNOTE: The following 'replacing module' warning indicates normal operation:")
  @test nothing !== Base._require_from_serialized(myid(), Foo_module, true)
 
  let Foo = eval(Main, Foo_module)
@@ -39,9 +47,19 @@ try
  deps = Base.cache_dependencies(cachefile)
  @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))),
  [:Base,:Core,:Main])
- @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+ @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
  end
 
+ Baz_file = joinpath(dir, "Baz.jl")
+ open(Baz_file, "w") do f
+ print(f, """
+ compilethis(false)
+ module Baz
+ end
+ """)
+ end
+ println(STDERR, "\nNOTE: The following 'LoadError: compilethis(false)' indicates normal operation")
+ @test_throws ErrorException Base.compile("Baz") # from compilethis(false)
 finally
  splice!(Base.LOAD_CACHE_PATH, 1)
  splice!(LOAD_PATH, 1)