Make compilecache atomic (#36416)

When several Julia processes compile the same package concurrently
(e.g. during a cluster run), they can conflict on the compile cache
file. This change makes a Julia process create a compile cache in a
temporary file and atomically rename it to the final cache file.

Co-authored-by: Takafumi Arakaki <tkf@@users.noreply.github.com>

NEWS.md

@@ -18,6 +18,8 @@ Compiler/Runtime improvements
  This allows executable-relative paths to be embedded within executables on all
  platforms, not just MacOS, which the syntax is borrowed from. ([#35627])
 * Constant propogation now occurs through keyword arguments ([#35976])
+* The precompilation cache is now created atomically ([#36416]). Invoking _n_
+ Julia processes simultaneously may create _n_ temporary caches.
 
 Command-line option changes
 ---------------------------

base/loading.jl

@@ -1301,9 +1301,9 @@ const MAX_NUM_PRECOMPILE_FILES = 10
 function compilecache(pkg::PkgId, path::String)
  # decide where to put the resulting cache file
  cachefile = compilecache_path(pkg)
+ cachepath = dirname(cachefile)
  # prune the directory with cache files
  if pkg.uuid !== nothing
- cachepath = dirname(cachefile)
  entrypath, entryfile = cache_file_entry(pkg)
  cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
  if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
@@ -1321,20 +1321,34 @@ function compilecache(pkg::PkgId, path::String)
  # run the expression and cache the result
  verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
  @logmsg verbosity "Precompiling $pkg"
- p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid)
- if success(p)
- # append checksum to the end of the .ji file:
- open(cachefile, "a+") do f
- write(f, _crc32c(seekstart(f)))
+
+ # create a temporary file in `cachepath` directory, write the cache in it,
+ # write the checksum, _and then_ atomically move the file to `cachefile`.
+ tmppath, tmpio = mktemp(cachepath)
+ local p
+ try
+ close(tmpio)
+ p = create_expr_cache(path, tmppath, concrete_deps, pkg.uuid)
+ if success(p)
+ # append checksum to the end of the .ji file:
+ open(tmppath, "a+") do f
+ write(f, _crc32c(seekstart(f)))
+ end
+ # inherit permission from the source file
+ chmod(tmppath, filemode(path) & 0o777)
+
+ # this is atomic according to POSIX:
+ rename(tmppath, cachefile)
+ return cachefile
  end
- # inherit permission from the source file
- chmod(cachefile, filemode(path) & 0o777)
- elseif p.exitcode == 125
+ finally
+ rm(tmppath, force=true)
+ end
+ if p.exitcode == 125
  return PrecompilableError()
  else
  error("Failed to precompile $pkg to $cachefile.")
  end
- return cachefile
 end
 
 module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m)

doc/src/manual/faq.md

@@ -935,6 +935,22 @@ Julia compiles and uses its own copy of OpenBLAS, with threads currently capped
 
 Modifying OpenBLAS settings or compiling Julia with a different BLAS library, eg [Intel MKL](https://software.intel.com/en-us/mkl), may provide performance improvements. You can use [MKL.jl](https://github.com/JuliaComputing/MKL.jl), a package that makes Julia's linear algebra use Intel MKL BLAS and LAPACK instead of OpenBLAS, or search the discussion forum for suggestions on how to set this up manually. Note that Intel MKL cannot be bundled with Julia, as it is not open source.
 
+## Computing cluster
+
+### How do I manage precompilation caches in distributed file systems?
+
+When using `julia` in high-performance computing (HPC) facilities, invoking
+_n_ `julia` processes simultaneously creates at most _n_ temporary copies of
+precompilation cache files. If this is an issue (slow and/or small distributed
+file system), you may:
+
+1. Use `julia` with `--compiled-modules=no` flag to turn off precompilation.
+2. Configure a private writable depot using `pushfirst!(DEPOT_PATH, private_path)`
+ where `private_path` is a path unique to this `julia` process. This
+ can also be done by setting environment variable `JULIA_DEPOT_PATH` to
+ `$private_path:$HOME/.julia`.
+3. Create a symlink from `~/.julia/compiled` to a directory in a scratch space.
+
 ## Julia Releases
 
 ### Do I want to use the Stable, LTS, or nightly version of Julia?

src/dump.c

@@ -2096,11 +2096,10 @@ JL_DLLEXPORT void jl_init_restored_modules(jl_array_t *init_order)
 JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
 {
  JL_TIMING(SAVE_MODULE);
- char *tmpfname = strcat(strcpy((char *) alloca(strlen(fname)+8), fname), ".XXXXXX");
  ios_t f;
  jl_array_t *mod_array = NULL, *udeps = NULL;
- if (ios_mkstemp(&f, tmpfname) == NULL) {
- jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", tmpfname);
+ if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
+ jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname);
  return 1;
  }
  JL_GC_PUSH2(&mod_array, &udeps);
@@ -2213,12 +2212,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
  }
  write_int32(&f, 0); // mark the end of the source text
  ios_close(&f);
-
  JL_GC_POP();
- if (jl_fs_rename(tmpfname, fname) < 0) {
- jl_printf(JL_STDERR, "Cannot write cache file \"%s\".\n", fname);
- return 1;
- }
 
  return 0;
 }