diff --git a/NEWS.md b/NEWS.md
index e6076f2933db8..2c77e9e470921 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -18,6 +18,8 @@ Compiler/Runtime improvements
   This allows executable-relative paths to be embedded within executables on all
   platforms, not just MacOS, which the syntax is borrowed from. ([#35627])
 * Constant propogation now occurs through keyword arguments ([#35976])
+* The precompilation cache is now created atomically ([#36416]). Invoking _n_
+  Julia processes simultaneously may create _n_ temporary caches.
 
 Command-line option changes
 ---------------------------
diff --git a/base/loading.jl b/base/loading.jl
index 523cc1d077442..1ca0b0b800f07 100644
--- a/base/loading.jl
+++ b/base/loading.jl
@@ -1301,9 +1301,9 @@ const MAX_NUM_PRECOMPILE_FILES = 10
 function compilecache(pkg::PkgId, path::String)
     # decide where to put the resulting cache file
     cachefile = compilecache_path(pkg)
+    cachepath = dirname(cachefile)
     # prune the directory with cache files
     if pkg.uuid !== nothing
-        cachepath = dirname(cachefile)
         entrypath, entryfile = cache_file_entry(pkg)
         cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
         if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
@@ -1321,20 +1321,34 @@ function compilecache(pkg::PkgId, path::String)
     # run the expression and cache the result
     verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
     @logmsg verbosity "Precompiling $pkg"
-    p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid)
-    if success(p)
-        # append checksum to the end of the .ji file:
-        open(cachefile, "a+") do f
-            write(f, _crc32c(seekstart(f)))
+
+    # create a temporary file in `cachepath` directory, write the cache in it,
+    # write the checksum, _and then_ atomically move the file to `cachefile`.
+    tmppath, tmpio = mktemp(cachepath)
+    local p
+    try
+        close(tmpio)
+        p = create_expr_cache(path, tmppath, concrete_deps, pkg.uuid)
+        if success(p)
+            # append checksum to the end of the .ji file:
+            open(tmppath, "a+") do f
+                write(f, _crc32c(seekstart(f)))
+            end
+            # inherit permission from the source file
+            chmod(tmppath, filemode(path) & 0o777)
+
+            # this is atomic according to POSIX:
+            rename(tmppath, cachefile)
+            return cachefile
         end
-        # inherit permission from the source file
-        chmod(cachefile, filemode(path) & 0o777)
-    elseif p.exitcode == 125
+    finally
+        rm(tmppath, force=true)
+    end
+    if p.exitcode == 125
         return PrecompilableError()
     else
         error("Failed to precompile $pkg to $cachefile.")
     end
-    return cachefile
 end
 
 module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m)
diff --git a/doc/src/manual/faq.md b/doc/src/manual/faq.md
index c3ef33c3d098e..ff55617c157ef 100644
--- a/doc/src/manual/faq.md
+++ b/doc/src/manual/faq.md
@@ -935,6 +935,22 @@ Julia compiles and uses its own copy of OpenBLAS, with threads currently capped
 
 Modifying OpenBLAS settings or compiling Julia with a different BLAS library, eg [Intel MKL](https://software.intel.com/en-us/mkl), may provide performance improvements. You can use [MKL.jl](https://github.com/JuliaComputing/MKL.jl), a package that makes Julia's linear algebra use Intel MKL BLAS and LAPACK instead of OpenBLAS, or search the discussion forum for suggestions on how to set this up manually. Note that Intel MKL cannot be bundled with Julia, as it is not open source.
 
+## Computing cluster
+
+### How do I manage precompilation caches in distributed file systems?
+
+When using `julia` in high-performance computing (HPC) facilities, invoking
+_n_ `julia` processes simultaneously creates at most _n_ temporary copies of
+precompilation cache files. If this is an issue (slow and/or small distributed
+file system), you may:
+
+1. Use `julia` with `--compiled-modules=no` flag to turn off precompilation.
+2. Configure a private writable depot using `pushfirst!(DEPOT_PATH, private_path)`
+   where `private_path` is a path unique to this `julia` process.  This
+   can also be done by setting environment variable `JULIA_DEPOT_PATH` to
+   `$private_path:$HOME/.julia`.
+3. Create a symlink from `~/.julia/compiled` to a directory in a scratch space.
+
 ## Julia Releases
 
 ### Do I want to use the Stable, LTS, or nightly version of Julia?
diff --git a/src/dump.c b/src/dump.c
index 06d7042f86072..5b8c73965e1fd 100644
--- a/src/dump.c
+++ b/src/dump.c
@@ -2096,11 +2096,10 @@ JL_DLLEXPORT void jl_init_restored_modules(jl_array_t *init_order)
 JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
 {
     JL_TIMING(SAVE_MODULE);
-    char *tmpfname = strcat(strcpy((char *) alloca(strlen(fname)+8), fname), ".XXXXXX");
     ios_t f;
     jl_array_t *mod_array = NULL, *udeps = NULL;
-    if (ios_mkstemp(&f, tmpfname) == NULL) {
-        jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", tmpfname);
+    if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
+        jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname);
         return 1;
     }
     JL_GC_PUSH2(&mod_array, &udeps);
@@ -2213,12 +2212,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
     }
     write_int32(&f, 0); // mark the end of the source text
     ios_close(&f);
-
     JL_GC_POP();
-    if (jl_fs_rename(tmpfname, fname) < 0) {
-        jl_printf(JL_STDERR, "Cannot write cache file \"%s\".\n", fname);
-        return 1;
-    }
 
     return 0;
 }