provide Base.crc32c checksum and use it in checking cache staleness

LICENSE.md

@@ -29,6 +29,7 @@ for exceptions.
 
 Julia includes code from the following projects, which have their own licenses:
 
+- [crc32c.c](http://stackoverflow.com/questions/17645167/implementing-sse-4-2s-crc32c-in-software) (CRC-32c checksum code by Mark Adler) [[ZLib](https://opensource.org/licenses/Zlib)].
 - [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed.
 - [LLVM](http://llvm.org/releases/3.7.0/LICENSE.TXT) (for parts of src/jitlayers.cpp and src/disasm.cpp) [BSD-3, effectively]
 - [MUSL](http://git.musl-libc.org/cgit/musl/tree/COPYRIGHT) (for getopt implementation on Windows) [MIT]

base/loading.jl

@@ -212,15 +212,17 @@ end
 # to synchronize multiple tasks trying to import/using something
 const package_locks = Dict{Symbol,Condition}()
 
+cache_checksum(path) = isfile(path) ? crc32c(read(path)) : 0x00000000
+
 # used to optionally track dependencies when requiring a module:
-const _require_dependencies = Tuple{String,Float64}[]
+const _require_dependencies = Tuple{String,Float64,UInt32}[]
 const _track_dependencies = [false]
 function _include_dependency(_path::AbstractString)
     prev = source_path(nothing)
     path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path)
     if myid() == 1 && _track_dependencies[1]
         apath = abspath(path)
-        push!(_require_dependencies, (apath, mtime(apath)))
+        push!(_require_dependencies, (apath, mtime(apath), cache_checksum(path)))
     end
     return path, prev
 end
@@ -513,7 +515,7 @@ isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_read_verify_header, Cint, (Pt
 
 function cache_dependencies(f::IO)
     modules = Tuple{Symbol,UInt64}[]
-    files = Tuple{String,Float64}[]
+    files = Tuple{String,Float64,UInt32}[]
     while true
         n = ntoh(read(f, Int32))
         n == 0 && break
@@ -525,7 +527,7 @@ function cache_dependencies(f::IO)
     while true
         n = ntoh(read(f, Int32))
         n == 0 && break
-        push!(files, (String(read(f, n)), ntoh(read(f, Float64))))
+        push!(files, (String(read(f, n)), ntoh(read(f, Float64)), ntoh(read(f, UInt32))))
     end
     return modules, files
 end
@@ -550,9 +552,11 @@ function stale_cachefile(modpath, cachefile)
         if files[1][1] != modpath
             return true # cache file was compiled from a different path
         end
-        for (f,ftime) in files
-            # Issue #13606: compensate for Docker images rounding mtimes
-            if mtime(f) ∉ (ftime, floor(ftime))
+        for (f,ftime,checksum) in files
+            # mtime check of ftime and floor(ftime) is due to issue #13606:
+            # compensate for Docker images rounding mtimes
+            if mtime(f) ∉ (ftime, floor(ftime)) &&
+               cache_checksum(f) != checksum
                 return true
             end
         end

base/util.jl

@@ -514,3 +514,15 @@ if is_windows()
     end
 
 end
+
+"""
+    crc32c(data, crc::UInt32=0x00000000)
+
+Compute the CRC-32c checksum of the given `data`, which can be
+an `Array{UInt8}` or a `String`.  Optionally, you can pass
+a starting `crc` integer to be mixed in with the checksum.
+(Technically, a little-endian checksum is computed.)
+"""
+function crc32c end
+crc32c(a::Array{UInt8}, crc::UInt32=0x00000000) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, sizeof(a))
+crc32c(s::String, crc::UInt32=0x00000000) = crc32c(s.data, crc)

contrib/add_license_to_files.jl

@@ -57,6 +57,7 @@ const skipfiles = [
     "../src/support/tzfile.h",
     "../src/support/utf8.c",
     "../test/perf/micro/randmtzig.c",
+    "../src/support/crc32c.c",
 ]
 
 const ext_prefix = Dict([

src/dump.c

@@ -1084,7 +1084,7 @@ static void write_mod_list(ios_t *s)
 }
 
 // "magic" string and version header of .ji file
-static const int JI_FORMAT_VERSION = 2;
+static const int JI_FORMAT_VERSION = 3;
 static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature
 static const uint16_t BOM = 0xFEFF; // byte-order marker
 static void write_header(ios_t *s)
@@ -1101,8 +1101,8 @@ static void write_header(ios_t *s)
     ios_write(s, commit, strlen(commit)+1);
 }
 
-// serialize the global _require_dependencies array of pathnames that
-// are include depenencies
+// serialize the global _require_dependencies array of
+// (pathname, timestamp, checksum) that are include dependencies
 static void write_dependency_list(ios_t *s)
 {
     size_t total_size = 0;
@@ -1124,7 +1124,7 @@ static void write_dependency_list(ios_t *s)
         for (size_t i=0; i < l; i++) {
             jl_value_t *dep = jl_fieldref(jl_array_ptr_ref(udeps, i), 0);
             size_t slen = jl_string_len(dep);
-            total_size += 4 + slen + 8;
+            total_size += 4 + slen + 8 + 4;
         }
         total_size += 4;
     }
@@ -1140,6 +1140,7 @@ static void write_dependency_list(ios_t *s)
             write_int32(s, slen);
             ios_write(s, jl_string_data(dep), slen);
             write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 1)));
+            write_int32(s, jl_unbox_int32(jl_fieldref(deptuple, 2)));
         }
         write_int32(s, 0); // terminator, for ease of reading
     }

src/support/Makefile

@@ -7,7 +7,7 @@ override CXXFLAGS += $(JCXXFLAGS)
 override CPPFLAGS += $(JCPPFLAGS)
 
 SRCS := hashing timefuncs ptrhash operators utf8 ios htable bitvector \
-	int2str libsupportinit arraylist strtod
+	int2str libsupportinit arraylist strtod crc32c
 ifeq ($(OS),WINNT)
 SRCS += asprintf wsasocketpair strptime
 ifeq ($(ARCH),i686)