diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 6c8ba75730f2..955b3cf96196 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2148,6 +2148,9 @@ pub const LazyPath = union(enum) { /// Applied after `up`. sub_path: []const u8 = "", + + /// If true, the file is hashed only on the suffix, not the full absolute path + content_hashed: bool = false, }, /// An absolute path or a path relative to the current working directory of @@ -2339,6 +2342,7 @@ pub const LazyPath = union(enum) { .file = gen.file, .up = gen.up, .sub_path = b.dupePath(gen.sub_path), + .content_hashed = gen.content_hashed, } }, .dependency => |dep| .{ .dependency = dep }, }; diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 08da13c8aa4b..6669a2d54583 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -357,7 +357,7 @@ pub const Manifest = struct { /// ``` /// var file_contents = cache_hash.files.keys()[file_index].contents.?; /// ``` - pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { + pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize, content_hashed: bool) !usize { assert(self.manifest_file == null); const gpa = self.cache.gpa; @@ -378,8 +378,19 @@ pub const Manifest = struct { .bin_digest = undefined, }; - self.hash.add(prefixed_path.prefix); - self.hash.addBytes(prefixed_path.sub_path); + const content_prefix = "zig-cache" ++ std.fs.path.sep_str ++ "o"; + // if a file is hashed based on content, we want to skip the zig-build/o/{HASH}/ part + // as the HASH will represent the content of a previous step + // but keep the suffix as it might be relevant to a custom command + if (content_hashed and prefixed_path.sub_path.len > content_prefix.len and + mem.eql(u8, prefixed_path.sub_path[0..content_prefix.len], content_prefix)) + { + self.hash.add(@as(u8, '?')); + self.hash.addBytes(prefixed_path.sub_path[content_prefix.len + hex_digest_len + 1 ..]); + } else { + self.hash.add(prefixed_path.prefix); + self.hash.addBytes(prefixed_path.sub_path); + } return gop.index; } @@ -387,13 +398,13 @@ pub const Manifest = struct { pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { self.hash.add(optional_file_path != null); const file_path = optional_file_path orelse return; - _ = try self.addFile(file_path, null); + _ = try self.addFile(file_path, null, false); } pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void { self.hash.add(list_of_files.len); for (list_of_files) |file_path| { - _ = try self.addFile(file_path, null); + _ = try self.addFile(file_path, null, false); } } @@ -506,16 +517,14 @@ pub const Manifest = struct { if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; if (file_path.len == 0) return error.InvalidFormat; + const prefixed_path: PrefixedPath = .{ + .prefix = prefix, + .sub_path = file_path, // expires with file_contents + }; const cache_hash_file = f: { - const prefixed_path: PrefixedPath = .{ - .prefix = prefix, - .sub_path = file_path, // expires with file_contents - }; if (idx < input_file_count) { const file = &self.files.keys()[idx]; - if (!file.prefixed_path.eql(prefixed_path)) - return error.InvalidFormat; file.stat = .{ .size = stat_size, @@ -561,11 +570,13 @@ pub const Manifest = struct { self.failed_file_index = idx; return err; }; + + const name_match = pp.eql(prefixed_path); const size_match = actual_stat.size == cache_hash_file.stat.size; const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime; const inode_match = actual_stat.inode == cache_hash_file.stat.inode; - if (!size_match or !mtime_match or !inode_match) { + if (!name_match or !size_match or !mtime_match or !inode_match) { self.manifest_dirty = true; cache_hash_file.stat = .{ @@ -1078,7 +1089,7 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); + _ = try ch.addFile(temp_file, null, false); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit()); @@ -1093,7 +1104,7 @@ test "cache file and then recall it" { ch.hash.add(true); ch.hash.add(@as(u16, 1234)); ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); + _ = try ch.addFile(temp_file, null, false); // Cache hit! We just "built" the same file try testing.expect(try ch.hit()); @@ -1144,7 +1155,7 @@ test "check that changing a file makes cache fail" { defer ch.deinit(); ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); + const temp_file_idx = try ch.addFile(temp_file, 100, false); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit()); @@ -1163,7 +1174,7 @@ test "check that changing a file makes cache fail" { defer ch.deinit(); ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); + const temp_file_idx = try ch.addFile(temp_file, 100, false); // A file that we depend on has been updated, so the cache should not contain an entry for it try testing.expectEqual(false, try ch.hit()); @@ -1267,7 +1278,7 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); + _ = try ch.addFile(temp_file1, null, false); // There should be nothing in the cache try testing.expectEqual(false, try ch.hit()); @@ -1282,7 +1293,7 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); + _ = try ch.addFile(temp_file1, null, false); try testing.expect(try ch.hit()); digest2 = ch.final(); @@ -1305,7 +1316,7 @@ test "Manifest with files added after initial hash work" { defer ch.deinit(); ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); + _ = try ch.addFile(temp_file1, null, false); // A file that we depend on has been updated, so the cache should not contain an entry for it try testing.expectEqual(false, try ch.hit()); diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 515736dbc117..822b0a2ce3a7 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -102,7 +102,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { man.hash.add(@as(u32, 0xe18b7baf)); const full_src_path = objcopy.input_file.getPath2(b, step); - _ = try man.addFile(full_src_path, null); + _ = try man.addFile(full_src_path, null, false); man.hash.addOptionalListOfBytes(objcopy.only_sections); man.hash.addOptional(objcopy.pad_to); man.hash.addOptional(objcopy.format); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index cbec6602c07d..1e407f43ca89 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -232,7 +232,7 @@ pub fn addPrefixedOutputFileArg( run.setName(b.fmt("{s} ({s})", .{ run.step.name, basename })); } - return .{ .generated = .{ .file = &output.generated_file } }; + return .{ .generated = .{ .file = &output.generated_file, .content_hashed = true } }; } /// Appends an input file to the command line arguments. @@ -596,7 +596,11 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { const file_path = file.lazy_path.getPath2(b, step); try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, file_path })); man.hash.addBytes(file.prefix); - _ = try man.addFile(file_path, null); + const content_hashed = switch (file.lazy_path) { + .generated => |gf| gf.content_hashed, + else => false, + }; + _ = try man.addFile(file_path, null, content_hashed); }, .directory_source => |file| { const file_path = file.lazy_path.getPath2(b, step); @@ -613,7 +617,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try argv_list.append(file_path); - _ = try man.addFile(file_path, null); + _ = try man.addFile(file_path, null, false); }, .output_file, .output_directory => |output| { man.hash.addBytes(output.prefix); @@ -637,7 +641,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }, .lazy_path => |lazy_path| { const file_path = lazy_path.getPath2(b, step); - _ = try man.addFile(file_path, null); + _ = try man.addFile(file_path, null, true); }, .none => {}, } @@ -653,10 +657,10 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { hashStdIo(&man.hash, run.stdio); for (run.extra_file_dependencies) |file_path| { - _ = try man.addFile(b.pathFromRoot(file_path), null); + _ = try man.addFile(b.pathFromRoot(file_path), null, false); } for (run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(b, step), null); + _ = try man.addFile(lazy_path.getPath2(b, step), null, false); } if (try step.cacheHit(&man) and !has_side_effects) { diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 401c5b78ece2..464e71bc7a48 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -265,7 +265,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { man.hash.addBytes(bytes); }, .copy => |file_source| { - _ = try man.addFile(file_source.getPath2(b, step), null); + _ = try man.addFile(file_source.getPath2(b, step), null, true); }, } } diff --git a/src/Compilation.zig b/src/Compilation.zig index b83031173311..d5111bc4472f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -861,7 +861,7 @@ pub const cache_helpers = struct { } pub fn hashCSource(self: *Cache.Manifest, c_source: CSourceFile) !void { - _ = try self.addFile(c_source.src_path, null); + _ = try self.addFile(c_source.src_path, null, false); // Hash the extra flags, with special care to call addFile for file parameters. // TODO this logic can likely be improved by utilizing clang_options_data.zig. const file_args = [_][]const u8{"-include"}; @@ -872,7 +872,7 @@ pub const cache_helpers = struct { for (file_args) |file_arg| { if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { arg_i += 1; - _ = try self.addFile(c_source.extra_flags[arg_i], null); + _ = try self.addFile(c_source.extra_flags[arg_i], null, false); } } } @@ -1179,7 +1179,7 @@ fn addModuleTableToCacheHash( }, .files => |man| if (mod.root_src_path.len != 0) { const pkg_zig_file = try mod.root.joinString(arena, mod.root_src_path); - _ = try man.addFile(pkg_zig_file, null); + _ = try man.addFile(pkg_zig_file, null, false); }, } @@ -2403,13 +2403,13 @@ fn addNonIncrementalStuffToCacheManifest( } for (comp.objects) |obj| { - _ = try man.addFile(obj.path, null); + _ = try man.addFile(obj.path, null, false); man.hash.add(obj.must_link); man.hash.add(obj.loption); } for (comp.c_object_table.keys()) |key| { - _ = try man.addFile(key.src.src_path, null); + _ = try man.addFile(key.src.src_path, null, false); man.hash.addOptional(key.src.ext); man.hash.addListOfBytes(key.src.extra_flags); } @@ -2418,11 +2418,11 @@ fn addNonIncrementalStuffToCacheManifest( for (comp.win32_resource_table.keys()) |key| { switch (key.src) { .rc => |rc_src| { - _ = try man.addFile(rc_src.src_path, null); + _ = try man.addFile(rc_src.src_path, null, false); man.hash.addListOfBytes(rc_src.extra_flags); }, .manifest => |manifest_path| { - _ = try man.addFile(manifest_path, null); + _ = try man.addFile(manifest_path, null, false); }, } } @@ -4770,7 +4770,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // the XML data as a RT_MANIFEST resource. This means we can skip preprocessing, // include paths, CLI options, etc. if (win32_resource.src == .manifest) { - _ = try man.addFile(src_path, null); + _ = try man.addFile(src_path, null, false); const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); @@ -4853,7 +4853,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // We now know that we're compiling an .rc file const rc_src = win32_resource.src.rc; - _ = try man.addFile(rc_src.src_path, null); + _ = try man.addFile(rc_src.src_path, null, false); man.hash.addListOfBytes(rc_src.extra_flags); const rc_basename_noext = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; diff --git a/src/glibc.zig b/src/glibc.zig index 5ec0442d6a8e..81fcfbbefcd7 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -691,7 +691,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: *std.Progress.Node) !vo man.hash.add(target_version); const full_abilists_path = try comp.zig_lib_directory.join(arena, &.{abilists_path}); - const abilists_index = try man.addFile(full_abilists_path, abilists_max_size); + const abilists_index = try man.addFile(full_abilists_path, abilists_max_size, false); if (try man.hit()) { const digest = man.final(); diff --git a/src/link.zig b/src/link.zig index 8aec0799e216..28cdcea5ac15 100644 --- a/src/link.zig +++ b/src/link.zig @@ -44,7 +44,7 @@ pub fn hashAddSystemLibs( for (hm.values()) |value| { man.hash.add(value.needed); man.hash.add(value.weak); - if (value.path) |p| _ = try man.addFile(p, null); + if (value.path) |p| _ = try man.addFile(p, null, false); } } @@ -736,16 +736,16 @@ pub const File = struct { base.releaseLock(); for (objects) |obj| { - _ = try man.addFile(obj.path, null); + _ = try man.addFile(obj.path, null, false); man.hash.add(obj.must_link); man.hash.add(obj.loption); } for (comp.c_object_table.keys()) |key| { - _ = try man.addFile(key.status.success.object_path, null); + _ = try man.addFile(key.status.success.object_path, null, false); } if (!build_options.only_core_functionality) { for (comp.win32_resource_table.keys()) |key| { - _ = try man.addFile(key.status.success.res_path, null); + _ = try man.addFile(key.status.success.res_path, null, false); } } try man.addOptionalFile(zcu_obj_path); diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index 47753cbf0135..c27e00262772 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -72,15 +72,15 @@ pub fn linkWithLLD(self: *Coff, arena: Allocator, prog_node: *std.Progress.Node) comptime assert(Compilation.link_hash_implementation_version == 13); for (comp.objects) |obj| { - _ = try man.addFile(obj.path, null); + _ = try man.addFile(obj.path, null, false); man.hash.add(obj.must_link); } for (comp.c_object_table.keys()) |key| { - _ = try man.addFile(key.status.success.object_path, null); + _ = try man.addFile(key.status.success.object_path, null, false); } if (!build_options.only_core_functionality) { for (comp.win32_resource_table.keys()) |key| { - _ = try man.addFile(key.status.success.res_path, null); + _ = try man.addFile(key.status.success.res_path, null, false); } } try man.addOptionalFile(module_obj_path); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 8a3192f93ea3..e78ed42dd5dd 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2217,12 +2217,12 @@ fn linkWithLLD(self: *Elf, arena: Allocator, prog_node: *std.Progress.Node) !voi man.hash.add(self.allow_undefined_version); man.hash.addOptional(self.enable_new_dtags); for (comp.objects) |obj| { - _ = try man.addFile(obj.path, null); + _ = try man.addFile(obj.path, null, false); man.hash.add(obj.must_link); man.hash.add(obj.loption); } for (comp.c_object_table.keys()) |key| { - _ = try man.addFile(key.status.success.object_path, null); + _ = try man.addFile(key.status.success.object_path, null, false); } try man.addOptionalFile(module_obj_path); try man.addOptionalFile(compiler_rt_path); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 4255b298bc0d..5b528d3a84ae 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -165,7 +165,7 @@ pub fn hashAddFrameworks(man: *Cache.Manifest, hm: []const Framework) !void { for (hm) |value| { man.hash.add(value.needed); man.hash.add(value.weak); - _ = try man.addFile(value.path, null); + _ = try man.addFile(value.path, null, false); } } diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index e82395ecea4e..e6fb00930819 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -3378,11 +3378,11 @@ fn linkWithLLD(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) !vo comptime assert(Compilation.link_hash_implementation_version == 13); for (comp.objects) |obj| { - _ = try man.addFile(obj.path, null); + _ = try man.addFile(obj.path, null, false); man.hash.add(obj.must_link); } for (comp.c_object_table.keys()) |key| { - _ = try man.addFile(key.status.success.object_path, null); + _ = try man.addFile(key.status.success.object_path, null, false); } try man.addOptionalFile(module_obj_path); try man.addOptionalFile(compiler_rt_path); diff --git a/src/mingw.zig b/src/mingw.zig index 803c0f936752..f1d8557cc7bd 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -191,7 +191,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void { var man = cache.obtain(); defer man.deinit(); - _ = try man.addFile(def_file_path, null); + _ = try man.addFile(def_file_path, null, false); const final_lib_basename = try std.fmt.allocPrint(comp.gpa, "{s}.lib", .{lib_name}); errdefer comp.gpa.free(final_lib_basename);