From 55806e3f0ad371c2b6d23597cb09ebf34c3d32c5 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 3 Feb 2024 06:10:47 +0100 Subject: [PATCH] Compilation: detect and regenerate missing cache artifacts While it is not clear why artifacts are occasionally missing for which the manifest remains intact, regardless of how it happens, the cache should be at least somewhat resilient to kills, crashes, power loss, etc. affecting compilation processes anyway. Workaround #18763 --- lib/std/Build/Cache.zig | 6 +++--- src/Compilation.zig | 24 +++++++++++++++++++++--- stage1/wasi.c | 17 +++++++++++++---- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 4ae90c74a7ec..d1567bf2d357 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -482,11 +482,11 @@ pub const Manifest = struct { self.want_refresh_timestamp = true; - while (true) { + const input_file_count = self.files.items.len; + while (true) : (self.unhit(bin_digest, input_file_count)) { const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max); defer gpa.free(file_contents); - const input_file_count = self.files.items.len; var any_file_changed = false; var line_iter = mem.tokenizeScalar(u8, file_contents, '\n'); var idx: usize = 0; @@ -914,7 +914,7 @@ pub const Manifest = struct { self.have_exclusive_lock = false; } - fn upgradeToExclusiveLock(self: *Manifest) !bool { + pub fn upgradeToExclusiveLock(self: *Manifest) !bool { if (self.have_exclusive_lock) return false; assert(self.manifest_file != null); diff --git a/src/Compilation.zig b/src/Compilation.zig index beb2c8cf9970..cc21b79912b3 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2053,6 +2053,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void whole.cache_manifest = &man; try addNonIncrementalStuffToCacheManifest(comp, arena, &man); + const bin_digest = man.hash.peekBin(); + const input_file_count = man.files.items.len; const is_hit = man.hit() catch |err| { const i = man.failed_file_index orelse return err; const pp = man.files.items[i].prefixed_path orelse return err; @@ -2063,12 +2065,28 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void .{ prefix, pp.sub_path, @errorName(err) }, ); }; - if (is_hit) { - comp.last_update_was_cache_hit = true; - log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); + if (is_hit) unhit: { const digest = man.final(); comp.wholeCacheModeSetBinFilePath(whole, &digest); + for ([_]?[]u8{ + whole.bin_sub_path, + whole.implib_sub_path, + whole.docs_sub_path, + }) |maybe_sub_path| if (maybe_sub_path) |sub_path| { + comp.local_cache_directory.handle.access(sub_path, .{}) catch |err| { + log.debug("CacheMode.whole missing artifact for {s}: {s}", .{ + comp.root_name, + @errorName(err), + }); + man.unhit(bin_digest, input_file_count); + assert(try man.upgradeToExclusiveLock()); + break :unhit; + }; + }; + + comp.last_update_was_cache_hit = true; + log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); assert(whole.lock == null); whole.lock = man.toOwnedLock(); diff --git a/stage1/wasi.c b/stage1/wasi.c index b6896b9ceb43..bbbaac488bc0 100644 --- a/stage1/wasi.c +++ b/stage1/wasi.c @@ -203,7 +203,9 @@ struct DirEntry { static uint32_t fd_len; static struct FileDescriptor { uint32_t de; + enum wasi_fdflags fdflags; FILE *stream; + uint64_t fs_rights_inheriting; } *fds; static void *dupe(const void *data, size_t len) { @@ -707,13 +709,17 @@ uint32_t wasi_snapshot_preview1_path_filestat_get(uint32_t fd, uint32_t flags, u } uint32_t wasi_snapshot_preview1_fd_fdstat_get(uint32_t fd, uint32_t res_fdstat) { - (void)fd; - (void)res_fdstat; + uint8_t *const m = *wasm_memory; + struct wasi_fdstat *res_fdstat_ptr = (struct wasi_fdstat *)&m[res_fdstat]; #if LOG_TRACE fprintf(stderr, "wasi_snapshot_preview1_fd_fdstat_get(%u)\n", fd); #endif - panic("unimplemented"); + if (fd >= fd_len || fds[fd].de >= de_len) return wasi_errno_badf; + res_fdstat_ptr->fs_filetype = des[fds[fd].de].filetype; + res_fdstat_ptr->fs_flags = fds[fd].fdflags; + res_fdstat_ptr->padding = 0; + res_fdstat_ptr->fs_rights_inheriting = fds[fd].fs_rights_inheriting; return wasi_errno_success; } @@ -770,7 +776,6 @@ uint32_t wasi_snapshot_preview1_fd_write(uint32_t fd, uint32_t iovs, uint32_t io uint32_t wasi_snapshot_preview1_path_open(uint32_t fd, uint32_t dirflags, uint32_t path, uint32_t path_len, uint32_t oflags, uint64_t fs_rights_base, uint64_t fs_rights_inheriting, uint32_t fdflags, uint32_t res_fd) { uint8_t *const m = *wasm_memory; const char *path_ptr = (const char *)&m[path]; - (void)fs_rights_inheriting; uint32_t *res_fd_ptr = (uint32_t *)&m[res_fd]; #if LOG_TRACE fprintf(stderr, "wasi_snapshot_preview1_path_open(%u, 0x%X, \"%.*s\", 0x%X, 0x%llX, 0x%llX, 0x%X)\n", fd, dirflags, (int)path_len, path_ptr, oflags, (unsigned long long)fs_rights_base, (unsigned long long)fs_rights_inheriting, fdflags); @@ -792,10 +797,12 @@ uint32_t wasi_snapshot_preview1_path_open(uint32_t fd, uint32_t dirflags, uint32 fds = new_fds; fds[fd_len].de = de; + fds[fd_len].fdflags = fdflags; switch (des[de].filetype) { case wasi_filetype_directory: fds[fd_len].stream = NULL; break; default: panic("unimplemented"); } + fds[fd_len].fs_rights_inheriting = fs_rights_inheriting; #if LOG_TRACE fprintf(stderr, "fd = %u\n", fd_len); @@ -855,7 +862,9 @@ uint32_t wasi_snapshot_preview1_path_open(uint32_t fd, uint32_t dirflags, uint32 fprintf(stderr, "fd = %u\n", fd_len); #endif fds[fd_len].de = de; + fds[fd_len].fdflags = fdflags; fds[fd_len].stream = stream; + fds[fd_len].fs_rights_inheriting = fs_rights_inheriting; *res_fd_ptr = fd_len; fd_len += 1; return wasi_errno_success;