Add .unpack = false and --no-unpack to build.zig.zon and zig fetch

closes ziglang#17895

Enhances zig with the ability to fetch a dependency of any file type.

src/Package/Fetch.zig

@@ -33,6 +33,7 @@ location_tok: std.zig.Ast.TokenIndex,
 hash_tok: std.zig.Ast.TokenIndex,
 name_tok: std.zig.Ast.TokenIndex,
 lazy_status: LazyStatus,
+unpack: bool,
 parent_package_root: Cache.Path,
 parent_manifest_ast: ?*const std.zig.Ast,
 prog_node: std.Progress.Node,
@@ -345,12 +346,12 @@ pub fn run(f: *Fetch) RunError!void {
         .path_or_url => |path_or_url| {
             if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| {
                 var resource: Resource = .{ .dir = dir };
-                return f.runResource(path_or_url, &resource, null);
+                return f.runResource(path_or_url, &resource, null, f.unpack);
             } else |dir_err| {
                 const file_err = if (dir_err == error.NotDir) e: {
                     if (fs.cwd().openFile(path_or_url, .{})) |file| {
                         var resource: Resource = .{ .file = file };
-                        return f.runResource(path_or_url, &resource, null);
+                        return f.runResource(path_or_url, &resource, null, f.unpack);
                     } else |err| break :e err;
                 } else dir_err;
 
@@ -362,7 +363,7 @@ pub fn run(f: *Fetch) RunError!void {
                 };
                 var server_header_buffer: [header_buffer_size]u8 = undefined;
                 var resource = try f.initResource(uri, &server_header_buffer);
-                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);
+                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, f.unpack);
             }
         },
     };
@@ -424,7 +425,7 @@ pub fn run(f: *Fetch) RunError!void {
     );
     var server_header_buffer: [header_buffer_size]u8 = undefined;
     var resource = try f.initResource(uri, &server_header_buffer);
-    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);
+    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, f.unpack);
 }
 
 pub fn deinit(f: *Fetch) void {
@@ -438,6 +439,7 @@ fn runResource(
     uri_path: []const u8,
     resource: *Resource,
     remote_hash: ?Manifest.MultiHashHexDigest,
+    unpack: bool,
 ) RunError!void {
     defer resource.deinit();
     const arena = f.arena.allocator();
@@ -468,7 +470,7 @@ fn runResource(
         defer tmp_directory.handle.close();
 
         // Fetch and unpack a resource into a temporary directory.
-        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
+        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory, unpack);
 
         var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
 
@@ -712,6 +714,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                 .hash_tok = dep.hash_tok,
                 .name_tok = dep.name_tok,
                 .lazy_status = if (dep.lazy) .available else .eager,
+                .unpack = dep.unpack,
                 .parent_package_root = f.package_root,
                 .parent_manifest_ast = &f.manifest_ast,
                 .prog_node = f.prog_node,
@@ -915,9 +918,10 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
 
     if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
         const path = try uri.path.toRawMaybeAlloc(arena);
-        return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| {
+        const path_relative = std.mem.trimLeft(u8, path, "/");
+        return .{ .file = f.parent_package_root.openFile(path_relative, .{}) catch |err| {
             return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
-                f.parent_package_root, path, @errorName(err),
+                f.parent_package_root, path_relative, @errorName(err),
             }));
         } };
     }
@@ -1055,8 +1059,35 @@ fn unpackResource(
     resource: *Resource,
     uri_path: []const u8,
     tmp_directory: Cache.Directory,
+    unpack: bool,
 ) RunError!UnpackResult {
     const eb = &f.error_bundle;
+
+    if (!unpack) {
+        const basename = std.fs.path.basename(uri_path)
+        var out_file = tmp_directory.handle.createFile(
+            basename,
+            .{},
+        ) catch |err| return f.fail(f.location_tok, try eb.printString(
+            "failed to create tmp file: {s}",
+            .{@errorName(err)},
+        ));
+        defer out_file.close();
+        var buf: [std.mem.page_size]u8 = undefined;
+        while (true) {
+            const len = resource.reader().readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "read stream failed: {s}",
+                .{@errorName(err)},
+            ));
+            if (len == 0) break;
+            out_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "write temporary file failed: {s}",
+                .{@errorName(err)},
+            ));
+        }
+        return .{};
+    }
+
     const file_type = switch (resource.*) {
         .file => FileType.fromPath(uri_path) orelse
             return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),

src/Package/Manifest.zig

@@ -25,6 +25,7 @@ pub const Dependency = struct {
     node: Ast.Node.Index,
     name_tok: Ast.TokenIndex,
     lazy: bool,
+    unpack: bool,
 
     pub const Location = union(enum) {
         url: []const u8,
@@ -302,6 +303,7 @@ const Parse = struct {
             .node = node,
             .name_tok = 0,
             .lazy = false,
+            .unpack = true,
         };
         var has_location = false;
 
@@ -350,6 +352,12 @@ const Parse = struct {
                     error.ParseFailure => continue,
                     else => |e| return e,
                 };
+            } else if (mem.eql(u8, field_name, "unpack")) {
+                dep.unpack = parseBool(p, field_init) catch |err| switch (err) {
+                    error.ParseFailure => continue,
+                    else => |e| return e,
+                };
+                if (dep.unpack) return fail(p, main_tokens[field_init], "unpack cannot be set to true, omit it instead", .{});
             } else {
                 // Ignore unknown fields so that we can add fields in future zig
                 // versions without breaking older zig versions.

src/main.zig

@@ -5073,6 +5073,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .hash_tok = 0,
                     .name_tok = 0,
                     .lazy_status = .eager,
+                    // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                    // TODO: double check this is correct
+                    .unpack = true,
                     .parent_package_root = build_mod.root,
                     .parent_manifest_ast = null,
                     .prog_node = fetch_prog_node,
@@ -6814,6 +6817,7 @@ const usage_fetch =
     \\  --save=[name]                 Add the fetched package to build.zig.zon as name
     \\  --save-exact                  Add the fetched package to build.zig.zon, storing the URL verbatim
     \\  --save-exact=[name]           Add the fetched package to build.zig.zon as name, storing the URL verbatim
+    \\  --no-unpack                   Don't unpack the package
     \\
 ;
 
@@ -6835,6 +6839,7 @@ fn cmdFetch(
         yes: ?[]const u8,
         exact: ?[]const u8,
     } = .no;
+    var unpack = true;
 
     {
         var i: usize = 0;
@@ -6859,6 +6864,8 @@ fn cmdFetch(
                     save = .{ .exact = null };
                 } else if (mem.startsWith(u8, arg, "--save-exact=")) {
                     save = .{ .exact = arg["--save-exact=".len..] };
+                } else if (mem.eql(u8, arg, "--no-unpack")) {
+                    unpack = false;
                 } else {
                     fatal("unrecognized parameter: '{s}'", .{arg});
                 }
@@ -6913,6 +6920,7 @@ fn cmdFetch(
         .hash_tok = 0,
         .name_tok = 0,
         .lazy_status = .eager,
+        .unpack = unpack,
         .parent_package_root = undefined,
         .parent_manifest_ast = null,
         .prog_node = root_prog_node,

test/standalone/build.zig.zon

@@ -69,6 +69,9 @@
         .@"extern" = .{
             .path = "extern",
         },
+        .fetch = .{
+            .path = "fetch",
+        },
         .dep_diamond = .{
             .path = "dep_diamond",
         },

test/standalone/fetch/build.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    b.default_step = b.step("test", "Test it");
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "--build-file",
+        });
+        run.addFileArg(b.path("example/build.zig"));
+        b.default_step.dependOn(&run.step);
+    }
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "--build-file",
+        });
+        run.addFileArg(b.path("unpacktrue/build.zig"));
+        run.addCheck(.{ .expect_stderr_match = "error: unpack cannot be set to true, omit it instead" });
+        b.default_step.dependOn(&run.step);
+    }
+}

test/standalone/fetch/example/build.zig

@@ -0,0 +1,5 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    _ = b;
+}

test/standalone/fetch/example/build.zig.zon

@@ -0,0 +1,16 @@
+.{
+    .name = "fetch",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://localhost/example_dep_file",
+            .hash = "122008dbadc3bbdeaf08a92bcd337c2dc994efc3c8c1e9a7f59a23901df660e89d8f",
+            .unpack = false,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "example_dep_file",
+    },
+}

test/standalone/fetch/example/example_dep_file

@@ -0,0 +1 @@
+This is an example file.

test/standalone/fetch/unpacktrue/build.zig

@@ -0,0 +1,3 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {}

test/standalone/fetch/unpacktrue/build.zig.zon

@@ -0,0 +1,15 @@
+.{
+    .name = "unpacktrue",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://localhost/bar",
+            .hash = "1220f3b02ca452c26a96b48d2912b7fc907bef8d0b85c2e8f7e4a5c8bd95cdbfbae6",
+            .unpack = true,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+    },
+}